From 51b7cb8fc2458fe1ff3ea902c4b90a00b6ebb45c Mon Sep 17 00:00:00 2001 From: Danny McClanahan Date: Wed, 12 Sep 2018 00:40:07 +0000 Subject: [PATCH 001/796] Convert deprecated fatal_warnings usages to use the new compiler_option_sets from pants 1.9.x! Problem `fatal_warnings` is now deprecated in favor of the more extensible `compiler_option_sets` pants provides in 1.9.x. Solution Convert all `fatal_warnings` arguments to use the new 1.9.x `compiler_option_sets` in all `pants.ini.*`, `BUILD`, and other files. This was done with a combination of `rg` and `sed`: ``` rg -l 'fatal_warnings\s*=\s*.?True' \ | parallel -L50 "sed -r -i -e 's#fatal_warnings(\s*=\s*)\"?True\"?\s*,#compiler_option_sets\1{\"fatal_warnings\"},#g'" \ && rg -l 'fatal_warnings\s*=\s*.?False' \ | parallel -L50 "sed -r -i -e 's#fatal_warnings(\s*=\s*)\"?False\"?\s*,#compiler_option_sets\1\{\},#g'" \ && rg -l 'fatal_warnings.*=' ``` Result Fixes deprecation warnings when running pants on jvm code. JIRA Issues: DPB-10250 TBR=true NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D213208 --- util-app/src/main/java/BUILD | 2 +- util-app/src/main/scala/BUILD | 2 +- util-app/src/test/java/BUILD | 2 +- util-app/src/test/scala/BUILD | 2 +- util-benchmark/src/main/scala/BUILD | 2 +- util-cache-guava/src/main/scala/BUILD | 2 +- util-cache-guava/src/test/java/BUILD | 2 +- util-cache-guava/src/test/scala/BUILD | 2 +- util-cache/src/main/scala/BUILD | 2 +- util-cache/src/test/java/BUILD | 2 +- util-cache/src/test/scala/BUILD | 4 ++-- util-codec/src/main/scala/BUILD | 2 +- util-codec/src/test/scala/BUILD | 2 +- util-collection/src/main/scala/BUILD | 2 +- util-collection/src/test/scala/BUILD | 2 +- util-core/src/main/java/BUILD | 2 +- util-core/src/main/scala/BUILD | 2 +- util-core/src/test/java/BUILD | 2 +- util-core/src/test/scala/BUILD | 2 +- util-function/src/main/java/BUILD | 2 +- util-hashing/src/main/scala/BUILD | 2 +- util-hashing/src/test/scala/BUILD | 2 +- .../src/test/scala/com/twitter/util/capturepoints/BUILD | 2 +- util-jvm/src/main/scala/BUILD | 2 +- util-jvm/src/test/java/BUILD | 2 +- util-jvm/src/test/scala/BUILD | 2 +- util-lint/src/main/scala/BUILD | 2 +- util-lint/src/test/scala/BUILD | 2 +- util-logging/src/main/scala/BUILD | 2 +- util-logging/src/test/java/BUILD | 2 +- util-logging/src/test/scala/BUILD | 2 +- util-reflect/src/main/scala/BUILD | 2 +- util-reflect/src/test/scala/BUILD | 2 +- util-registry/src/main/scala/BUILD | 2 +- util-registry/src/test/java/BUILD | 2 +- util-registry/src/test/scala/BUILD | 2 +- util-security/src/main/scala/BUILD | 2 +- util-security/src/test/java/BUILD | 2 +- util-security/src/test/scala/BUILD | 2 +- util-slf4j-api/src/main/scala/BUILD | 2 +- util-slf4j-api/src/test/java/BUILD | 2 +- util-slf4j-api/src/test/scala/BUILD | 2 +- util-slf4j-jul-bridge/src/main/scala/BUILD | 2 +- util-stats/src/main/scala/BUILD | 2 +- util-stats/src/test/java/BUILD | 2 +- util-stats/src/test/scala/BUILD | 2 +- util-test/src/main/scala/BUILD | 2 +- util-test/src/test/scala/BUILD | 2 +- util-thrift/src/main/scala/BUILD | 2 +- util-thrift/src/test/java/BUILD | 2 +- util-thrift/src/test/scala/BUILD | 2 +- util-tunable/src/main/scala/BUILD | 2 +- .../src/main/scala/com/twitter/util/tunable/linter/BUILD | 2 +- util-tunable/src/test/scala/BUILD | 2 +- util-zk-test/src/main/scala/BUILD | 2 +- util-zk-test/src/test/java/BUILD | 2 +- util-zk-test/src/test/scala/BUILD | 2 +- util-zk/src/main/scala/BUILD | 2 +- util-zk/src/test/scala/BUILD | 2 +- 59 files changed, 60 insertions(+), 60 deletions(-) diff --git a/util-app/src/main/java/BUILD b/util-app/src/main/java/BUILD index 5441291046..7dd64b3ad9 100644 --- a/util-app/src/main/java/BUILD +++ b/util-app/src/main/java/BUILD @@ -1,6 +1,6 @@ java_library( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = artifact( org = "com.twitter", name = "util-app-java", diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index 4a7dd01433..451b5d05fa 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-app", diff --git a/util-app/src/test/java/BUILD b/util-app/src/test/java/BUILD index a1f3950237..84ed3d60db 100644 --- a/util-app/src/test/java/BUILD +++ b/util-app/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index 67a9019c1c..9cbb7f85a3 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index 88ff37f0f1..8843c1f265 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/org/openjdk/jmh:jmh-core", diff --git a/util-cache-guava/src/main/scala/BUILD b/util-cache-guava/src/main/scala/BUILD index 84fdbe5475..47bcf67204 100644 --- a/util-cache-guava/src/main/scala/BUILD +++ b/util-cache-guava/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-cache-guava", diff --git a/util-cache-guava/src/test/java/BUILD b/util-cache-guava/src/test/java/BUILD index f41f341c77..12372efcc7 100644 --- a/util-cache-guava/src/test/java/BUILD +++ b/util-cache-guava/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/google/guava", "3rdparty/jvm/junit", diff --git a/util-cache-guava/src/test/scala/BUILD b/util-cache-guava/src/test/scala/BUILD index 7dc39a0789..3ca6340ec2 100644 --- a/util-cache-guava/src/test/scala/BUILD +++ b/util-cache-guava/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/google/code/findbugs:jsr305", "3rdparty/jvm/com/google/guava", diff --git a/util-cache/src/main/scala/BUILD b/util-cache/src/main/scala/BUILD index f1e5b11ca8..12b85099f2 100644 --- a/util-cache/src/main/scala/BUILD +++ b/util-cache/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-cache", diff --git a/util-cache/src/test/java/BUILD b/util-cache/src/test/java/BUILD index cd15d7536c..a1ef1beb6b 100644 --- a/util-cache/src/test/java/BUILD +++ b/util-cache/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/google/guava", "3rdparty/jvm/junit", diff --git a/util-cache/src/test/scala/BUILD b/util-cache/src/test/scala/BUILD index e2adcffa8a..08ba112d36 100644 --- a/util-cache/src/test/scala/BUILD +++ b/util-cache/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/junit", @@ -17,7 +17,7 @@ scala_library( "com/twitter/cache/AbstractFutureCacheTest.scala", "com/twitter/cache/AbstractLoadingFutureCacheTest.scala", ], - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-codec/src/main/scala/BUILD b/util-codec/src/main/scala/BUILD index d576eacc6b..8655e57304 100644 --- a/util-codec/src/main/scala/BUILD +++ b/util-codec/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-codec", diff --git a/util-codec/src/test/scala/BUILD b/util-codec/src/test/scala/BUILD index 0a758329c1..077f627e3c 100644 --- a/util-codec/src/test/scala/BUILD +++ b/util-codec/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-collection/src/main/scala/BUILD b/util-collection/src/main/scala/BUILD index 70b90ebb47..cfe7877044 100644 --- a/util-collection/src/main/scala/BUILD +++ b/util-collection/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-collection", diff --git a/util-collection/src/test/scala/BUILD b/util-collection/src/test/scala/BUILD index 92e867cbc5..a6e49c881d 100644 --- a/util-collection/src/test/scala/BUILD +++ b/util-collection/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-core/src/main/java/BUILD b/util-core/src/main/java/BUILD index ec624fbc75..20ea336458 100644 --- a/util-core/src/main/java/BUILD +++ b/util-core/src/main/java/BUILD @@ -1,6 +1,6 @@ java_library( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = artifact( org = "com.twitter", name = "util-core-java", diff --git a/util-core/src/main/scala/BUILD b/util-core/src/main/scala/BUILD index d89ef3c23b..abb459964f 100644 --- a/util-core/src/main/scala/BUILD +++ b/util-core/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-core", diff --git a/util-core/src/test/java/BUILD b/util-core/src/test/java/BUILD index 05f296d36e..5bdeb19127 100644 --- a/util-core/src/test/java/BUILD +++ b/util-core/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/junit", diff --git a/util-core/src/test/scala/BUILD b/util-core/src/test/scala/BUILD index 09d037242a..3f0d575739 100644 --- a/util-core/src/test/scala/BUILD +++ b/util-core/src/test/scala/BUILD @@ -8,7 +8,7 @@ junit_tests( # # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` # for this target, we could try enabling this again. - fatal_warnings = False, + compiler_option_sets = {}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-function/src/main/java/BUILD b/util-function/src/main/java/BUILD index 4e3a84dba8..02d4f368a2 100644 --- a/util-function/src/main/java/BUILD +++ b/util-function/src/main/java/BUILD @@ -1,6 +1,6 @@ java_library( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = artifact( org = "com.twitter", name = "util-function-java", diff --git a/util-hashing/src/main/scala/BUILD b/util-hashing/src/main/scala/BUILD index 0a048c193f..3c18e8017c 100644 --- a/util-hashing/src/main/scala/BUILD +++ b/util-hashing/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-hashing", diff --git a/util-hashing/src/test/scala/BUILD b/util-hashing/src/test/scala/BUILD index 80c1ef9ac4..95f83bfa5f 100644 --- a/util-hashing/src/test/scala/BUILD +++ b/util-hashing/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD index 2bcee27eb7..3edc8e9b63 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = globs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, strict_deps = True, dependencies = [ "3rdparty/jvm/org/scalatest", diff --git a/util-jvm/src/main/scala/BUILD b/util-jvm/src/main/scala/BUILD index 2d1e2415fa..d7aba085b4 100644 --- a/util-jvm/src/main/scala/BUILD +++ b/util-jvm/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-jvm", diff --git a/util-jvm/src/test/java/BUILD b/util-jvm/src/test/java/BUILD index 0a56246fcd..f19e7bd14c 100644 --- a/util-jvm/src/test/java/BUILD +++ b/util-jvm/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "util/util-jvm/src/main/scala", diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index ca099b7fb2..181cf7af01 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-lint/src/main/scala/BUILD b/util-lint/src/main/scala/BUILD index 672435d27e..98291863b0 100644 --- a/util-lint/src/main/scala/BUILD +++ b/util-lint/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-lint", diff --git a/util-lint/src/test/scala/BUILD b/util-lint/src/test/scala/BUILD index ef1a07ad6c..5487109e51 100644 --- a/util-lint/src/test/scala/BUILD +++ b/util-lint/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-logging/src/main/scala/BUILD b/util-logging/src/main/scala/BUILD index 092bc18284..2d4741117d 100644 --- a/util-logging/src/main/scala/BUILD +++ b/util-logging/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-logging", diff --git a/util-logging/src/test/java/BUILD b/util-logging/src/test/java/BUILD index 2a9bf3bb10..f1f271ce4d 100644 --- a/util-logging/src/test/java/BUILD +++ b/util-logging/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "util/util-logging/src/main/scala", diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 949edcdc96..61cdfb74f1 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-reflect/src/main/scala/BUILD b/util-reflect/src/main/scala/BUILD index 31c0235a79..86d1d68f26 100644 --- a/util-reflect/src/main/scala/BUILD +++ b/util-reflect/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-reflect", diff --git a/util-reflect/src/test/scala/BUILD b/util-reflect/src/test/scala/BUILD index b944a1382f..df9bea67f6 100644 --- a/util-reflect/src/test/scala/BUILD +++ b/util-reflect/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-registry/src/main/scala/BUILD b/util-registry/src/main/scala/BUILD index 0f12f986e2..52865c8e2f 100644 --- a/util-registry/src/main/scala/BUILD +++ b/util-registry/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-registry", diff --git a/util-registry/src/test/java/BUILD b/util-registry/src/test/java/BUILD index 57b1d46123..c958405e33 100644 --- a/util-registry/src/test/java/BUILD +++ b/util-registry/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index 0f5ffdbacf..9f47327643 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-security/src/main/scala/BUILD b/util-security/src/main/scala/BUILD index bc52977657..8ea39d4af3 100644 --- a/util-security/src/main/scala/BUILD +++ b/util-security/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-security", diff --git a/util-security/src/test/java/BUILD b/util-security/src/test/java/BUILD index 7f674e6c57..e0fc120cf9 100644 --- a/util-security/src/test/java/BUILD +++ b/util-security/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "util/util-core/src/main/scala", diff --git a/util-security/src/test/scala/BUILD b/util-security/src/test/scala/BUILD index e5abb760cf..320dcc0964 100644 --- a/util-security/src/test/scala/BUILD +++ b/util-security/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-slf4j-api/src/main/scala/BUILD b/util-slf4j-api/src/main/scala/BUILD index f22a32ae04..056dfa6aaf 100644 --- a/util-slf4j-api/src/main/scala/BUILD +++ b/util-slf4j-api/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-slf4j", diff --git a/util-slf4j-api/src/test/java/BUILD b/util-slf4j-api/src/test/java/BUILD index bf67c899bd..e415f5b644 100644 --- a/util-slf4j-api/src/test/java/BUILD +++ b/util-slf4j-api/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/slf4j:slf4j-api", diff --git a/util-slf4j-api/src/test/scala/BUILD b/util-slf4j-api/src/test/scala/BUILD index 181d695241..b46087de5f 100644 --- a/util-slf4j-api/src/test/scala/BUILD +++ b/util-slf4j-api/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala", "*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", diff --git a/util-slf4j-jul-bridge/src/main/scala/BUILD b/util-slf4j-jul-bridge/src/main/scala/BUILD index 03616c6c52..34571a74c0 100644 --- a/util-slf4j-jul-bridge/src/main/scala/BUILD +++ b/util-slf4j-jul-bridge/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-slf4j-jul-bridge", diff --git a/util-stats/src/main/scala/BUILD b/util-stats/src/main/scala/BUILD index c32122ce4a..ac704c3c25 100644 --- a/util-stats/src/main/scala/BUILD +++ b/util-stats/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-stats", diff --git a/util-stats/src/test/java/BUILD b/util-stats/src/test/java/BUILD index da4d41aafb..05b139b929 100644 --- a/util-stats/src/test/java/BUILD +++ b/util-stats/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "util/util-core/src/main/scala", diff --git a/util-stats/src/test/scala/BUILD b/util-stats/src/test/scala/BUILD index 77b6c92de3..82321d2d04 100644 --- a/util-stats/src/test/scala/BUILD +++ b/util-stats/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-test/src/main/scala/BUILD b/util-test/src/main/scala/BUILD index c0163b8395..9f4bcaf81d 100644 --- a/util-test/src/main/scala/BUILD +++ b/util-test/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-test", diff --git a/util-test/src/test/scala/BUILD b/util-test/src/test/scala/BUILD index 7d3ae63b7f..70eb5fdb8e 100644 --- a/util-test/src/test/scala/BUILD +++ b/util-test/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-thrift/src/main/scala/BUILD b/util-thrift/src/main/scala/BUILD index 01e5d0a011..1e9084b28b 100644 --- a/util-thrift/src/main/scala/BUILD +++ b/util-thrift/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-thrift", diff --git a/util-thrift/src/test/java/BUILD b/util-thrift/src/test/java/BUILD index 3824246be9..967b1e8f00 100644 --- a/util-thrift/src/test/java/BUILD +++ b/util-thrift/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/org/apache/thrift:libthrift", ], diff --git a/util-thrift/src/test/scala/BUILD b/util-thrift/src/test/scala/BUILD index de9be2e063..625d2299fc 100644 --- a/util-thrift/src/test/scala/BUILD +++ b/util-thrift/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-tunable/src/main/scala/BUILD b/util-tunable/src/main/scala/BUILD index b6dde0cb2b..e5cd7f0aca 100644 --- a/util-tunable/src/main/scala/BUILD +++ b/util-tunable/src/main/scala/BUILD @@ -4,7 +4,7 @@ scala_library( "*.scala", exclude = [rglobs("util/tunable/linter/*")], ), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-tunable", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD index 208d12477e..6a83f5e3de 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD @@ -14,7 +14,7 @@ scala_library( jvm_binary( name = "configuration-linter", main = "com.twitter.util.tunable.linter.ConfigurationLinter", - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ ":scala", ], diff --git a/util-tunable/src/test/scala/BUILD b/util-tunable/src/test/scala/BUILD index c1409da3c6..049d36ab76 100644 --- a/util-tunable/src/test/scala/BUILD +++ b/util-tunable/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-zk-test/src/main/scala/BUILD b/util-zk-test/src/main/scala/BUILD index 3e8171168f..5b001bc546 100644 --- a/util-zk-test/src/main/scala/BUILD +++ b/util-zk-test/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-zk-test", diff --git a/util-zk-test/src/test/java/BUILD b/util-zk-test/src/test/java/BUILD index e21fe2662c..ad490fcffe 100644 --- a/util-zk-test/src/test/java/BUILD +++ b/util-zk-test/src/test/java/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.java"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "util/util-zk-test", diff --git a/util-zk-test/src/test/scala/BUILD b/util-zk-test/src/test/scala/BUILD index ab2be0a286..17cba974a6 100644 --- a/util-zk-test/src/test/scala/BUILD +++ b/util-zk-test/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", diff --git a/util-zk/src/main/scala/BUILD b/util-zk/src/main/scala/BUILD index bc1401a136..655324cde4 100644 --- a/util-zk/src/main/scala/BUILD +++ b/util-zk/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", name = "util-zk", diff --git a/util-zk/src/test/scala/BUILD b/util-zk/src/test/scala/BUILD index d81ef1f44e..8a4fe40163 100644 --- a/util-zk/src/test/scala/BUILD +++ b/util-zk/src/test/scala/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = rglobs("*.scala"), - fatal_warnings = True, + compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", From a4faaa7a6bde6fe52433cad7b006837ed1a73cad Mon Sep 17 00:00:00 2001 From: Seth Tisue Date: Wed, 12 Sep 2018 02:04:19 +0000 Subject: [PATCH 002/796] util-core: tweak sources to compile on JDK 9+ Problem compilation errors on JDK 9+, for example: [error] /Users/tisue/twitter-util/util-core/src/main/scala/com/twitter/io/Buf.scala:873:25: ambiguous reference to overloaded definition, [error] both method position in class ByteBuffer of type (x$1: Int)java.nio.ByteBuffer [error] and method position in class Buffer of type ()Int [error] match expected type ? [error] val limit = dup.position + math.min(until, length) [error] ^ this came up in the Scala 2.12 community build the cause is that the Java folks added some new overloads that scalac considers ambiguous Solution disambiguate by explicitly supplying a single empty argument list Result * downstream projects in the Scala community build become unblocked, improving test coverage of Scala itself * twitter-util becomes better prepared for that eventual far-off day when it is built on JDK 11 instead of JDK 8 Signed-off-by: Jordan Parker Differential Revision: https://phabricator.twitter.biz/D213822 --- util-core/src/main/scala/com/twitter/io/Buf.scala | 8 ++++---- util-core/src/test/scala/com/twitter/io/BufTest.scala | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 2d6869c204..edcde9bf99 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -870,9 +870,9 @@ object Buf { else if (isSliceIdentity(from, until)) this else { val dup = underlying.duplicate() - val limit = dup.position + math.min(until, length) - if (dup.limit > limit) dup.limit(limit) - dup.position(dup.position + from) + val limit = dup.position() + math.min(until, length) + if (dup.limit() > limit) dup.limit(limit) + dup.position(dup.position() + from) new ByteBuffer(dup) } } @@ -886,7 +886,7 @@ object Buf { protected def unsafeByteArrayBuf: Option[Buf.ByteArray] = if (underlying.hasArray) { val array = underlying.array - val begin = underlying.arrayOffset + underlying.position + val begin = underlying.arrayOffset + underlying.position() val end = begin + underlying.remaining Some(new ByteArray(array, begin, end)) } else None diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index e13b76a83b..6688691533 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -763,7 +763,7 @@ class BufTest val bb0 = java.nio.ByteBuffer.wrap(bytes) val Buf.ByteBuffer.Shared(bb1) = Buf.ByteBuffer.Owned(bb0) bb1.position(3) - assert(bb0.position == 0) + assert(bb0.position() == 0) } test("Buf.ByteBuffer.Direct.unapply is unsafe") { @@ -780,7 +780,7 @@ class BufTest val Buf.ByteBuffer(bb1) = Buf.ByteBuffer.Owned(bb0) assert(bb1.isReadOnly) bb1.position(3) - assert(bb0.position == 0) + assert(bb0.position() == 0) } test("ByteArray.coerce(ByteArray)") { From 1f50e03f0e4207e8ca9529d98f652a621e87e35e Mon Sep 17 00:00:00 2001 From: David Rusek Date: Thu, 13 Sep 2018 15:38:49 +0000 Subject: [PATCH 003/796] util-core: Replace custom Reader.chunked implementation with Reader.framed Problem Reader.framed is generic enough to implement a simpler to reason about implementation of a Reader.chunked. Solution Implement Reader.chunked in terms of a framed reader. JIRA Issues: CSL-6967 Differential Revision: https://phabricator.twitter.biz/D213844 --- .../main/scala/com/twitter/io/Reader.scala | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 4050610f3e..1c9a1eefa9 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -3,6 +3,8 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.util._ import java.io.{File, FileInputStream, FileNotFoundException, InputStream} +import scala.annotation.tailrec +import scala.collection.mutable.ListBuffer /** * A Reader represents a stream of `A`s. @@ -38,35 +40,24 @@ object Reader { def empty[A]: Reader[A] = Null.asInstanceOf[Reader[A]] - // See Reader.chunked - private final class Chunked(r: Reader[Buf], chunkSize: Int) - extends Reader[Buf] with (Option[Buf] => Future[Option[Buf]]) { - + // see Reader.chunked + private final class ChunkedFramer(chunkSize: Int) extends (Buf => Seq[Buf]) { require(chunkSize > 0, s"chunkSize should be > 0 but was $chunkSize") - private[this] var state = Buf.Empty - - // We only enter here when state (buffer) is empty. - def apply(in: Option[Buf]): Future[Option[Buf]] = synchronized { - in match { - case Some(b) => - state = b - read(chunkSize) - case None => Future.None - } - } - - def read(n: Int): Future[Option[Buf]] = synchronized { - // flatMap to `this` to prevent allocating - if (state.isEmpty) r.read(Int.MaxValue).flatMap(this) + @tailrec + private def loop(acc: Seq[Buf], in: Buf) : Seq[Buf] = { + if (in.length < chunkSize) acc :+ in else { - val result = state.slice(0, chunkSize) - state = state.slice(chunkSize, Int.MaxValue) - Future.value(Some(result)) + loop( + acc :+ in.slice(0, chunkSize), + in.slice(chunkSize, in.length) + ) } } - def discard(): Unit = r.discard() + def apply(in: Buf): Seq[Buf] = { + loop(ListBuffer(), in) + } } // see Reader.framed @@ -123,7 +114,8 @@ object Reader { * @note The `n` (number of bytes to read) argument on the returned reader is ignored * (`Int.MaxValue` is used instead). */ - def chunked(r: Reader[Buf], chunkSize: Int): Reader[Buf] = new Chunked(r, chunkSize) + def chunked(r: Reader[Buf], chunkSize: Int): Reader[Buf] = + new Framed(r, new ChunkedFramer(chunkSize)) /** * Reader from a Buf. From 94864832a593f5f75c696c8dab913ceba69e81dc Mon Sep 17 00:00:00 2001 From: steve Date: Thu, 13 Sep 2018 15:41:09 +0000 Subject: [PATCH 004/796] util-tunable: Make tunables observable via Vars Problem Tunables are currently "pull" only, to get the value you need to read it from the tunable. There are cases however where it would be useful to be notified when a tunable changes (eg, to update a configuration for APIs that don't support tunables). Solution Add an `asVar` API to Tunable that returns a `Var` representing the tunable's (current and future) value. Differential Revision: https://phabricator.twitter.biz/D211622 --- CHANGELOG.rst | 3 + .../src/main/scala/com/twitter/util/Var.scala | 6 +- .../com/twitter/util/tunable/Tunable.scala | 46 ++++++++++---- .../twitter/util/tunable/TunableTest.scala | 63 +++++++++++++++++++ 4 files changed, 104 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index af32da932f..48cb02192d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -28,6 +28,9 @@ New Features * util-security: Add `NullSslSession` related objects for use with non-existent `SSLSession`s. ``PHAB_ID=D201421`` +* util-tunable: Introducing `Tunable.asVar` that allows observing changes to tunables. + ``PHAB_ID=D211622`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index a26ff4acdc..6945c7c740 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -226,18 +226,20 @@ object Var { v } - private case class Value[T](v: T) extends Var[T] { + private case class Value[T](v: T) extends Var[T] with Extractable[T] { protected def observe(depth: Int, obs: Observer[T]): Closable = { obs.claim(this) obs.publish(this, v, 0) Closable.nop } + + def apply(): T = v } /** * Create a new, constant, v-valued Var. */ - def value[T](v: T): Var[T] = Value(v) + def value[T](v: T): Var[T] with Extractable[T] = Value(v) /** * Collect a collection of Vars into a Var of collection. diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/Tunable.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/Tunable.scala index a65c4e0a88..952c8755a0 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/Tunable.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/Tunable.scala @@ -1,5 +1,7 @@ package com.twitter.util.tunable +import com.twitter.util.Var + /** * A [[Tunable]] is an abstraction for an object that produces a Some(value) or None when applied. * Implementations may enable mutation, such that successive applications of the [[Tunable]] @@ -13,6 +15,12 @@ package com.twitter.util.tunable */ sealed abstract class Tunable[T](val id: String) { self => + /** + * Returns a [[Var]] that observes this [[Tunable]]. Observers of the [[Var]] will be notified + * whenever the value of this [[Tunable]] is updated. + */ + def asVar: Var[Option[T]] + // validate id is not empty if (id.trim.isEmpty) throw new IllegalArgumentException("Tunable id must not be empty") @@ -35,6 +43,12 @@ sealed abstract class Tunable[T](val id: String) { self => */ def orElse(that: Tunable[T]): Tunable[T] = new Tunable[T](id) { + + def asVar: Var[Option[T]] = self.asVar.flatMap { + case Some(_) => self.asVar + case None => that.asVar + } + override def toString: String = s"${self.toString}.orElse(${that.toString})" @@ -49,6 +63,9 @@ sealed abstract class Tunable[T](val id: String) { self => */ def map[A](f: T => A): Tunable[A] = new Tunable[A](id) { + + def asVar: Var[Option[A]] = self.asVar.map(_.map(f)) + def apply(): Option[A] = self().map(f) } } @@ -58,16 +75,17 @@ object Tunable { /** * A [[Tunable]] that always returns `value` when applied. */ - final class Const[T](id: String, private val value: T) extends Tunable[T](id) { - private[this] val SomeValue = Some(value) + final class Const[T](id: String, value: T) extends Tunable[T](id) { + private val holder = Var.value(Some(value)) + + def apply(): Option[T] = holder() - def apply(): Option[T] = - SomeValue + def asVar: Var[Option[T]] = holder } object Const { def unapply[T](tunable: Tunable[T]): Option[T] = tunable match { - case tunable: Tunable.Const[T] => Some(tunable.value) + case tunable: Tunable.Const[T] => tunable.holder() case _ => None } } @@ -78,8 +96,11 @@ object Tunable { def const[T](id: String, value: T): Const[T] = new Const(id, value) private[this] val NoneTunable = new Tunable[Any]("com.twitter.util.tunable.NoneTunable") { + def apply(): Option[Any] = None + + val asVar: Var[Option[Any]] = Var.value(None) } /** @@ -91,30 +112,31 @@ object Tunable { /** * A [[Tunable]] whose value can be changed. Operations are thread-safe. */ - final class Mutable[T] private[tunable] (id: String, @volatile private var _value: Option[T]) + final class Mutable[T] private[tunable] (id: String, _value: Option[T]) extends Tunable[T](id) { + private[this] final val mutableHolder = Var(_value) + + def asVar: Var[Option[T]] = mutableHolder + /** * Set the value of the [[Tunable.Mutable]]. * * Note that setting the value to `null` will result in a value of Some(null) when the * [[Tunable]] is applied. */ - def set(value: T): Unit = - _value = Some(value) + def set(value: T): Unit = mutableHolder.update(Some(value)) /** * Clear the value of the [[Tunable.Mutable]]. Calling `apply` on the [[Tunable.Mutable]] * will produce `None`. */ - def clear(): Unit = - _value = None + def clear(): Unit = mutableHolder.update(None) /** * Get the current value of the [[Tunable.Mutable]] */ - def apply(): Option[T] = - _value + def apply(): Option[T] = mutableHolder() } /** diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/TunableTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/TunableTest.scala index 72cb1d3908..9a5b875ac0 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/TunableTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/TunableTest.scala @@ -170,4 +170,67 @@ class TunableTest extends FunSuite { assert(composed() == None) } + + test("mutable tunables are observable") { + val tunable = Tunable.mutable("id", 5) + val obs = tunable.asVar + + assert(obs.sample().contains(5)) + } + + test("empty mutable tunables are observable") { + val tunable = Tunable.emptyMutable[Int]("id") + val obs = tunable.asVar + + assert(obs.sample().isEmpty) + tunable.set(5) + assert(obs.sample().contains(5)) + } + + test("mutable tunable changes are observed") { + val tunable = Tunable.mutable("id", 5) + val obs = tunable.asVar + + assert(obs.sample().contains(5)) + tunable.set(6) + assert(obs.sample().contains(6)) + } + + test("orElse Tunable is observable if first value is defined") { + val tunable = Tunable.mutable("a", 5) + val orElsed = tunable.orElse(Tunable.const("b", 6)) + + val obs = orElsed.asVar + assert(obs.sample().contains(5)) + tunable.clear() + assert(obs.sample().contains(6)) + } + + test("orElse Tunable is observable if first value is undefined") { + val tunable = Tunable.emptyMutable[Int]("a") + val orElsed = tunable.orElse(Tunable.const("b", 5)) + + val obs = orElsed.asVar + assert(obs.sample().contains(5)) + tunable.set(6) + assert(obs.sample().contains(6)) + } + + test("map of mutable Tunables is observable") { + val tunable = Tunable.mutable("id", 5) + val mapped = tunable.map(_ + 1) + val obs = mapped.asVar + + assert(obs.sample().contains(6)) + tunable.set(6) + assert(obs.sample().contains(7)) + } + + test("map of const Tunables is observable") { + val tunable = Tunable.const("id", 5) + val mapped = tunable.map(_ + 1) + val obs = mapped.asVar + + assert(obs.sample().contains(6)) + } } From 6f70e9d81c404ed9a94f1d095c938fe8e262f3fb Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 13 Sep 2018 22:29:34 +0000 Subject: [PATCH 005/796] util-internal: Update ConfigbusSubscriber after ServerInfo initialized Problem: ConfigbusTunable map won't compose json files defined with $environment or $instanceId. This is because Tunable map is loaded before ServerInfo initialized, and the map does not have the environment info. This is introduced after we added a new tunable in finagle-stats which changed loading lifecycle. Solution: Reload all `StandardTunableMap`s and re-subscribe `ConfigbusTunableMap`s after ServerInfo.intialized(). JIRA Issues: CSL-6801 Differential Revision: https://phabricator.twitter.biz/D208027 --- .../tunable/ServiceLoadedTunableMap.scala | 60 +++++++++++++++---- .../tunable/ServiceLoadedTunableMapTest.scala | 34 ++++++----- 2 files changed, 69 insertions(+), 25 deletions(-) diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/ServiceLoadedTunableMap.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/ServiceLoadedTunableMap.scala index 2b7c88e549..3b5a75e174 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/ServiceLoadedTunableMap.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/ServiceLoadedTunableMap.scala @@ -1,6 +1,9 @@ package com.twitter.util.tunable import com.twitter.app.LoadService +import com.twitter.concurrent.Once +import java.util.concurrent.ConcurrentHashMap +import java.util.function.BiFunction /** * [[TunableMap]] loaded through `com.twitter.app.LoadService`. @@ -16,6 +19,8 @@ trait ServiceLoadedTunableMap extends TunableMap { object ServiceLoadedTunableMap { + private[this] val loadedMaps = new ConcurrentHashMap[String, TunableMap]() + /** * Uses `com.twitter.app.LoadService` to load the [[TunableMap]] for a given `id`. * @@ -24,18 +29,51 @@ object ServiceLoadedTunableMap { * If more than one match is found, an `IllegalStateException` is thrown. */ def apply(id: String): TunableMap = { + init() + loadedMaps.getOrDefault(id, NullTunableMap) + } - val tunableMaps = LoadService[ServiceLoadedTunableMap]().filter(_.id == id).toList - - tunableMaps match { - case Nil => - NullTunableMap - case tunableMap :: Nil => - tunableMap - case _ => - throw new IllegalStateException( - s"Found multiple `ServiceLoadedTunableMap`s for $id: ${tunableMaps.mkString(", ")}" - ) + private[this] val init = Once { + val serviceLoadedTunableMap = LoadService[ServiceLoadedTunableMap]().toList + serviceLoadedTunableMap.foreach(setLoadedMap) + } + + private[this] def setLoadedMap(serviceLoadedTunableMap: ServiceLoadedTunableMap): Unit = { + loadedMaps.compute( + serviceLoadedTunableMap.id, + new BiFunction[String, TunableMap, TunableMap] { + def apply( + id: String, + curr: TunableMap + ): TunableMap = { + if (curr != null) throw new IllegalStateException( + s"Found multiple `ServiceLoadedTunableMap`s for $id: $curr, $serviceLoadedTunableMap" + ) + else serviceLoadedTunableMap + } + } + ) + } + + /** + * Uses `com.twitter.app.LoadService` to load all [[ServiceLoadedTunableMap]]s, + * The subtype `com.twitter.util.tunable.ConfigbusTunableMap`s would re-construct + * and re-subscribe to the ConfigBus. + * + * @note this should be called after ServerInfo.initialized(). + */ + private[twitter] def reloadAll(): Unit = { + val serviceLoadedTunableMaps = LoadService[ServiceLoadedTunableMap]().toList + serviceLoadedTunableMaps.foreach { serviceLoadedTunableMap => + loadedMaps.computeIfPresent( + serviceLoadedTunableMap.id, + new BiFunction[String, TunableMap, TunableMap] { + def apply( + id: String, + curr: TunableMap + ): TunableMap = serviceLoadedTunableMap + } + ) } } } diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/ServiceLoadedTunableMapTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/ServiceLoadedTunableMapTest.scala index 47b150d8c9..e0486392ac 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/ServiceLoadedTunableMapTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/ServiceLoadedTunableMapTest.scala @@ -25,26 +25,32 @@ class ServiceLoadedTunableTestClient2Dup extends ServiceLoadedTunableMap with Tu class ServiceLoadedTunableMapTest extends FunSuite { - test("NullTunableMap returned when no matches") { - val tunableMap = ServiceLoadedTunableMap("Non-existent-id") - assert(tunableMap eq NullTunableMap) - } - - test("TunableMap returned when there is one match for id") { - val tunableMap = ServiceLoadedTunableMap("client1") - - assert(tunableMap.entries.size == 2) - assert(tunableMap(TunableMap.Key[String]("tunableId1"))() == Some("foo")) - assert(tunableMap(TunableMap.Key[Int]("tunableId2"))() == Some(5)) - } - test( "IllegalArgumentException thrown when there is more than one ServiceLoadedTunableMap " + "for a given serviceName/id" ) { - intercept[IllegalStateException] { + val ex = intercept[IllegalStateException] { ServiceLoadedTunableMap("client2") } + assert(ex.getMessage.contains("Found multiple `ServiceLoadedTunableMap`s for client2")) + } + + test("NullTunableMap returned when no matches") { + intercept[IllegalStateException] { + val tunableMap = ServiceLoadedTunableMap("Non-existent-id") + assert(tunableMap eq NullTunableMap) + } + } + + test("TunableMap returned when there is one match for id") { + intercept[IllegalStateException] { + + val tunableMap = ServiceLoadedTunableMap("client1") + + assert(tunableMap.entries.size == 2) + assert(tunableMap(TunableMap.Key[String]("tunableId1"))() == Some("foo")) + assert(tunableMap(TunableMap.Key[Int]("tunableId2"))() == Some(5)) + } } } From d5f34617a073d183297cc78758b5c31bdfd2b3f0 Mon Sep 17 00:00:00 2001 From: Shane Delmore Date: Fri, 14 Sep 2018 02:55:31 +0000 Subject: [PATCH 006/796] util: Enable scalafmt linting and format code Problem / Solution We are standardizing on a set of scalafmt rules. JIRA Issues: EE-1955 Differential Revision: https://phabricator.twitter.biz/D214677 --- .../scala/com/twitter/app/LoadService.scala | 10 +- .../test/scala/com/twitter/app/AppTest.scala | 4 +- .../com/twitter/io/BufConcatBenchmark.scala | 28 +- .../cache/LazilyEvictingCacheTest.scala | 32 +- .../com/twitter/concurrent/AsyncStream.scala | 12 +- .../com/twitter/concurrent/SpoolSource.scala | 4 +- .../com/twitter/conversions/percent.scala | 2 +- .../src/main/scala/com/twitter/io/Buf.scala | 15 +- .../main/scala/com/twitter/io/Reader.scala | 5 +- .../scala/com/twitter/util/Base64Long.scala | 6 +- .../main/scala/com/twitter/util/Future.scala | 11 +- .../twitter/util/FutureEventListener.scala | 1 - .../com/twitter/util/FutureTransformer.scala | 1 - .../main/scala/com/twitter/util/Local.scala | 4691 +++++++++++++++-- .../main/scala/com/twitter/util/Time.scala | 3 +- .../com/twitter/util/WindowedAdder.scala | 6 +- .../com/twitter/concurrent/SpoolTest.scala | 4 +- .../test/scala/com/twitter/io/BufTest.scala | 17 +- .../scala/com/twitter/io/ReaderTest.scala | 3 +- .../scala/com/twitter/util/FutureTest.scala | 129 +- .../TwitterFuturesCapturePointsTest.scala | 14 +- .../main/scala/com/twitter/jvm/Heapster.scala | 4 +- .../com/twitter/logging/ScribeHandlers.scala | 1 + .../util/security/NullSslSession.scala | 6 +- .../util/security/NullSslSessionContext.scala | 4 +- .../twitter/util/security/X509CrlFile.scala | 2 +- .../util/security/NullSslSessionTest.scala | 1 - .../util/security/X509CrlFileTest.scala | 4 +- .../util/logging/Slf4jBridgeUtility.scala | 3 +- .../finagle/stats/CumulativeGauge.scala | 81 +- .../finagle/stats/InMemoryStatsReceiver.scala | 4 +- .../com/twitter/util/tunable/Tunable.scala | 3 +- .../com/twitter/zk/AsyncCallbackPromise.scala | 8 +- .../com/twitter/zk/NativeConnector.scala | 15 +- .../src/main/scala/com/twitter/zk/ZNode.scala | 5 +- .../zk/coordination/ZkAsyncSemaphore.scala | 7 +- .../scala/com/twitter/zk/ZkClientTest.scala | 4 +- 37 files changed, 4667 insertions(+), 483 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/LoadService.scala b/util-app/src/main/scala/com/twitter/app/LoadService.scala index 6ee368356b..44f35dbf89 100644 --- a/util-app/src/main/scala/com/twitter/app/LoadService.scala +++ b/util-app/src/main/scala/com/twitter/app/LoadService.scala @@ -32,6 +32,7 @@ object LoadService { val iface: Class[T], val implementations: Seq[T] ) { + /** * Convenience constructor for binding a single implementation. */ @@ -100,14 +101,17 @@ object LoadService { val prevs = binds.synchronized { if (loaded.contains(binding.iface)) { throw new IllegalStateException( - s"LoadService.apply has already been called for ${binding.iface.getName}.") + s"LoadService.apply has already been called for ${binding.iface.getName}." + ) } binds.put(binding.iface, binding.implementations) } prevs.foreach { ps => bindDupes.put(binding.iface, ()) - log.warning(s"Replaced existing `LoadService.bind` for `${binding.iface.getName}`, " + - s"old='$ps', new='${binding.implementations}'") + log.warning( + s"Replaced existing `LoadService.bind` for `${binding.iface.getName}`, " + + s"old='$ps', new='${binding.implementations}'" + ) } } diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 0e38ecfa58..dc81410b57 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -306,7 +306,7 @@ class AppTest extends FunSuite { test("App: exit functions properly capture non-fatal exceptions") { val app = new ErrorOnExitApp { def main(): Unit = { - onExit{ + onExit { throw new Exception("FORCED ON EXIT") } @@ -345,7 +345,7 @@ class AppTest extends FunSuite { test("App: exit functions properly capture mix of non-fatal and fatal exceptions") { val app = new ErrorOnExitApp { def main(): Unit = { - onExit{ + onExit { throw new Exception("FORCED ON EXIT") } diff --git a/util-benchmark/src/main/scala/com/twitter/io/BufConcatBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/io/BufConcatBenchmark.scala index 6fe0d215ed..659005f53a 100644 --- a/util-benchmark/src/main/scala/com/twitter/io/BufConcatBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/io/BufConcatBenchmark.scala @@ -30,10 +30,22 @@ object BufConcatBenchmark { val buf4: Buf = buf3.concat(buf1) val bufMatrix: Array[(Buf, Buf)] = Array( - (buf1, buf1), (buf1, buf2), (buf1, buf3), (buf1, buf4), - (buf2, buf1), (buf2, buf2), (buf2, buf3), (buf2, buf4), - (buf3, buf1), (buf3, buf2), (buf3, buf3), (buf3, buf4), - (buf4, buf1), (buf4, buf2), (buf4, buf3), (buf4, buf4) + (buf1, buf1), + (buf1, buf2), + (buf1, buf3), + (buf1, buf4), + (buf2, buf1), + (buf2, buf2), + (buf2, buf3), + (buf2, buf4), + (buf3, buf1), + (buf3, buf2), + (buf3, buf3), + (buf3, buf4), + (buf4, buf1), + (buf4, buf2), + (buf4, buf3), + (buf4, buf4) ) @State(Scope.Thread) @@ -69,11 +81,9 @@ class BufConcatBenchmark extends StdBenchAnnotations { class ConcatTwoBufsBenchmark extends StdBenchAnnotations { import BufConcatBenchmark._ - @Param(Array( - "0", "1", "2", "3", - "4", "5", "6", "7", - "8", "9", "10", "11", - "12", "13", "14", "15")) + @Param( + Array("0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15") + ) var index: Int = _ @Benchmark diff --git a/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala index 5e74c224eb..1ee5271278 100644 --- a/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala @@ -85,14 +85,16 @@ class LazilyEvictingCacheTest extends FunSuite with MockitoSugar { var loadCount = 0 val cache = new LoadingFutureCache( - Caffeine.newBuilder().build( - new CacheLoader[String, Future[Int]] { - def load(k: String): Future[Int] = { - loadCount += 1 - Future.value(loadCount) + Caffeine + .newBuilder() + .build( + new CacheLoader[String, Future[Int]] { + def load(k: String): Future[Int] = { + loadCount += 1 + Future.value(loadCount) + } } - } - ) + ) ) val fCache = new LazilyEvictingCache(cache) @@ -117,14 +119,16 @@ class LazilyEvictingCacheTest extends FunSuite with MockitoSugar { var loadCount = 0 val cache = new LoadingFutureCache( - Caffeine.newBuilder().build( - new CacheLoader[String, Future[Int]] { - def load(k: String): Future[Int] = { - loadCount += 1 - Future.value(loadCount) + Caffeine + .newBuilder() + .build( + new CacheLoader[String, Future[Int]] { + def load(k: String): Future[Int] = { + loadCount += 1 + Future.value(loadCount) + } } - } - ) + ) ) val fCache = new LazilyEvictingCache(cache) diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 8858313d96..df22e803c4 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -350,12 +350,12 @@ sealed abstract class AsyncStream[+A] { } /** - * Helper method used to avoid scanLeft being one behind in case of Embed AsyncStream. - * - * scanLeftEmbed, unlike scanLeft, does not return the initial value `z` and is there to - * prevent the Embed case from returning duplicate initial `z` values for scanLeft. - * - */ + * Helper method used to avoid scanLeft being one behind in case of Embed AsyncStream. + * + * scanLeftEmbed, unlike scanLeft, does not return the initial value `z` and is there to + * prevent the Embed case from returning duplicate initial `z` values for scanLeft. + * + */ private def scanLeftEmbed[B](z: B)(f: (B, A) => B): AsyncStream[B] = this match { case Embed(fas) => Embed(fas.map(_.scanLeftEmbed(z)(f))) diff --git a/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala b/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala index f7892edc80..2edce47a4f 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala @@ -106,7 +106,9 @@ class SpoolSource[A](interruptHandler: PartialFunction[Throwable, Unit]) { } @tailrec - private[this] def updatingTailCall(newPromise: Promise[Spool[A]])(f: Promise[Spool[A]] => Unit): Unit = { + private[this] def updatingTailCall( + newPromise: Promise[Spool[A]] + )(f: Promise[Spool[A]] => Unit): Unit = { val currentPromise = promiseRef.get // if the current promise is emptyPromise, then this source has already been closed if (currentPromise ne emptyPromise) { diff --git a/util-core/src/main/scala/com/twitter/conversions/percent.scala b/util-core/src/main/scala/com/twitter/conversions/percent.scala index e49fefb8b8..c04ee79cb3 100644 --- a/util-core/src/main/scala/com/twitter/conversions/percent.scala +++ b/util-core/src/main/scala/com/twitter/conversions/percent.scala @@ -21,7 +21,7 @@ object percent { private val BigDecimal100 = BigDecimal(100.0) - class RichDoublePercent private[conversions](val wrapped: Double) extends AnyVal { + class RichDoublePercent private[conversions] (val wrapped: Double) extends AnyVal { // convert wrapped to BigDecimal to preserve precision when dividing Doubles def percent: Double = if (wrapped.equals(Double.NaN) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index edcde9bf99..97dbcafa19 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -123,7 +123,8 @@ abstract class Buf { outer => ) case _ => Buf.Composite.Impl( - new IndexedThree(left.a, left.b, right), leftLength + right.length + new IndexedThree(left.a, left.b, right), + leftLength + right.length ) } @@ -139,7 +140,8 @@ abstract class Buf { outer => ) case Buf.Composite.Impl(right: IndexedThree, rightLength) => Buf.Composite.Impl( - (new VectorBuilder[Buf] += left.a += left.b += left.c += right.a += right.b += right.c).result(), + (new VectorBuilder[Buf] += left.a += left.b += left.c += right.a += right.b += right.c) + .result(), leftLength + rightLength ) case _ => @@ -154,7 +156,8 @@ abstract class Buf { outer => case Buf.Composite.Impl(right: Vector[Buf], rightLength) => Buf.Composite.Impl(left +: right, left.length + rightLength) case Buf.Composite.Impl(right: IndexedTwo, rightLength) => - Buf.Composite.Impl(new IndexedThree(left, right.a, right.b), left.length + rightLength) + Buf.Composite + .Impl(new IndexedThree(left, right.a, right.b), left.length + rightLength) case Buf.Composite.Impl(right: IndexedThree, rightLength) => Buf.Composite.Impl( (new VectorBuilder[Buf] += left += right.a += right.b += right.c).result(), @@ -557,9 +560,9 @@ object Buf { def unapply(buf: Composite): Option[IndexedSeq[Buf]] = Some(buf.bufs) - /** - * Basic implementation of a [[Buf]] created from n-`Bufs`. - */ + /** + * Basic implementation of a [[Buf]] created from n-`Bufs`. + */ private[Buf] final case class Impl(bufs: IndexedSeq[Buf], protected val computedLength: Int) extends Buf.Composite { diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 1c9a1eefa9..0cb3c420ea 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -45,7 +45,7 @@ object Reader { require(chunkSize > 0, s"chunkSize should be > 0 but was $chunkSize") @tailrec - private def loop(acc: Seq[Buf], in: Buf) : Seq[Buf] = { + private def loop(acc: Seq[Buf], in: Buf): Seq[Buf] = { if (in.length < chunkSize) acc :+ in else { loop( @@ -62,7 +62,8 @@ object Reader { // see Reader.framed private final class Framed(r: Reader[Buf], framer: Buf => Seq[Buf]) - extends Reader[Buf] with (Option[Buf] => Future[Option[Buf]]) { + extends Reader[Buf] + with (Option[Buf] => Future[Option[Buf]]) { private[this] var frames: Seq[Buf] = Seq.empty diff --git a/util-core/src/main/scala/com/twitter/util/Base64Long.scala b/util-core/src/main/scala/com/twitter/util/Base64Long.scala index 59b26e8be3..0d4af19c35 100644 --- a/util-core/src/main/scala/com/twitter/util/Base64Long.scala +++ b/util-core/src/main/scala/com/twitter/util/Base64Long.scala @@ -74,7 +74,11 @@ object Base64Long { * The number is treated as unsigned, so there is never a leading negative sign, and the * representations of negative numbers are larger than positive numbers. */ - def toBase64(builder: StringBuilder, l: Long, alphabet: Int => Char = StandardBase64Alphabet): Unit = { + def toBase64( + builder: StringBuilder, + l: Long, + alphabet: Int => Char = StandardBase64Alphabet + ): Unit = { if (l == 0) { // Special case for zero: Just like in decimal, if the number is // zero, represent it as a single zero digit. diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 9e8f88e9a5..353c9f44bc 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1324,13 +1324,12 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( case class NextThrewException(cause: Throwable) extends IllegalArgumentException("'next' threw an exception", cause) - private class Each[A](next: => Future[A], body: A => Unit) - extends (Try[A] => Future[Nothing]) { + private class Each[A](next: => Future[A], body: A => Unit) extends (Try[A] => Future[Nothing]) { def apply(t: Try[A]): Future[Nothing] = t match { case Return(a) => body(a) go() - case t@Throw(_) => + case t @ Throw(_) => Future.const(t.cast[Nothing]) } def go(): Future[Nothing] = { @@ -1923,7 +1922,7 @@ abstract class Future[+A] extends Awaitable[A] { self => tx match { case Return(_) if !isFirst && get.isReturn => p.setValue(fn(Await.result(Future.this), Await.result(other))) - case t@Throw(_) if isFirst || get.isReturn => + case t @ Throw(_) if isFirst || get.isReturn => p.update(t.cast[C]) case _ => } @@ -1967,7 +1966,9 @@ abstract class Future[+A] extends Awaitable[A] { self => s"Cannot call proxyTo on an already satisfied Promise: ${Await.result(other.liftToTry)}" ) } - respond { res => other.update(res) } + respond { res => + other.update(res) + } } /** diff --git a/util-core/src/main/scala/com/twitter/util/FutureEventListener.scala b/util-core/src/main/scala/com/twitter/util/FutureEventListener.scala index 4b8f2a2699..deb533eb4a 100644 --- a/util-core/src/main/scala/com/twitter/util/FutureEventListener.scala +++ b/util-core/src/main/scala/com/twitter/util/FutureEventListener.scala @@ -23,4 +23,3 @@ trait FutureEventListener[T] { */ def onFailure(cause: Throwable): Unit } - diff --git a/util-core/src/main/scala/com/twitter/util/FutureTransformer.scala b/util-core/src/main/scala/com/twitter/util/FutureTransformer.scala index dcf01653d1..854534f206 100644 --- a/util-core/src/main/scala/com/twitter/util/FutureTransformer.scala +++ b/util-core/src/main/scala/com/twitter/util/FutureTransformer.scala @@ -50,4 +50,3 @@ abstract class FutureTransformer[-A, +B] { */ def handle(throwable: Throwable): B = throw throwable } - diff --git a/util-core/src/main/scala/com/twitter/util/Local.scala b/util-core/src/main/scala/com/twitter/util/Local.scala index 1af139a9c9..6674c1a28f 100644 --- a/util-core/src/main/scala/com/twitter/util/Local.scala +++ b/util-core/src/main/scala/com/twitter/util/Local.scala @@ -27,7 +27,8 @@ object Local { } private final class Context1( - k1: Key, v1: Some[_] + k1: Key, + v1: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -42,8 +43,10 @@ object Local { } private final class Context2( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -129,9 +132,12 @@ object Local { } */ private final class Context3( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -154,10 +160,14 @@ object Local { } private final class Context4( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -183,11 +193,16 @@ object Local { } private final class Context5( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -216,12 +231,18 @@ object Local { } private final class Context6( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -253,13 +274,20 @@ object Local { } private final class Context7( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_], - k7: Key, v7: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -294,14 +322,22 @@ object Local { } private final class Context8( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_], - k7: Key, v7: Some[_], - k8: Key, v8: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -328,26 +364,42 @@ object Local { def set(k: Key, v: Some[_]): Context = if (k eq k1) new Context8(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) - else if (k eq k2) new Context8(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) - else if (k eq k3) new Context8(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) - else if (k eq k4) new Context8(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8) - else if (k eq k5) new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8) - else if (k eq k6) new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8) - else if (k eq k7) new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8) - else if (k eq k8) new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v) + else if (k eq k2) + new Context8(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + else if (k eq k3) + new Context8(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + else if (k eq k4) + new Context8(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8) + else if (k eq k5) + new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8) + else if (k eq k6) + new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8) + else if (k eq k7) + new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8) + else if (k eq k8) + new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v) else new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k, v) } private final class Context9( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_], - k7: Key, v7: Some[_], - k8: Key, v8: Some[_], - k9: Key, v9: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -364,40 +416,89 @@ object Local { def remove(k: Key): Context = if (k eq k1) new Context8(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k2) new Context8(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k3) new Context8(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k4) new Context8(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k5) new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k6) new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9) - else if (k eq k7) new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9) - else if (k eq k8) new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9) - else if (k eq k9) new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + else if (k eq k2) + new Context8(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + else if (k eq k3) + new Context8(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + else if (k eq k4) + new Context8(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + else if (k eq k5) + new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9) + else if (k eq k6) + new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9) + else if (k eq k7) + new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9) + else if (k eq k8) + new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9) + else if (k eq k9) + new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context9(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k2) new Context9(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k3) new Context9(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k4) new Context9(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k5) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8, k9, v9) - else if (k eq k6) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8, k9, v9) - else if (k eq k7) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8, k9, v9) - else if (k eq k8) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v, k9, v9) - else if (k eq k9) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v) - else new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k, v) + if (k eq k1) + new Context9(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + else if (k eq k2) + new Context9(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + else if (k eq k3) + new Context9(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + else if (k eq k4) + new Context9(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + else if (k eq k5) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8, k9, v9) + else if (k eq k6) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8, k9, v9) + else if (k eq k7) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8, k9, v9) + else if (k eq k8) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v, k9, v9) + else if (k eq k9) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v) + else + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k, + v + ) } private final class Context10( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_], - k7: Key, v7: Some[_], - k8: Key, v8: Some[_], - k9: Key, v9: Some[_], - k10: Key, v10: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -414,44 +515,309 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context9(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k2) new Context9(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k3) new Context9(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k4) new Context9(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k5) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k6) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k7) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9, k10, v10) - else if (k eq k8) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9, k10, v10) - else if (k eq k9) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k10, v10) - else if (k eq k10) new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + if (k eq k1) + new Context9(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + else if (k eq k2) + new Context9(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + else if (k eq k3) + new Context9(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + else if (k eq k4) + new Context9(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + else if (k eq k5) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + else if (k eq k6) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9, k10, v10) + else if (k eq k7) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9, k10, v10) + else if (k eq k8) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9, k10, v10) + else if (k eq k9) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k10, v10) + else if (k eq k10) + new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context10(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k2) new Context10(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k3) new Context10(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k4) new Context10(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k5) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k6) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8, k9, v9, k10, v10) - else if (k eq k7) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8, k9, v9, k10, v10) - else if (k eq k8) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v, k9, v9, k10, v10) - else if (k eq k9) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v, k10, v10) - else if (k eq k10) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v) - else new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k, v) + if (k eq k1) + new Context10( + k1, + v, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10 + ) + else if (k eq k2) + new Context10( + k1, + v1, + k2, + v, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10 + ) + else if (k eq k3) + new Context10( + k1, + v1, + k2, + v2, + k3, + v, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10 + ) + else if (k eq k4) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10 + ) + else if (k eq k5) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10 + ) + else if (k eq k6) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10 + ) + else if (k eq k7) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v, + k8, + v8, + k9, + v9, + k10, + v10 + ) + else if (k eq k8) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v, + k9, + v9, + k10, + v10 + ) + else if (k eq k9) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v, + k10, + v10 + ) + else if (k eq k10) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v + ) + else + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k, + v + ) } private final class Context11( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_], - k7: Key, v7: Some[_], - k8: Key, v8: Some[_], - k9: Key, v9: Some[_], - k10: Key, v10: Some[_], - k11: Key, v11: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -469,47 +835,591 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context10(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k2) new Context10(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k3) new Context10(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k4) new Context10(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k5) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k6) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k7) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k8) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9, k10, v10, k11, v11) - else if (k eq k9) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k10, v10, k11, v11) - else if (k eq k10) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k11, v11) - else if (k eq k11) new Context10(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + if (k eq k1) + new Context10( + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k2) + new Context10( + k1, + v1, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k3) + new Context10( + k1, + v1, + k2, + v2, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k4) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k5) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k6) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k7) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k8) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k9) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k10, + v10, + k11, + v11 + ) + else if (k eq k10) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k11, + v11 + ) + else if (k eq k11) + new Context10( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10 + ) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context11(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k2) new Context11(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k3) new Context11(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k4) new Context11(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k5) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k6) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k7) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8, k9, v9, k10, v10, k11, v11) - else if (k eq k8) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v, k9, v9, k10, v10, k11, v11) - else if (k eq k9) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v, k10, v10, k11, v11) - else if (k eq k10) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v, k11, v11) - else if (k eq k11) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v) - else new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k, v) + if (k eq k1) + new Context11( + k1, + v, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k2) + new Context11( + k1, + v1, + k2, + v, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k3) + new Context11( + k1, + v1, + k2, + v2, + k3, + v, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k4) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k5) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k6) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k7) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k8) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v, + k9, + v9, + k10, + v10, + k11, + v11 + ) + else if (k eq k9) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v, + k10, + v10, + k11, + v11 + ) + else if (k eq k10) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v, + k11, + v11 + ) + else if (k eq k11) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v + ) + else + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k, + v + ) } private final class Context12( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_], - k7: Key, v7: Some[_], - k8: Key, v8: Some[_], - k9: Key, v9: Some[_], - k10: Key, v10: Some[_], - k11: Key, v11: Some[_], - k12: Key, v12: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -528,50 +1438,691 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context11(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k2) new Context11(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k3) new Context11(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k4) new Context11(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k5) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k6) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k7) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k8) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k9) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k10, v10, k11, v11, k12, v12) - else if (k eq k10) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k11, v11, k12, v12) - else if (k eq k11) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k12, v12) - else if (k eq k12) new Context11(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11) + if (k eq k1) + new Context11( + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k2) + new Context11( + k1, + v1, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k3) + new Context11( + k1, + v1, + k2, + v2, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k4) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k5) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k6) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k7) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k8) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k9) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k10) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k11, + v11, + k12, + v12 + ) + else if (k eq k11) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k12, + v12 + ) + else if (k eq k12) + new Context11( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11 + ) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context12(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k2) new Context12(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k3) new Context12(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k4) new Context12(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k5) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k6) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k7) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k8) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v, k9, v9, k10, v10, k11, v11, k12, v12) - else if (k eq k9) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v, k10, v10, k11, v11, k12, v12) - else if (k eq k10) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v, k11, v11, k12, v12) - else if (k eq k11) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v, k12, v12) - else if (k eq k12) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v) - else new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k, v) + if (k eq k1) + new Context12( + k1, + v, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k2) + new Context12( + k1, + v1, + k2, + v, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k3) + new Context12( + k1, + v1, + k2, + v2, + k3, + v, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k4) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k5) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k6) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k7) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k8) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k9) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v, + k10, + v10, + k11, + v11, + k12, + v12 + ) + else if (k eq k10) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v, + k11, + v11, + k12, + v12 + ) + else if (k eq k11) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v, + k12, + v12 + ) + else if (k eq k12) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v + ) + else + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k, + v + ) } private final class Context13( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_], - k7: Key, v7: Some[_], - k8: Key, v8: Some[_], - k9: Key, v9: Some[_], - k10: Key, v10: Some[_], - k11: Key, v11: Some[_], - k12: Key, v12: Some[_], - k13: Key, v13: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -591,53 +2142,799 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context12(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k2) new Context12(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k3) new Context12(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k4) new Context12(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k5) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k6) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k7) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k8) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k9) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k10) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k11, v11, k12, v12, k13, v13) - else if (k eq k11) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k12, v12, k13, v13) - else if (k eq k12) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k13, v13) - else if (k eq k13) new Context12(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12) + if (k eq k1) + new Context12( + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k2) + new Context12( + k1, + v1, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k3) + new Context12( + k1, + v1, + k2, + v2, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k4) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k5) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k6) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k7) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k8) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k9) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k10) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k11) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k12, + v12, + k13, + v13 + ) + else if (k eq k12) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k13, + v13 + ) + else if (k eq k13) + new Context12( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12 + ) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context13(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k2) new Context13(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k3) new Context13(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k4) new Context13(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k5) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k6) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k7) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k8) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k9) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v, k10, v10, k11, v11, k12, v12, k13, v13) - else if (k eq k10) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v, k11, v11, k12, v12, k13, v13) - else if (k eq k11) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v, k12, v12, k13, v13) - else if (k eq k12) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v, k13, v13) - else if (k eq k13) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v) - else new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k, v) + if (k eq k1) + new Context13( + k1, + v, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k2) + new Context13( + k1, + v1, + k2, + v, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k3) + new Context13( + k1, + v1, + k2, + v2, + k3, + v, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k4) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k5) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k6) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k7) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k8) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k9) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k10) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v, + k11, + v11, + k12, + v12, + k13, + v13 + ) + else if (k eq k11) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v, + k12, + v12, + k13, + v13 + ) + else if (k eq k12) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v, + k13, + v13 + ) + else if (k eq k13) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v + ) + else + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k, + v + ) } private final class Context14( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_], - k7: Key, v7: Some[_], - k8: Key, v8: Some[_], - k9: Key, v9: Some[_], - k10: Key, v10: Some[_], - k11: Key, v11: Some[_], - k12: Key, v12: Some[_], - k13: Key, v13: Some[_], - k14: Key, v14: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_], + k14: Key, + v14: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -658,56 +2955,915 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context13(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k2) new Context13(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k3) new Context13(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k4) new Context13(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k5) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k6) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k7) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k8) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k9) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k10) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k11) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k12, v12, k13, v13, k14, v14) - else if (k eq k12) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k13, v13, k14, v14) - else if (k eq k13) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k14, v14) - else if (k eq k14) new Context13(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13) + if (k eq k1) + new Context13( + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k2) + new Context13( + k1, + v1, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k3) + new Context13( + k1, + v1, + k2, + v2, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k4) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k5) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k6) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k7) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k8) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k9) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k10) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k11) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k12) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k13, + v13, + k14, + v14 + ) + else if (k eq k13) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k14, + v14 + ) + else if (k eq k14) + new Context13( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13 + ) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context14(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k2) new Context14(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k3) new Context14(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k4) new Context14(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k5) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k6) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k7) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k8) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k9) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k10) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v, k11, v11, k12, v12, k13, v13, k14, v14) - else if (k eq k11) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v, k12, v12, k13, v13, k14, v14) - else if (k eq k12) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v, k13, v13, k14, v14) - else if (k eq k13) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v, k14, v14) - else if (k eq k14) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v) - else new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k, v) + if (k eq k1) + new Context14( + k1, + v, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k2) + new Context14( + k1, + v1, + k2, + v, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k3) + new Context14( + k1, + v1, + k2, + v2, + k3, + v, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k4) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k5) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k6) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k7) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k8) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k9) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k10) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k11) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v, + k12, + v12, + k13, + v13, + k14, + v14 + ) + else if (k eq k12) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v, + k13, + v13, + k14, + v14 + ) + else if (k eq k13) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v, + k14, + v14 + ) + else if (k eq k14) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v + ) + else + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k, + v + ) } private final class Context15( - k1: Key, v1: Some[_], - k2: Key, v2: Some[_], - k3: Key, v3: Some[_], - k4: Key, v4: Some[_], - k5: Key, v5: Some[_], - k6: Key, v6: Some[_], - k7: Key, v7: Some[_], - k8: Key, v8: Some[_], - k9: Key, v9: Some[_], - k10: Key, v10: Some[_], - k11: Key, v11: Some[_], - k12: Key, v12: Some[_], - k13: Key, v13: Some[_], - k14: Key, v14: Some[_], - k15: Key, v15: Some[_] + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_], + k14: Key, + v14: Some[_], + k15: Key, + v15: Some[_] ) extends Context { def get(k: Key): Option[_] = @@ -729,44 +3885,976 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context14(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k2) new Context14(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k3) new Context14(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k4) new Context14(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k5) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k6) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k7) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k8) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k9) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k10) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k11) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k12) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k13, v13, k14, v14, k15, v15) - else if (k eq k13) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k14, v14, k15, v15) - else if (k eq k14) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k15, v15) - else if (k eq k15) new Context14(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14) + if (k eq k1) + new Context14( + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k2) + new Context14( + k1, + v1, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k3) + new Context14( + k1, + v1, + k2, + v2, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k4) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k5) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k6) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k7) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k8) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k9) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k10) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k11) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k12) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k13) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k14, + v14, + k15, + v15 + ) + else if (k eq k14) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k15, + v15 + ) + else if (k eq k15) + new Context14( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14 + ) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context15(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k2) new Context15(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k3) new Context15(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k4) new Context15(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k5) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k6) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k7) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k8) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k9) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k10) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v, k11, v11, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k11) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v, k12, v12, k13, v13, k14, v14, k15, v15) - else if (k eq k12) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v, k13, v13, k14, v14, k15, v15) - else if (k eq k13) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v, k14, v14, k15, v15) - else if (k eq k14) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v, k15, v15) - else if (k eq k15) new Context15(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10, k11, v11, k12, v12, k13, v13, k14, v14, k15, v) + if (k eq k1) + new Context15( + k1, + v, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k2) + new Context15( + k1, + v1, + k2, + v, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k3) + new Context15( + k1, + v1, + k2, + v2, + k3, + v, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k4) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k5) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k6) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k7) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k8) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k9) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k10) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k11) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k12) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v, + k13, + v13, + k14, + v14, + k15, + v15 + ) + else if (k eq k13) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v, + k14, + v14, + k15, + v15 + ) + else if (k eq k14) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v, + k15, + v15 + ) + else if (k eq k15) + new Context15( + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10, + k11, + v11, + k12, + v12, + k13, + v13, + k14, + v14, + k15, + v + ) else new ContextN(k, v, this) } private final class ContextN( - kN: Key, vN: Some[_], rest: Context + kN: Key, + vN: Some[_], + rest: Context ) extends Context { def get(k: Key): Option[_] = @@ -796,7 +4884,6 @@ object Local { * This should be treated as an opaque value and direct modifications * and access are considered verboten. */ - private[this] val localCtx = new ThreadLocal[Context] { override def initialValue(): Context = Context.empty } diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index b94e583d4d..20a90d7e0f 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -225,7 +225,8 @@ trait TimeLike[This <: TimeLike[This]] extends Ordered[This] { self: This => def floor(increment: Duration): This = (this, increment) match { case (num, ns) if num.isZero && ns.isZero => Undefined case (num, ns) if num.isFinite && ns.isZero => if (num.inNanoseconds < 0) Bottom else Top - case (num, denom) if num.isFinite && denom.isFinite => fromNanoseconds((num.inNanoseconds / denom.inNanoseconds) * denom.inNanoseconds) + case (num, denom) if num.isFinite && denom.isFinite => + fromNanoseconds((num.inNanoseconds / denom.inNanoseconds) * denom.inNanoseconds) case (self, n) if n.isFinite => self case (_, _) => Undefined } diff --git a/util-core/src/main/scala/com/twitter/util/WindowedAdder.scala b/util-core/src/main/scala/com/twitter/util/WindowedAdder.scala index e1ceffab5d..70529c61ca 100644 --- a/util-core/src/main/scala/com/twitter/util/WindowedAdder.scala +++ b/util-core/src/main/scala/com/twitter/util/WindowedAdder.scala @@ -27,7 +27,11 @@ private[twitter] object WindowedAdder { } } -private[twitter] class WindowedAdder private[WindowedAdder] (window: Long, N: Int, now: () => Long) { +private[twitter] class WindowedAdder private[WindowedAdder] ( + window: Long, + N: Int, + now: () => Long +) { private[this] val writer = new LongAdder() @volatile private[this] var gen = 0 private[this] val expiredGen = new AtomicInteger(gen) diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala index f51abd6697..f2bc733c7b 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala @@ -474,7 +474,9 @@ class SpoolTest extends WordSpec with GeneratorDrivenPropertyChecks { * Confirms that the given operation does not consume an entire Spool, and then * returns the resulting Spool and tail check for further validation. */ - def applyLazily(f: Spool[Int] => Future[Spool[Int]]): (Future[Spool[Int]], Future[Spool[Int]]) = { + def applyLazily( + f: Spool[Int] => Future[Spool[Int]] + ): (Future[Spool[Int]], Future[Spool[Int]]) = { val tail = new Promise[Spool[Int]] // A spool where only the head is valid. diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index 6688691533..3605252bb8 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -12,7 +12,7 @@ import org.scalatest.prop.{Checkers, GeneratorDrivenPropertyChecks} import scala.collection.mutable class BufTest - extends FunSuite + extends FunSuite with MockitoSugar with GeneratorDrivenPropertyChecks with Checkers @@ -69,14 +69,15 @@ class BufTest test("Buf.ByteArray.[Owned|Shared](data, begin, end) validates non-negative indexes") { val data = Array[Byte]() - Seq(-1 -> 1, 1 -> -1).foreach { case (begin, end) => - intercept[IllegalArgumentException] { - Buf.ByteArray.Owned(data, begin, end) - } + Seq(-1 -> 1, 1 -> -1).foreach { + case (begin, end) => + intercept[IllegalArgumentException] { + Buf.ByteArray.Owned(data, begin, end) + } - intercept[IllegalArgumentException] { - Buf.ByteArray.Shared(data, begin, end) - } + intercept[IllegalArgumentException] { + Buf.ByteArray.Shared(data, begin, end) + } } } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 9009291812..b499e2e2d9 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -82,7 +82,8 @@ class ReaderTest i <- Gen.posNum[Int].suchThat(_ <= s.length) } yield (s, i) - forAll(stringAndChunk) { case (s, i) => + forAll(stringAndChunk) { + case (s, i) => val r = Reader.chunked(Reader.fromBuf(Buf.Utf8(s)), i) def readLoop(): Unit = await(r.read(Int.MaxValue)) match { diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index 78762eabcf..2afd1c32dd 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -93,11 +93,13 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty queue add promise promise } - iteration.onSuccess { _ => - complete = true - }.onFailure { _ => - failure = true - } + iteration + .onSuccess { _ => + complete = true + } + .onFailure { _ => + failure = true + } assert(!complete) assert(!failure) } @@ -173,11 +175,13 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty promise } - iteration.onSuccess { _ => - complete = true - }.onFailure { _ => - failure = true - } + iteration + .onSuccess { _ => + complete = true + } + .onFailure { _ => + failure = true + } assert(!complete) assert(!failure) } @@ -756,7 +760,10 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty } } - def testJoin(label: String, joiner: ((Future[Int], Future[Int]) => Future[(Int, Int)])): Unit = { + def testJoin( + label: String, + joiner: ((Future[Int], Future[Int]) => Future[(Int, Int)]) + ): Unit = { s"join($label)" should { trait JoinHelper { val p0 = new HandledPromise[Int] @@ -879,18 +886,20 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty val inner1 = new Promise[Int] var ran = false val f: Future[Int] = Future.monitored { - inner1.ensure { - // Note that these are sequenced so that interrupts - // will be delivered before inner's handler is cleared. - ran = true - try { - inner.update(Return(1)) - } catch { - case _: Throwable => fail() + inner1 + .ensure { + // Note that these are sequenced so that interrupts + // will be delivered before inner's handler is cleared. + ran = true + try { + inner.update(Return(1)) + } catch { + case _: Throwable => fail() + } + } + .ensure { + throw exc } - }.ensure { - throw exc - } inner } assert(!ran) @@ -967,24 +976,33 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty val e = new Exception("rdrr") "values" in { - const.value(1).transform { - case Return(v) => const.value(v + 1) - case Throw(_) => const.value(0) - }.mustProduce(Return(2)) + const + .value(1) + .transform { + case Return(v) => const.value(v + 1) + case Throw(_) => const.value(0) + } + .mustProduce(Return(2)) } "exceptions" in { - const.exception(e).transform { - case Return(_) => const.value(1) - case Throw(_) => const.value(0) - }.mustProduce(Return(0)) + const + .exception(e) + .transform { + case Return(_) => const.value(1) + case Throw(_) => const.value(0) + } + .mustProduce(Return(0)) } "exceptions thrown during transformation" in { - const.value(1).transform { - case Return(_) => const.value(throw e) - case Throw(_) => const.value(0) - }.mustProduce(Throw(e)) + const + .value(1) + .transform { + case Return(_) => const.value(throw e) + case Throw(_) => const.value(0) + } + .mustProduce(Throw(e)) } "non local returns executed during transformation" in { @@ -1034,7 +1052,9 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty val actual = intercept[FatalException] { Monitor.using(m) { - const.value(1).transform { _ => throw exc } + const.value(1).transform { _ => + throw exc + } } } @@ -1052,7 +1072,8 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty .transformedBy(new FutureTransformer[Int, Int] { override def flatMap(value: Int): Future[Int] = const.value(value + 1) override def rescue(t: Throwable): Future[Int] = const.value(0) - }).mustProduce(Return(2)) + }) + .mustProduce(Return(2)) } "rescue" in { @@ -1061,7 +1082,8 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty .transformedBy(new FutureTransformer[Int, Int] { override def flatMap(value: Int): Future[Int] = const.value(value + 1) override def rescue(t: Throwable): Future[Int] = const.value(0) - }).mustProduce(Return(0)) + }) + .mustProduce(Return(0)) } "exceptions thrown during transformation" in { @@ -1070,7 +1092,8 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty .transformedBy(new FutureTransformer[Int, Int] { override def flatMap(value: Int): Future[Int] = throw e override def rescue(t: Throwable): Future[Int] = const.value(0) - }).mustProduce(Throw(e)) + }) + .mustProduce(Throw(e)) } "map" in { @@ -1079,7 +1102,8 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty .transformedBy(new FutureTransformer[Int, Int] { override def map(value: Int): Int = value + 1 override def handle(t: Throwable): Int = 0 - }).mustProduce(Return(2)) + }) + .mustProduce(Return(2)) } "handle" in { @@ -1088,11 +1112,15 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty .transformedBy(new FutureTransformer[Int, Int] { override def map(value: Int): Int = value + 1 override def handle(t: Throwable): Int = 0 - }).mustProduce(Return(0)) + }) + .mustProduce(Return(0)) } } - def testSequence(which: String, seqop: (Future[Unit], () => Future[Unit]) => Future[Unit]): Unit = { + def testSequence( + which: String, + seqop: (Future[Unit], () => Future[Unit]) => Future[Unit] + ): Unit = { which when { "successes" should { "interruption of the produced future" which { @@ -1326,14 +1354,17 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty val p = new Promise[Int] p.ensure { - i = i + j + k + h + 1 - }.ensure { - j = i + j + k + h + 1 - }.ensure { - k = i + j + k + h + 1 - }.ensure { - h = i + j + k + h + 1 - } + i = i + j + k + h + 1 + } + .ensure { + j = i + j + k + h + 1 + } + .ensure { + k = i + j + k + h + 1 + } + .ensure { + h = i + j + k + h + 1 + } assert(i == 0) assert(j == 0) diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/TwitterFuturesCapturePointsTest.scala b/util-intellij/src/test/scala/com/twitter/util/capturepoints/TwitterFuturesCapturePointsTest.scala index 7145f70f59..b9984e7278 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/TwitterFuturesCapturePointsTest.scala +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/TwitterFuturesCapturePointsTest.scala @@ -27,13 +27,21 @@ class TwitterFuturesCapturePointsTest extends FunSuite { /** * test that [methodName] is a member of `ClassOf[com.twitter.util.className]` */ - private[this] def testMethod(methodType: String, className: String, methodName: String = "apply"): Unit = { + private[this] def testMethod( + methodType: String, + className: String, + methodName: String = "apply" + ): Unit = { val m = Class.forName(className) val methodArray = findMethod(m, methodName) - assert(methodArray.nonEmpty, s": couldn't find $methodType method ${m.getTypeName}.$methodName.") + assert( + methodArray.nonEmpty, + s": couldn't find $methodType method ${m.getTypeName}.$methodName." + ) } - val capturePointsXML = XML.load(getClass.getResourceAsStream("/util-intellij/TwitterFuturesCapturePoints.xml")) + val capturePointsXML = + XML.load(getClass.getResourceAsStream("/util-intellij/TwitterFuturesCapturePoints.xml")) /** * perform "capture method" and "insert method" tests for each capture point diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala b/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala index d39e38ccdf..ebca1f7fa7 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala @@ -34,7 +34,9 @@ class Heapster(klass: Class[_]) { def start(): Unit = { startM.invoke(null) } def shutdown(): Unit = { stopM.invoke(null) } - def setSamplingPeriod(period: java.lang.Integer): Unit = { setSamplingPeriodM.invoke(null, period) } + def setSamplingPeriod(period: java.lang.Integer): Unit = { + setSamplingPeriodM.invoke(null, period) + } def clearProfile(): Unit = { clearProfileM.invoke(null) } def dumpProfile(forceGC: java.lang.Boolean): Array[Byte] = dumpProfileM.invoke(null, forceGC).asInstanceOf[Array[Byte]] diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala index 5e42b2a764..8617ea7c2a 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala @@ -4,6 +4,7 @@ import com.twitter.finagle.stats.NullStatsReceiver import com.twitter.logging.ScribeHandler._ object ScribeHandlers { + /** * For Java compatibility. Calls [[ScribeHandler.apply()]] method with * the provided arguments, so Java users don't need to provide the diff --git a/util-security/src/main/scala/com/twitter/util/security/NullSslSession.scala b/util-security/src/main/scala/com/twitter/util/security/NullSslSession.scala index 11ee7c3ff6..3152c63017 100644 --- a/util-security/src/main/scala/com/twitter/util/security/NullSslSession.scala +++ b/util-security/src/main/scala/com/twitter/util/security/NullSslSession.scala @@ -27,8 +27,8 @@ object NullSslSession extends SSLSession { def getSessionContext: SSLSessionContext = NullSslSessionContext def getValue(name: String): Object = "" def getValueNames: Array[String] = Array.empty - def invalidate: Unit = { } + def invalidate: Unit = {} def isValid: Boolean = false - def putValue(name: String, value: Object): Unit = { } - def removeValue(name: String): Unit = { } + def putValue(name: String, value: Object): Unit = {} + def removeValue(name: String): Unit = {} } diff --git a/util-security/src/main/scala/com/twitter/util/security/NullSslSessionContext.scala b/util-security/src/main/scala/com/twitter/util/security/NullSslSessionContext.scala index 1eb5184adc..66d8f5ca2b 100644 --- a/util-security/src/main/scala/com/twitter/util/security/NullSslSessionContext.scala +++ b/util-security/src/main/scala/com/twitter/util/security/NullSslSessionContext.scala @@ -12,6 +12,6 @@ object NullSslSessionContext extends SSLSessionContext { def getSession(name: Array[Byte]): SSLSession = NullSslSession def getSessionCacheSize: Int = 0 def getSessionTimeout: Int = 0 - def setSessionCacheSize(size: Int): Unit = { } - def setSessionTimeout(seconds: Int): Unit = { } + def setSessionCacheSize(size: Int): Unit = {} + def setSessionTimeout(seconds: Int): Unit = {} } diff --git a/util-security/src/main/scala/com/twitter/util/security/X509CrlFile.scala b/util-security/src/main/scala/com/twitter/util/security/X509CrlFile.scala index 9816ffc8e6..90e5259c15 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X509CrlFile.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X509CrlFile.scala @@ -9,7 +9,7 @@ import java.security.cert.{CertificateFactory, X509CRL} /** * A representation of an X.509 Certificate Revocation List (CRL) * PEM-encoded and stored in a file. - * + * * @example * -----BEGIN X509 CRL----- * base64encodedbytes diff --git a/util-security/src/test/scala/com/twitter/util/security/NullSslSessionTest.scala b/util-security/src/test/scala/com/twitter/util/security/NullSslSessionTest.scala index 233891974b..342deb740b 100644 --- a/util-security/src/test/scala/com/twitter/util/security/NullSslSessionTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/NullSslSessionTest.scala @@ -44,4 +44,3 @@ class NullSslSessionTest extends FunSuite { } } - diff --git a/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala index 91ed6814e9..907bac34bf 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala @@ -40,7 +40,7 @@ class X509CrlFileTest extends FunSuite { // Lines were manually deleted from a real crl file val tempFile = TempFile.fromResourcePath("/crl/csl-intermediate-garbage.crl") // deleteOnExit is handled by TempFile - + val crlFile = new X509CrlFile(tempFile) val tryCrl = crlFile.readX509Crl() @@ -51,7 +51,7 @@ class X509CrlFileTest extends FunSuite { test("File is a Certificate Revocation List") { val tempFile = TempFile.fromResourcePath("/crl/csl-intermediate.crl") // deleteOnExit is handled by TempFile - + val crlFile = new X509CrlFile(tempFile) val tryCrl = crlFile.readX509Crl() diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala index f19b4df8be..d6968d39f7 100644 --- a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala @@ -49,7 +49,8 @@ object Slf4jBridgeUtility extends Logging { .getLogger(this.getClass) .warn( "Detected [org.slf4j.impl.JDK14LoggerFactory] on classpath. " + - "SLF4JBridgeHandler will not be installed, see: http://www.slf4j.org/legacy.html#julRecursion") + "SLF4JBridgeHandler will not be installed, see: http://www.slf4j.org/legacy.html#julRecursion" + ) false } catch { case e: ClassNotFoundException => diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index 5473b51b0d..8185e63cb8 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -144,53 +144,54 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => case _ => whenNotPresent(verbosity) } - private[this] def whenNotPresent(verbosity: Verbosity) = new JFunction[Seq[String], CumulativeGauge] { - def apply(key: Seq[String]): CumulativeGauge = new CumulativeGauge { - self.registerGauge(verbosity, key, getValue) - - // we grab the read lock on increment and decrement, which can race each - // other. we only grab the write lock when we tear down the cumulative - // gauge, so that we can ensure no more increments happen after death. - private[this] val lock = new StampedLock() - private[this] val registers = new AtomicInteger(0) - - // protected by `lock` - private[this] var dead = false - - def register(): Boolean = { - val stamp = lock.readLock() - try { - if (dead) return false - registers.incrementAndGet() - } finally { - lock.unlockRead(stamp) + private[this] def whenNotPresent(verbosity: Verbosity) = + new JFunction[Seq[String], CumulativeGauge] { + def apply(key: Seq[String]): CumulativeGauge = new CumulativeGauge { + self.registerGauge(verbosity, key, getValue) + + // we grab the read lock on increment and decrement, which can race each + // other. we only grab the write lock when we tear down the cumulative + // gauge, so that we can ensure no more increments happen after death. + private[this] val lock = new StampedLock() + private[this] val registers = new AtomicInteger(0) + + // protected by `lock` + private[this] var dead = false + + def register(): Boolean = { + val stamp = lock.readLock() + try { + if (dead) return false + registers.incrementAndGet() + } finally { + lock.unlockRead(stamp) + } + true } - true - } - def deregister(): Unit = { - val readStamp = lock.readLock() - if (registers.decrementAndGet() == 0) { - // we could skip a cas by using tryConvertToWriteLock, but it makes - // the code much uglier. - lock.unlockRead(readStamp) - val writeStamp = lock.writeLock() - - try { - if (!dead && registers.get == 0) { - dead = true - gauges.remove(key) - self.deregisterGauge(key) + def deregister(): Unit = { + val readStamp = lock.readLock() + if (registers.decrementAndGet() == 0) { + // we could skip a cas by using tryConvertToWriteLock, but it makes + // the code much uglier. + lock.unlockRead(readStamp) + val writeStamp = lock.writeLock() + + try { + if (!dead && registers.get == 0) { + dead = true + gauges.remove(key) + self.deregisterGauge(key) + } + } finally { + lock.unlockWrite(writeStamp) } - } finally { - lock.unlockWrite(writeStamp) + } else { + lock.unlockRead(readStamp) } - } else { - lock.unlockRead(readStamp) } } } - } def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = { var gauge: Gauge = null diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index dd106b5bd3..c9af5acd83 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -178,13 +178,13 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { p.println("-------") } for ((k, g) <- sortedGauges) - p.print(f"$k%s ${ g() }%f\n") + p.print(f"$k%s ${g()}%f\n") if (includeHeaders && sortedStats.nonEmpty) { p.println("\nStats:") p.println("------") } for ((k, s) <- sortedStats if s.nonEmpty) { - p.print(f"$k%s ${ s.sum / s.size }%f ${statValuesToStr(s)}\n") + p.print(f"$k%s ${s.sum / s.size}%f ${statValuesToStr(s)}\n") } } diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/Tunable.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/Tunable.scala index 952c8755a0..088a319de2 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/Tunable.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/Tunable.scala @@ -112,8 +112,7 @@ object Tunable { /** * A [[Tunable]] whose value can be changed. Operations are thread-safe. */ - final class Mutable[T] private[tunable] (id: String, _value: Option[T]) - extends Tunable[T](id) { + final class Mutable[T] private[tunable] (id: String, _value: Option[T]) extends Tunable[T](id) { private[this] final val mutableHolder = Var(_value) diff --git a/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala b/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala index 7ce12f3fab..8f0fa1aa07 100644 --- a/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala +++ b/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala @@ -46,7 +46,13 @@ class ExistsCallbackPromise(znode: ZNode) class ChildrenCallbackPromise(znode: ZNode) extends AsyncCallbackPromise[ZNode.Children] with AsyncCallback.Children2Callback { - def processResult(rc: Int, path: String, ctx: AnyRef, children: JList[String], stat: Stat): Unit = { + def processResult( + rc: Int, + path: String, + ctx: AnyRef, + children: JList[String], + stat: Stat + ): Unit = { process(rc, path) { znode(stat, children.asScala.toSeq) } diff --git a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala index e9f4123788..55535b4b5b 100644 --- a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala +++ b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala @@ -123,12 +123,14 @@ object NativeConnector { protected[this] val connectPromise = new Promise[ZooKeeper] /** A ZooKeeper handle that will error if connectTimeout is specified and exceeded. */ - lazy val connected: Future[ZooKeeper] = connectTimeout.map { timeout => - connectPromise.within(timer, timeout).rescue { - case _: TimeoutException => - Future.exception(ConnectTimeoutException(connectString, timeout)) + lazy val connected: Future[ZooKeeper] = connectTimeout + .map { timeout => + connectPromise.within(timer, timeout).rescue { + case _: TimeoutException => + Future.exception(ConnectTimeoutException(connectString, timeout)) + } } - }.getOrElse(connectPromise) + .getOrElse(connectPromise) protected[this] val releasePromise = new Promise[Unit] val released: Future[Unit] = releasePromise @@ -142,7 +144,8 @@ object NativeConnector { val sessionEvents: Offer[StateEvent] = { val broker = new Broker[StateEvent] def loop(): Unit = { - sessionBroker.recv.sync() + sessionBroker.recv + .sync() .map { StateEvent(_) } .respond { case Return(StateEvent.Connected) => diff --git a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala index 50b64eb9b3..67d2351b16 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala @@ -220,7 +220,10 @@ trait ZNode { } /** Monitor a watch on this node. */ - def monitorWatch(watch: Future[ZNode.Watch[ZNode.Children]], knownChildren: Set[ZNode]): Unit = { + def monitorWatch( + watch: Future[ZNode.Watch[ZNode.Children]], + knownChildren: Set[ZNode] + ): Unit = { log.debug("monitoring %s with %d known children", path, knownChildren.size) watch onFailure { e => // An error occurred and there's not really anything we can do about it. diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala index 89f59aa1b9..82d79b3162 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala @@ -35,7 +35,12 @@ import org.apache.zookeeper.{CreateMode, KeeperException} * } // handle { ... } * }}} */ -class ZkAsyncSemaphore(zk: ZkClient, path: String, numPermits: Int, maxWaiters: Option[Int] = None) { +class ZkAsyncSemaphore( + zk: ZkClient, + path: String, + numPermits: Int, + maxWaiters: Option[Int] = None +) { import ZkAsyncSemaphore._ require(numPermits > 0) require(maxWaiters.getOrElse(0) >= 0) diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index db4e949d6a..91b46b5f4e 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -181,7 +181,9 @@ class ZkClientTest extends WordSpec with MockitoSugar { } } - def watchData(path: String)(result: => Future[ZNode.Data])(update: => Future[WatchedEvent]): Unit = { + def watchData( + path: String + )(result: => Future[ZNode.Data])(update: => Future[WatchedEvent]): Unit = { val w = ArgumentCaptor.forClass(classOf[Watcher]) val cb = ArgumentCaptor.forClass(classOf[AsyncCallback.DataCallback]) when(zk.getData(meq(path), w.capture(), cb.capture(), meq(null))) thenAnswer answer[ From 91132561c75a90608f295b4fe7ef97662e0a05ba Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Mon, 17 Sep 2018 16:53:59 +0000 Subject: [PATCH 007/796] Twitter-oss: update OSS libraries post-release to 18.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 18.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D214858 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cdd84070fa..a258287748 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 01542f6d5a..c7bf0dd21e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.9.0" +val releaseVersion = "18.10.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 4b12205c861526445e1fead0dca1698b7c7c2df4 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Mon, 17 Sep 2018 18:43:07 +0000 Subject: [PATCH 008/796] Disable false positives for StoreGaugesAsMemberVariables lint rule Problem We have a new scalafix lint rule, StoreGaugesAsMemberVariables, that flags cases where Gauges are not stored in member variables. However, because it is a syntactic rule, it sometimes flags some false positives. Solution Disable the rule where appropriate. JIRA Issues: CSL-6746 TBR=true Differential Revision: https://phabricator.twitter.biz/D214652 --- .../finagle/stats/NameTranslatingStatsReceiver.scala | 5 ++++- .../scala/com/twitter/finagle/stats/StatsReceiver.scala | 5 ++++- .../com/twitter/finagle/stats/StatsReceiverProxy.scala | 2 +- .../finagle/stats/VerbosityAdjustingStatsReceiver.scala | 6 +++++- .../twitter/finagle/stats/BlacklistStatsReceiverTest.scala | 3 ++- .../twitter/finagle/stats/BroadcastStatsReceiverTest.scala | 2 ++ .../twitter/finagle/stats/InMemoryStatsReceiverTest.scala | 2 ++ 7 files changed, 20 insertions(+), 5 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index 6a25303bb9..3309a279c4 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -23,8 +23,11 @@ abstract class NameTranslatingStatsReceiver( override def stat(verbosity: Verbosity, name: String*): Stat = self.stat(verbosity, translate(name): _*) - override def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = + override def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = { + // scalafix:off StoreGaugesAsMemberVariables self.addGauge(verbosity, translate(name): _*)(f) + // scalafix:on StoreGaugesAsMemberVariables + } override def toString: String = s"$self/$namespacePrefix" } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index d2461a74a0..02429f090f 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -43,8 +43,11 @@ object StatsReceivers { * Java compatible version of [[StatsReceiver.addGauge]]. */ @varargs - def addGauge(statsReceiver: StatsReceiver, callable: Callable[JFloat], name: String*): Gauge = + def addGauge(statsReceiver: StatsReceiver, callable: Callable[JFloat], name: String*): Gauge = { + // scalafix:off StoreGaugesAsMemberVariables statsReceiver.addGauge(name: _*)(callable.call()) + // scalafix:on StoreGaugesAsMemberVariables + } /** * Java compatible version of [[StatsReceiver.provideGauge]]. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala index 073233b896..0eeaece890 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala @@ -12,7 +12,7 @@ trait StatsReceiverProxy extends StatsReceiver with DelegatingStatsReceiver { def counter(verbosity: Verbosity, names: String*): Counter = self.counter(verbosity, names: _*) def stat(verbosity: Verbosity, names: String*): Stat = self.stat(verbosity, names: _*) def addGauge(verbosity: Verbosity, names: String*)(f: => Float): Gauge = - self.addGauge(verbosity, names: _*)(f) + self.addGauge(verbosity, names: _*)(f) // scalafix:ok StoreGaugesAsMemberVariables def underlying: Seq[StatsReceiver] = Seq(self) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala index 888b8e0a6f..878ec62dba 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala @@ -18,5 +18,9 @@ class VerbosityAdjustingStatsReceiver( override def addGauge( verbosity: Verbosity, names: String* - )(f: => Float): Gauge = self.addGauge(defaultVerbosity, names: _*)(f) + )(f: => Float): Gauge = { + // scalafix:off StoreGaugesAsMemberVariables + self.addGauge(defaultVerbosity, names: _*)(f) + // scalafix:on StoreGaugesAsMemberVariables + } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BlacklistStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/BlacklistStatsReceiverTest.scala index f854caf76a..3b232ee316 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/BlacklistStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/BlacklistStatsReceiverTest.scala @@ -6,6 +6,7 @@ import org.scalatest.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) class BlacklistStatsReceiverTest extends FunSuite { + // scalafix:off StoreGaugesAsMemberVariables test("BlacklistStatsReceiver blacklists properly") { val inmemory = new InMemoryStatsReceiver() val bsr = new BlacklistStatsReceiver(inmemory, { case _ => true }) @@ -62,5 +63,5 @@ class BlacklistStatsReceiverTest extends FunSuite { assert(inmemory.gauges(Seq("foo", "foo", "baz"))() == 3.0) assert(inmemory.stats.isEmpty) } - + // scalafix:on StoreGaugesAsMemberVariables } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala index 5ff400166b..7d5ce36ae0 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala @@ -65,7 +65,9 @@ class BroadcastStatsReceiverTest extends FunSuite with Matchers { assert(None == recv1.gauges.get(Seq("hi"))) assert(None == recv1.gauges.get(Seq("scopeA", "hi"))) + // scalafix:off StoreGaugesAsMemberVariables val gaugeA = broadcastA.addGauge("hi") { 5f } + // scalafix:on StoreGaugesAsMemberVariables assert(5f == recv1.gauges(Seq("hi"))()) assert(5f == recv1.gauges(Seq("scopeA", "hi"))()) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 73576483cf..1a933858df 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -10,6 +10,7 @@ import org.scalatest.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) class InMemoryStatsReceiverTest extends FunSuite with Eventually with IntegrationPatience { + // scalafix:off StoreGaugesAsMemberVariables test("clear") { val inMemoryStatsReceiver = new InMemoryStatsReceiver inMemoryStatsReceiver.counter("counter").incr() @@ -313,4 +314,5 @@ class InMemoryStatsReceiverTest extends FunSuite with Eventually with Integratio ps.close() } } + // scalafix:on StoreGaugesAsMemberVariables } From 4db06554f8eeadd78edc2ec0268dea11733964a6 Mon Sep 17 00:00:00 2001 From: Diego Puppin Date: Wed, 19 Sep 2018 02:41:06 +0000 Subject: [PATCH 009/796] util-core: Fix example in AsyncSemaphore Problem Incorrect example for AsyncSemaphore. Solution Corrected the example. Result Correct example in comments for AsyncSemaphore. Differential Revision: https://phabricator.twitter.biz/D216757 --- .../src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala index 8e3ea27c62..75e64b59f4 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala @@ -17,7 +17,7 @@ import scala.util.control.NonFatal * {{{ * val semaphore = new AsyncSemaphore(n) * ... - * semaphore.acquireAndRun() { + * semaphore.acquireAndRun { * somethingThatReturnsFutureT() * } * }}} From e2aca990a2257b727a2bc3042c0f7651afc1584e Mon Sep 17 00:00:00 2001 From: Martijn Hoekstra Date: Wed, 19 Sep 2018 19:49:41 +0000 Subject: [PATCH 010/796] util: address deprecations Problem util uses deprecated library and language features that will be removed in 2.13 Solution Enable deprecated warnings in build, and use undeprecated alternatives Result A deprecation-warning-free build Signed-off-by: Stefan Lance Differential Revision: https://phabricator.twitter.biz/D216075 --- build.sbt | 6 ++-- .../scala/com/twitter/cache/Refresh.scala | 2 +- .../main/scala/com/twitter/util/LruMap.scala | 32 +++++++++++++++++-- .../java/com/twitter/concurrent/Offers.java | 5 +-- .../java/com/twitter/concurrent/Spools.java | 18 +++++++++-- .../main/java/com/twitter/util/Closables.java | 8 ++--- .../com/twitter/util/javainterop/Scala.java | 19 ++++++----- .../scala/com/twitter/concurrent/Spool.scala | 2 +- .../scala/com/twitter/util/Closable.scala | 4 +-- .../main/scala/com/twitter/util/Pool.scala | 2 +- .../concurrent/SpoolCompilationTest.java | 14 ++++---- .../com/twitter/util/CancellableTest.scala | 3 +- .../scala/com/twitter/util/TimerTest.scala | 4 +-- .../com/twitter/logging/LoggerTest.scala | 6 ++-- .../stats/InMemoryStatsReceiverTest.scala | 4 +-- .../src/main/scala/com/twitter/zk/Event.scala | 13 ++++---- 16 files changed, 92 insertions(+), 50 deletions(-) diff --git a/build.sbt b/build.sbt index c7bf0dd21e..a378d4ecc7 100644 --- a/build.sbt +++ b/build.sbt @@ -22,8 +22,8 @@ val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion val slf4jSimple = "org.slf4j" % "slf4j-simple" % slf4jVersion val defaultProjectSettings = Seq( - scalaVersion := "2.12.4", - crossScalaVersions := Seq("2.11.11", "2.12.4") + scalaVersion := "2.12.6", + crossScalaVersions := Seq("2.11.11", "2.12.6") ) val baseSettings = Seq( @@ -43,9 +43,9 @@ val baseSettings = Seq( "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", scalacOptions := Seq( - // Note: Add -deprecation when deprecated methods are removed "-target:jvm-1.8", "-unchecked", + "-deprecation", "-feature", "-encoding", "utf8", // Needs -missing-interpolator due to https://issues.scala-lang.org/browse/SI-8761 diff --git a/util-cache/src/main/scala/com/twitter/cache/Refresh.scala b/util-cache/src/main/scala/com/twitter/cache/Refresh.scala index 261cd03a6b..98fe27a010 100644 --- a/util-cache/src/main/scala/com/twitter/cache/Refresh.scala +++ b/util-cache/src/main/scala/com/twitter/cache/Refresh.scala @@ -65,6 +65,6 @@ object Refresh { } } // Return result, which is a no-arg function that returns a future - result + result _ } } diff --git a/util-collection/src/main/scala/com/twitter/util/LruMap.scala b/util-collection/src/main/scala/com/twitter/util/LruMap.scala index a0791dc26d..79a5c32dae 100644 --- a/util-collection/src/main/scala/com/twitter/util/LruMap.scala +++ b/util-collection/src/main/scala/com/twitter/util/LruMap.scala @@ -3,7 +3,7 @@ package com.twitter.util import java.{util => ju} import java.util.LinkedHashMap import scala.collection.JavaConverters._ -import scala.collection.mutable.{Map, MapLike, SynchronizedMap} +import scala.collection.mutable.{Map, MapLike} /** * A wrapper trait for java.util.Map implementations to make them behave as scala Maps. @@ -63,8 +63,34 @@ class LruMap[K, V](val maxSize: Int, val underlying: ju.Map[K, V]) * A synchronized scala `Map` backed by an [[java.util.LinkedHashMap]] */ class SynchronizedLruMap[K, V](maxSize: Int, underlying: ju.Map[K, V]) - extends LruMap[K, V](maxSize, ju.Collections.synchronizedMap(underlying)) - with SynchronizedMap[K, V] { + extends LruMap[K, V](maxSize, ju.Collections.synchronizedMap(underlying)) { + override def get(key: K): Option[V] = synchronized { super.get(key) } + override def iterator: Iterator[(K, V)] = synchronized { super.iterator } + override def +=(kv: (K, V)): this.type = synchronized[this.type] { super.+=(kv) } + override def -=(key: K): this.type = synchronized[this.type] { super.-=(key) } + + override def size: Int = synchronized { super.size } + override def put(key: K, value: V): Option[V] = synchronized { super.put(key, value) } + override def update(key: K, value: V): Unit = synchronized { super.update(key, value) } + override def remove(key: K): Option[V] = synchronized { super.remove(key) } + override def clear(): Unit = synchronized { super.clear() } + override def getOrElseUpdate(key: K, default: => V): V = synchronized { + super.getOrElseUpdate(key, default) + } + override def transform(f: (K, V) => V): this.type = synchronized[this.type] { super.transform(f) } + override def retain(p: (K, V) => Boolean): this.type = synchronized[this.type] { super.retain(p) } + override def values: scala.collection.Iterable[V] = synchronized { super.values } + override def valuesIterator: Iterator[V] = synchronized { super.valuesIterator } + override def clone(): Self = synchronized { super.clone() } + override def foreach[U](f: ((K, V)) => U) = synchronized { super.foreach(f) } + override def apply(key: K): V = synchronized { super.apply(key) } + override def keySet: scala.collection.Set[K] = synchronized { super.keySet } + override def keys: scala.collection.Iterable[K] = synchronized { super.keys } + override def keysIterator: Iterator[K] = synchronized { super.keysIterator } + override def isEmpty: Boolean = synchronized { super.isEmpty } + override def contains(key: K): Boolean = synchronized { super.contains(key) } + override def isDefinedAt(key: K) = synchronized { super.isDefinedAt(key) } override def empty: SynchronizedLruMap[K, V] = new SynchronizedLruMap[K, V](maxSize) + //!!! todo: also add all other methods def this(maxSize: Int) = this(maxSize, LruMap.makeUnderlying(maxSize)) } diff --git a/util-core/src/main/java/com/twitter/concurrent/Offers.java b/util-core/src/main/java/com/twitter/concurrent/Offers.java index a26569b3a1..5ba7a04560 100644 --- a/util-core/src/main/java/com/twitter/concurrent/Offers.java +++ b/util-core/src/main/java/com/twitter/concurrent/Offers.java @@ -4,7 +4,7 @@ import com.twitter.util.Function0; import com.twitter.util.Future; import com.twitter.util.Timer; -import scala.collection.JavaConversions; +import scala.collection.JavaConverters; import scala.runtime.BoxedUnit; import java.util.ArrayList; @@ -60,7 +60,8 @@ public static Offer newTimeoutOffer(Duration duration, Timer timer) { * @see Offer$#choose(scala.collection.Seq) */ public static Offer choose(Collection> offers) { - return Offer$.MODULE$.choose(JavaConversions.asScalaBuffer(new ArrayList>(offers))); + scala.collection.Seq> scalaSeq = JavaConverters.collectionAsScalaIterableConverter(offers).asScala().toSeq(); + return Offer$.MODULE$.choose(scalaSeq); } /** diff --git a/util-core/src/main/java/com/twitter/concurrent/Spools.java b/util-core/src/main/java/com/twitter/concurrent/Spools.java index 6343b997b1..dab7625eaa 100644 --- a/util-core/src/main/java/com/twitter/concurrent/Spools.java +++ b/util-core/src/main/java/com/twitter/concurrent/Spools.java @@ -1,7 +1,10 @@ package com.twitter.concurrent; -import scala.collection.JavaConversions; +import com.twitter.util.Future; + +import scala.collection.JavaConverters; import scala.collection.Seq; +import scala.collection.immutable.List; import java.util.ArrayList; import java.util.Collection; @@ -21,9 +24,18 @@ private Spools() { } /** * Creates a new `Spool` of given `elems`. */ + @SuppressWarnings("unchecked") public static Spool newSpool(Collection elems) { - Seq seq = JavaConversions.asScalaBuffer(new ArrayList(elems)).toSeq(); - return new Spool.ToSpool(seq).toSpool(); + List buffer = (List)List.empty(); + for (T item : elems) { + buffer = buffer.$colon$colon(item); + } + Spool result = (Spool)EMPTY; + while(!buffer.isEmpty()){ + result = new Spool.Cons(buffer.head(), Future.value(result)); + buffer = (List)buffer.tail(); + } + return result; } /** diff --git a/util-core/src/main/java/com/twitter/util/Closables.java b/util-core/src/main/java/com/twitter/util/Closables.java index 5b75507731..d1e38ce9c2 100644 --- a/util-core/src/main/java/com/twitter/util/Closables.java +++ b/util-core/src/main/java/com/twitter/util/Closables.java @@ -1,6 +1,6 @@ package com.twitter.util; -import scala.collection.JavaConversions; +import scala.collection.JavaConverters; import scala.runtime.BoxedUnit; import java.util.Arrays; @@ -43,9 +43,7 @@ public static Future close(Object o, Duration after) { * @see com.twitter.util.Closable$#all(scala.collection.Seq) */ public static Closable all(Closable... closables) { - return Closable$.MODULE$.all( - JavaConversions.asScalaBuffer(Arrays.asList(closables)) - ); + return Closable$.MODULE$.all(closables); } /** @@ -53,7 +51,7 @@ public static Closable all(Closable... closables) { */ public static Closable sequence(Closable... closables) { return Closable$.MODULE$.sequence( - JavaConversions.asScalaBuffer(Arrays.asList(closables)) + scala.collection.mutable.WrappedArray$.MODULE$.make(closables) ); } diff --git a/util-core/src/main/java/com/twitter/util/javainterop/Scala.java b/util-core/src/main/java/com/twitter/util/javainterop/Scala.java index 50528ff78f..625868033a 100644 --- a/util-core/src/main/java/com/twitter/util/javainterop/Scala.java +++ b/util-core/src/main/java/com/twitter/util/javainterop/Scala.java @@ -3,14 +3,13 @@ import scala.Option; import scala.Tuple2; -import static scala.collection.JavaConversions.asScalaBuffer; -import static scala.collection.JavaConversions.asScalaSet; +import scala.collection.JavaConverters; /** * A collection of conversions from Java collections to their immutable * Scala variants. * - * See scala.collection.JavaConversions if you do not need immutable + * See scala.collection.JavaConverters if you do not need immutable * collections. */ public final class Scala { @@ -20,7 +19,7 @@ public final class Scala { /** * Converts a {@link java.util.List} to an immutable Scala Seq. * - * See scala.collection.JavaConversions.asScalaBuffer if you do + * See scala.collection.JavaConverters.asScalaBuffer if you do * not need the returned Seq to be immutable. * * @return an empty Seq if the input is null. @@ -32,14 +31,16 @@ public static scala.collection.immutable.Seq asImmutableSeq( if (jList == null) { return scala.collection.immutable.Seq$.MODULE$.empty(); } else { - return asScalaBuffer(jList).toList(); + //TODO: After dropping support for 2.11.x, replace with + //JavaConverters.asScalaBuffer(jlist).toList(); + return JavaConverters.asScalaBufferConverter(jList).asScala().toList(); } } /** * Converts a {@link java.util.Set} to an immutable Scala Set. * - * See scala.collection.JavaConversions.asScalaSet if you do + * See scala.collection.JavaConverters.asScalaSet if you do * not need the returned Set to be immutable. * * @return an empty Set if the input is null. @@ -51,14 +52,16 @@ public static scala.collection.immutable.Set asImmutableSet( if (jSet == null) { return scala.collection.immutable.Set$.MODULE$.empty(); } else { - return asScalaSet(jSet).toSet(); + //TODO: After dropping support for 2.11.x, replace with + //JavaConverters.asScalaSet(jSet).toSet + return JavaConverters.asScalaSetConverter(jSet).asScala().toSet(); } } /** * Converts a {@link java.util.Map} to an immutable Scala Map. * - * See scala.collection.JavaConversions.asScalaMap if you do + * See scala.collection.JavaConverters.asScalaMap if you do * not need the returned Map to be immutable. * * @return an empty Map if the input is null. diff --git a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala index 2ed1d09c7d..6bc3a160d9 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala @@ -384,7 +384,7 @@ object Spool { */ class ToSpool[A](s: Seq[A]) { def toSpool: Spool[A] = - s.reverse.foldLeft(Spool.empty: Spool[A]) { + s.reverseIterator.foldLeft(Spool.empty: Spool[A]) { case (tail, head) => head *:: Future.value(tail) } } diff --git a/util-core/src/main/scala/com/twitter/util/Closable.scala b/util-core/src/main/scala/com/twitter/util/Closable.scala index 776ad9dc1f..f30734ccdb 100644 --- a/util-core/src/main/scala/com/twitter/util/Closable.scala +++ b/util-core/src/main/scala/com/twitter/util/Closable.scala @@ -4,7 +4,7 @@ import java.lang.ref.{PhantomReference, Reference, ReferenceQueue} import java.{util => ju} import java.util.concurrent.atomic.AtomicReference import java.util.logging.{Level, Logger} -import scala.annotation.tailrec +import scala.annotation.{tailrec, varargs} import scala.util.control.{NonFatal => NF} /** @@ -72,7 +72,7 @@ object Closable { * Concurrent composition: creates a new closable which, when * closed, closes all of the underlying resources simultaneously. */ - def all(closables: Closable*): Closable = new Closable { + @varargs def all(closables: Closable*): Closable = new Closable { def close(deadline: Time): Future[Unit] = { val fs = closables.map { closable => safeClose(closable, deadline) diff --git a/util-core/src/main/scala/com/twitter/util/Pool.scala b/util-core/src/main/scala/com/twitter/util/Pool.scala index 02659d0eb0..86e2a36a80 100644 --- a/util-core/src/main/scala/com/twitter/util/Pool.scala +++ b/util-core/src/main/scala/com/twitter/util/Pool.scala @@ -41,7 +41,7 @@ class SimplePool[A](items: mutable.Queue[Future[A]]) extends Pool[A] { } abstract class FactoryPool[A](numItems: Int) extends Pool[A] { - private val healthyQueue = new HealthyQueue[A](makeItem, numItems, isHealthy) + private val healthyQueue = new HealthyQueue[A](makeItem _, numItems, isHealthy) private val simplePool = new SimplePool[A](healthyQueue) def reserve(): Future[A] = simplePool.reserve() diff --git a/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java b/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java index 19263fb872..c75c515b25 100644 --- a/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java +++ b/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java @@ -4,7 +4,7 @@ import com.twitter.util.Future; import org.junit.Assert; import org.junit.Test; -import scala.collection.JavaConversions; +import scala.collection.JavaConverters; import java.util.Arrays; import java.util.Collection; @@ -49,18 +49,18 @@ public void testSpoolCreation() { public void testSpoolConcat() throws Exception { Spool a = Spools.newSpool(Arrays.asList("a")); Spool b = Spools.newSpool(Arrays.asList("b")); - Spool c = Spools.newSpool(Arrays.asList("c")); + Spool cd = Spools.newSpool(Arrays.asList("c", "d")); Spool ab = a.concat(b); Spool abNothing = ab.concat(Spools.newEmptySpool()); - Spool abc = Await.result(ab.concat(Future.value(c))); + Spool abcd = Await.result(ab.concat(Future.value(cd))); - Collection listA = JavaConversions.seqAsJavaList(Await.result(ab.toSeq())); - Collection listB = JavaConversions.seqAsJavaList(Await.result(abNothing.toSeq())); - Collection listC = JavaConversions.seqAsJavaList(Await.result(abc.toSeq())); + Collection listA = JavaConverters.seqAsJavaListConverter(Await.result(ab.toSeq())).asJava(); + Collection listB = JavaConverters.seqAsJavaListConverter(Await.result(abNothing.toSeq())).asJava(); + Collection listC = JavaConverters.seqAsJavaListConverter(Await.result(abcd.toSeq())).asJava(); Assert.assertArrayEquals(new String[] { "a", "b"}, listA.toArray()); Assert.assertArrayEquals(new String[] { "a", "b"}, listB.toArray()); - Assert.assertArrayEquals(new String[] { "a", "b", "c"}, listC.toArray()); + Assert.assertArrayEquals(new String[] { "a", "b", "c" , "d"}, listC.toArray()); } } diff --git a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala index 0d84b5dd79..a5f2307869 100644 --- a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala @@ -9,7 +9,8 @@ class CancellableTest extends WordSpec { "CancellableSink" should { "cancel once" in { var count = 0 - val s = new CancellableSink { count += 1 } + def increment: Unit = count += 1 + val s = new CancellableSink(increment) s.cancel() assert(count == 1) s.cancel() diff --git a/util-core/src/test/scala/com/twitter/util/TimerTest.scala b/util-core/src/test/scala/com/twitter/util/TimerTest.scala index 18c2f594ac..c91f1ba048 100644 --- a/util-core/src/test/scala/com/twitter/util/TimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimerTest.scala @@ -289,7 +289,7 @@ class TimerTest extends FunSuite with MockitoSugar with Eventually with Integrat Time.withCurrentTimeFrozen { ctl => val timer = new MockTimer assert(timer.tasks.isEmpty) - val task = timer.schedule(Time.now + 3.seconds)() + val task = timer.schedule(Time.now + 3.seconds)(()) assert(timer.tasks.size == 1) task.cancel() assert(timer.tasks.isEmpty) @@ -300,7 +300,7 @@ class TimerTest extends FunSuite with MockitoSugar with Eventually with Integrat Time.withCurrentTimeFrozen { ctl => val timer = new MockTimer assert(timer.tasks.isEmpty) - val task = timer.schedule(3.seconds)() + val task = timer.schedule(3.seconds)(()) eventually { assert(timer.tasks.size == 1) } ctl.advance(30.seconds) timer.tick() diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index 0c664bab42..c199c23288 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -276,12 +276,13 @@ class LoggerTest extends WordSpec with TempFolder with BeforeAndAfter { } "file handler" in { + val separator = java.io.File.pathSeparator withTempFolder { val log: Logger = LoggerFactory( node = "com.twitter", level = Some(Level.DEBUG), handlers = FileHandler( - filename = folderName + "/test.log", + filename = folderName + separator + "test.log", rollPolicy = Policy.Never, append = false, level = Some(Level.INFO), @@ -296,7 +297,8 @@ class LoggerTest extends WordSpec with TempFolder with BeforeAndAfter { assert(log.getLevel == Level.DEBUG) assert(log.getHandlers().length == 1) val handler = log.getHandlers()(0).asInstanceOf[FileHandler] - assert(handler.filename == folderName + "/test.log") + val fileName = folderName + separator + "test.log" + assert(handler.filename == fileName) assert(handler.append == false) assert(handler.getLevel == Level.INFO) val formatter = handler.formatter diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 1a933858df..ca767a954e 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -232,7 +232,7 @@ class InMemoryStatsReceiverTest extends FunSuite with Eventually with Integratio inMemoryStatsReceiver.print(ps, includeHeaders = true) val content = new String(baos.toByteArray, StandardCharsets.UTF_8) - val parts = content.split('\n') + val parts = content.filterNot(_ == '\r').split('\n') assert(parts.length == 17) assert(parts(0) == "Counters:") @@ -295,7 +295,7 @@ class InMemoryStatsReceiverTest extends FunSuite with Eventually with Integratio inMemoryStatsReceiver.print(ps, includeHeaders = true) val content = new String(baos.toByteArray, StandardCharsets.UTF_8) - val parts = content.split('\n') + val parts = content.filterNot(_ == '\r').split('\n') assert(parts.length == 12) assert(parts(0) == "Counters:") diff --git a/util-zk/src/main/scala/com/twitter/zk/Event.scala b/util-zk/src/main/scala/com/twitter/zk/Event.scala index b206cbe4d7..149eddebf3 100644 --- a/util-zk/src/main/scala/com/twitter/zk/Event.scala +++ b/util-zk/src/main/scala/com/twitter/zk/Event.scala @@ -54,19 +54,18 @@ object StateEvent { } def apply(w: WatchedEvent): StateEvent = { - w.getState match { + val state = w.getState + state match { case KeeperState.AuthFailed => AuthFailed case KeeperState.SyncConnected => Connected case KeeperState.Disconnected => Disconnected case KeeperState.Expired => Expired case KeeperState.ConnectedReadOnly => ConnectedReadOnly case KeeperState.SaslAuthenticated => SaslAuthenticated - case KeeperState.Unknown => - throw new IllegalArgumentException("Can't convert deprecated state to StateEvent: Unknown") - case KeeperState.NoSyncConnected => - throw new IllegalArgumentException( - "Can't convert deprecated state to StateEvent: NoSyncConnected" - ) + case _ => + throw new IllegalArgumentException(s"Can't convert deprecated state to StateEvent: $state") + //NoSyncConnected and Unknown are depricated in zk 3.x, and should be + //expected to be removed in zk 4.x } } } From 76a608986e5240be8e2872c3bce978f2121d0a4d Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 19 Sep 2018 20:44:38 +0000 Subject: [PATCH 011/796] twitter-oss: Update to Scala 2.11.12 and 2.12.6 in SBT build Problem/Solution We should update our SBT builds to Scala 2.11.12 and 2.12.6. Additionally, the Slack credentials for each project are out-of-date and need to be updated. JIRA Issues: CSL-6972, CSL-6975 Differential Revision: https://phabricator.twitter.biz/D216930 --- .travis.yml | 6 +++--- build.sbt | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 85263e6ebc..22a61a44bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,15 +19,15 @@ before_cache: - find $HOME/.sbt -name "*.lock" -delete scala: - - 2.11.11 - - 2.12.4 + - 2.11.12 + - 2.12.6 jdk: - oraclejdk8 notifications: slack: - secure: E0knpocRWQT1YI4TgAZGxS7Xoe15wWZ5fbJsi0nfMhxHSlngrnxAknN60qovPa2J8R/opryX/0jwaUF2/5RPYVxf6ARNI428WkrltxctQeCugN8j5XqpBqXLR/+GPetqFe9WwF6ycg3bmThsXnap6/xNB75zMWj+2E1CYTqwpA0= + secure: BFX2aB0sq5cMxmxlWIwgHwJOjI5W6WBn3YSPqHYYz6VAK6fzlX2VUKsfyQJTUnqMYox/0qGzPUB4TFhVMH3c78ocNZNj/9tPOPLkMfHjAqobnq/89+hp4Q6fmWCMKVAGTzVNq83OCykYkzITcYWxo0Pyc2eguW6tHaHIgqnbCRY= before_script: # default $SBT_OPTS is irrelevant to sbt lancher diff --git a/build.sbt b/build.sbt index a378d4ecc7..9bdb0b870f 100644 --- a/build.sbt +++ b/build.sbt @@ -23,7 +23,7 @@ val slf4jSimple = "org.slf4j" % "slf4j-simple" % slf4jVersion val defaultProjectSettings = Seq( scalaVersion := "2.12.6", - crossScalaVersions := Seq("2.11.11", "2.12.6") + crossScalaVersions := Seq("2.11.12", "2.12.6") ) val baseSettings = Seq( @@ -299,7 +299,7 @@ lazy val utilIntellij = Project( baseSettings ).settings( name := "util-intellij", - scalaVersion := "2.11.11" + scalaVersion := "2.11.12" ).dependsOn(utilCore % "test") lazy val utilJvm = Project( From 2cc66e3c201d68c73c7a4bba7d64567158a6661f Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 20 Sep 2018 23:43:29 +0000 Subject: [PATCH 012/796] twitter-oss: Update TravisCI to use sudo = required Problem It has been recommended by the TravisCI support that we switch our builds to use `sudo: required` in order to get a fully virtualized environment for running our builds. See: https://docs.travis-ci.com/user/reference/trusty/#fully-virtualized-via-sudo-required We hadn't switched as we were beholden to needing a Java install which wasn't readily available previously but is available in the TravisCI "trusty" image. Solution Switch the TravisCI configuration to use `sudo: required` and `dist: trusty`. Differential Revision: https://phabricator.twitter.biz/D218377 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 22a61a44bf..7ce0c7deda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ -# container-based build -sudo: false +dist: trusty +sudo: required language: scala From e0b39d7c3ffb718f727f29e3dc2a18dbb76f15b0 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Mon, 24 Sep 2018 17:27:10 +0000 Subject: [PATCH 013/796] util-core: Remove synchronization from DetachableFuture Problem DetachableFuture uses synchronization to ensure its boolean field is only set once. Solution Use `Unsafe.compareAndSwapInt` to get the semantics without resorting to synchronization. Result JMH results show it taking 62% less ns/op. JIRA Issues: CSL-7003 Differential Revision: https://phabricator.twitter.biz/D217509 --- .../com/twitter/util/PromiseBenchmark.scala | 12 ++++++++++++ .../main/scala/com/twitter/util/Promise.scala | 18 +++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala index ebce037ca8..74779e5d88 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala @@ -21,6 +21,12 @@ class PromiseBenchmark extends StdBenchAnnotations { promise.detach() } + @Benchmark + def detachDetachableFuture(state: PromiseBenchmark.PromiseDetachState): Unit = { + import state._ + detachableFuture.detach() + } + // used to isolate the work in the `updateIfEmpty` benchmark @Benchmark def newUnsatisfiedPromise(): Promise[String] = { @@ -76,11 +82,17 @@ object PromiseBenchmark { var attachedFutures: List[Future[Unit]] = _ var promise: Promise[Unit] with Detachable = _ + var detachableFuture: Promise[Int] with Detachable = _ + @Setup def prepare(): Unit = { global = new Promise[Unit] attachedFutures = List.fill(numAttached) { Promise.attached(global) } promise = Promise.attached(global) + + // explicitly using a Future that is not a `Promise` to + // test the `DetachableFuture` code paths. + detachableFuture = Promise.attached(new ConstFuture(Return(5))) } } diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 6bb12e5af3..a9a8afeeaa 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -172,16 +172,12 @@ object Promise { with Detachable with (Try[A] => Unit) { - private[this] var detached: Boolean = false + // 0 represents not yet detached, 1 represents detached. + @volatile + private[this] var alreadyDetached: Int = 0 - def detach(): Boolean = synchronized { - if (detached) { - false - } else { - detached = true - true - } - } + def detach(): Boolean = + unsafe.compareAndSwapInt(this, detachedFutureOffset, 0, 1) def apply(result: Try[A]): Unit = if (detach()) update(result) @@ -277,6 +273,10 @@ object Promise { private val unsafe: sun.misc.Unsafe = Unsafe() private val stateOff: Long = unsafe.objectFieldOffset(classOf[Promise[_]].getDeclaredField("state")) + + private val detachedFutureOffset: Long = + unsafe.objectFieldOffset(classOf[DetachableFuture[_]].getDeclaredField("alreadyDetached")) + private val AlwaysUnit: Any => Unit = _ => () sealed trait Responder[A] { this: Future[A] => From e151b2da0b8543537ec017e89f6ae4c50e8c2cb0 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 24 Sep 2018 17:59:36 +0000 Subject: [PATCH 014/796] twitter-oss: Introduce Dodo build stage in TravisCI config Problem Currently, running the dodo script to build the dependencies of a project happens *before every job* when it only needs to run only once before all jobs. This causes unnecessary computation and churn. Additionally, we are running "./sbt update" unnecessarily as it was done because of the flakiness with `maven.twttr.com` in resolving dependencies which would kill builds. Since we no longer rely on it, we can remove the manual resolution of dependencies in the jobs. Solution Introduce a Dodo build stage and order it before the default test stage such that it runs before any testing and only once. The output is cached, which is then used in the subsequent jobs in the test build stage. This has the additional benefit of removing the time spent building with dodo from the test jobs, which means they have more time to finish within the TravisCI job timeout. Result Better stability with our TravisCI testing. Building with Dodo is only run once for the TravisCI build, it has up to the default time for a job to finish, and every test afterwards also now can spend up to the maximum timeout to actually run tests. JIRA Issues: CSL-6974 Differential Revision: https://phabricator.twitter.biz/D219136 --- .travis.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7ce0c7deda..0b8830f323 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,14 +30,12 @@ notifications: secure: BFX2aB0sq5cMxmxlWIwgHwJOjI5W6WBn3YSPqHYYz6VAK6fzlX2VUKsfyQJTUnqMYox/0qGzPUB4TFhVMH3c78ocNZNj/9tPOPLkMfHjAqobnq/89+hp4Q6fmWCMKVAGTzVNq83OCykYkzITcYWxo0Pyc2eguW6tHaHIgqnbCRY= before_script: - # default $SBT_OPTS is irrelevant to sbt lancher - - unset SBT_OPTS - - travis_retry ./sbt --error ++$TRAVIS_SCALA_VERSION update + - unset SBT_OPTS # default $SBT_OPTS is irrelevant to sbt launcher script: - - travis_retry ./sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport - - if [[ "$TRAVIS_SCALA_VERSION" == 2.11* ]]; then travis_retry ./sbt ++$TRAVIS_SCALA_VERSION "project util-intellij" coverage test coverageReport; fi - - ./sbt ++$TRAVIS_SCALA_VERSION coverageAggregate + - travis_retry ./sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport + - if [[ "$TRAVIS_SCALA_VERSION" == 2.11* ]]; then travis_retry ./sbt ++$TRAVIS_SCALA_VERSION "project util-intellij" coverage test coverageReport; fi after_success: - - bash <(curl -s https://codecov.io/bash) + - ./sbt ++$TRAVIS_SCALA_VERSION coverageAggregate + - bash <(curl -s https://codecov.io/bash) From 9cc7025c52f8d52acc0ccc7f3f713103bf8fcd6a Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 25 Sep 2018 16:30:59 +0000 Subject: [PATCH 015/796] util-core: Writer extends Closable Problem/Solution: `Writer` should support for initiating closures. JIRA Issues: CSL-6688 TBR=true Differential Revision: https://phabricator.twitter.biz/D218453 --- CHANGELOG.rst | 6 +++++ .../src/main/java/com/twitter/io/Writers.java | 4 ++-- .../com/twitter/io/OutputStreamWriter.scala | 3 +-- .../src/main/scala/com/twitter/io/Pipe.scala | 4 ++-- .../main/scala/com/twitter/io/Writer.scala | 23 +++++++------------ .../twitter/io/exp/MinimumThroughput.scala | 2 ++ .../com/twitter/io/WriterCompilationTest.java | 2 +- .../io/exp/MinimumThroughputTest.scala | 2 ++ 8 files changed, 24 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 48cb02192d..e9d374d00a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: `c.t.io.Writer` now extends `c.t.util.Closable`. `c.t.io.Writer.ClosableWriter` + is no longer exist. ``PHAB_ID=D218453`` + 18.9.0 ------- diff --git a/util-core/src/main/java/com/twitter/io/Writers.java b/util-core/src/main/java/com/twitter/io/Writers.java index b89a2afc9a..53bb484020 100644 --- a/util-core/src/main/java/com/twitter/io/Writers.java +++ b/util-core/src/main/java/com/twitter/io/Writers.java @@ -15,14 +15,14 @@ public final class Writers { /** * See {@code com.twitter.io.Writer.fromOutputStream} */ - public static Writer.ClosableWriter newOutputStreamWriter(OutputStream out) { + public static Writer newOutputStreamWriter(OutputStream out) { return Writer$.MODULE$.fromOutputStream(out); } /** * See {@code com.twitter.io.Writer.fromOutputStream} */ - public static Writer.ClosableWriter newOutputStreamWriter(OutputStream out, int bufsize) { + public static Writer newOutputStreamWriter(OutputStream out, int bufsize) { return Writer$.MODULE$.fromOutputStream(out, bufsize); } diff --git a/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala b/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala index 23b2dbe8b1..1f504d401c 100644 --- a/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala +++ b/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala @@ -1,6 +1,5 @@ package com.twitter.io -import com.twitter.io.Writer.ClosableWriter import com.twitter.util.{Future, FuturePool, Promise, Return, Throw, Time} import java.io.OutputStream import java.util.concurrent.atomic.AtomicReference @@ -9,7 +8,7 @@ import scala.annotation.tailrec /** * Construct a Writer from a given OutputStream. */ -private[io] class OutputStreamWriter(out: OutputStream, bufsize: Int) extends ClosableWriter[Buf] { +private[io] class OutputStreamWriter(out: OutputStream, bufsize: Int) extends Writer[Buf] { import com.twitter.io.OutputStreamWriter._ private[this] val done = new Promise[Unit] diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 5ebcdccc93..8be08abd28 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -1,6 +1,6 @@ package com.twitter.io -import com.twitter.util.{Closable, Future, Promise, Return, Time} +import com.twitter.util.{Future, Promise, Return, Time} /** * A synchronous in-memory pipe that connects [[Reader]] and [[Writer]] in the sense @@ -13,7 +13,7 @@ import com.twitter.util.{Closable, Future, Promise, Return, Time} * It is safe to call `read`, `write`, and `close` concurrently. The individual calls * are synchronized on the given [[Pipe]]. */ -final class Pipe[A <: Buf] extends Reader[A] with Writer[A] with Closable { +final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { import Pipe._ diff --git a/util-core/src/main/scala/com/twitter/io/Writer.scala b/util-core/src/main/scala/com/twitter/io/Writer.scala index 9c37d688ba..c93d2b32cf 100644 --- a/util-core/src/main/scala/com/twitter/io/Writer.scala +++ b/util-core/src/main/scala/com/twitter/io/Writer.scala @@ -7,7 +7,7 @@ import java.io.OutputStream * A Writer represents a sink for a stream of `A`s, providing a convenient interface * for the producer of such streams. */ -trait Writer[-A] { +trait Writer[-A] extends Closable { /** * Write a chunk. The returned future is completed when the chunk has been @@ -29,29 +29,22 @@ trait Writer[-A] { object Writer { /** - * A [[ClosableWriter]] instance that will always fail. Useful for situations + * A [[Writer]] instance that will always fail. Useful for situations * where writing to the [[Writer]] is nonsensical such as the [[Writer]] on the * `Response` returned by the Finagle HTTP client. */ - val FailingWriter: Writer[Buf] with Closable = fail[Buf] + val FailingWriter: Writer[Buf] = fail[Buf] - def fail[A]: Writer[A] with Closable = new Writer[A] with Closable { + def fail[A]: Writer[A] = new Writer[A] { def write(buf: A): Future[Unit] = Future.exception(new IllegalStateException("NullWriter")) def fail(cause: Throwable): Unit = () def close(deadline: Time): Future[Unit] = Future.Done } - /** - * Represents a [[Writer]] which is [[Closable]]. - * - * Exists primarily for Java compatibility. - */ - trait ClosableWriter[A] extends Writer[A] with Closable - val BufferSize: Int = 4096 /** - * Construct a [[ClosableWriter]] from a given OutputStream. + * Construct a [[Writer]] from a given OutputStream. * * This [[Writer]] is not thread safe. If multiple threads attempt to `write`, the * behavior is identical to multiple threads calling `write` on the underlying @@ -59,16 +52,16 @@ object Writer { * * @param bufsize Size of the copy buffer between Writer and OutputStream. */ - def fromOutputStream(out: OutputStream, bufsize: Int): ClosableWriter[Buf] = + def fromOutputStream(out: OutputStream, bufsize: Int): Writer[Buf] = new OutputStreamWriter(out, bufsize) /** - * Construct a [[ClosableWriter]] from a given OutputStream. + * Construct a [[Writer]] from a given OutputStream. * * This [[Writer]] is not thread safe. If multiple threads attempt to `write`, the * behavior is identical to multiple threads calling `write` on the underlying * OutputStream. */ - def fromOutputStream(out: OutputStream): ClosableWriter[Buf] = + def fromOutputStream(out: OutputStream): Writer[Buf] = fromOutputStream(out, BufferSize) } diff --git a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala b/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala index a756346931..28189d27d9 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala +++ b/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala @@ -129,6 +129,8 @@ object MinimumThroughput { } } } + + def close(deadline: Time): Future[Unit] = writer.close(deadline) } /** diff --git a/util-core/src/test/java/com/twitter/io/WriterCompilationTest.java b/util-core/src/test/java/com/twitter/io/WriterCompilationTest.java index ce8502f98a..d30f24441e 100644 --- a/util-core/src/test/java/com/twitter/io/WriterCompilationTest.java +++ b/util-core/src/test/java/com/twitter/io/WriterCompilationTest.java @@ -9,7 +9,7 @@ public class WriterCompilationTest { @Test public void testNewOutputStreamWriter() { - Writer.ClosableWriter w = Writers.newOutputStreamWriter(new ByteArrayOutputStream()); + Writer w = Writers.newOutputStreamWriter(new ByteArrayOutputStream()); w.close(); } diff --git a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala b/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala index 1eb09ff777..55c48e7fb0 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala +++ b/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala @@ -187,6 +187,8 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { } Future.Done } + + def close(deadline: Time): Future[Unit] = Future.Done } val writer = MinimumThroughput.writer( From cd01f464c4b85a21657695a6a021def6be96f90a Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 25 Sep 2018 17:05:26 +0000 Subject: [PATCH 016/796] util-logging: Remove synchronization from ThrottledHandler Problem ThrottledHandler has a bunch of unnecessary synchronization which leads to lock contention for some service owners. Solution Replace the synchronization with a volatile, AtomicInteger, and a ConcurrentHashMap. Result Less contention. JIRA Issues: CSL-7012 Differential Revision: https://phabricator.twitter.biz/D218182 --- .../twitter/logging/ThrottledHandler.scala | 76 +++++++++---------- .../logging/ThrottledHandlerTest.scala | 13 ++-- 2 files changed, 43 insertions(+), 46 deletions(-) diff --git a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala index f7b237ba32..a057a76a28 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala @@ -16,14 +16,14 @@ package com.twitter.logging -import java.util.concurrent.atomic.AtomicReference -import java.util.{logging => javalog} - -import scala.annotation.tailrec -import scala.collection.mutable - import com.twitter.conversions.time._ import com.twitter.util.{Duration, Time} +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} +import java.util.{logging => javalog} +import java.util.function.{Function => JFunction} +import scala.annotation.tailrec +import scala.collection.JavaConverters._ object ThrottledHandler { @@ -48,7 +48,9 @@ object ThrottledHandler { handler: HandlerFactory, duration: Duration = 0.seconds, maxToDisplay: Int = Int.MaxValue - ) = () => new ThrottledHandler(handler(), duration, maxToDisplay) + ): () => ThrottledHandler = () => new ThrottledHandler(handler(), duration, maxToDisplay) + + private val OneSecond = 1.second } /** @@ -72,33 +74,32 @@ class ThrottledHandler( val maxToDisplay: Int ) extends ProxyHandler(handler) { + // we accept some raciness here. it means that sometimes extra log lines will + // get logged. this was deemed preferable to holding locks. private class Throttle(startTime: Time, name: String, level: javalog.Level) { + @volatile private[this] var expired = false - private[this] var count = 0 - override def toString = "Throttle: startTime=" + startTime + " count=" + count + private[this] val count = new AtomicInteger(0) + + override def toString: String = "Throttle: startTime=" + startTime + " count=" + count.get() final def add(record: javalog.LogRecord, now: Time): Boolean = { - val (shouldPublish, added) = synchronized { - if (!expired) { - count += 1 - (count <= maxToDisplay, true) - } else { - (false, false) + if (!expired) { + if (count.incrementAndGet() <= maxToDisplay) { + doPublish(record) } + true + } else { + false } - - if (shouldPublish) doPublish(record) - added } final def removeIfExpired(now: Time): Boolean = { - val didExpire = synchronized { - expired = (now - startTime >= duration) - expired - } + val didExpire = now - startTime >= duration + expired = didExpire - if (didExpire && count > maxToDisplay) publishSwallowed() + if (didExpire && count.get() > maxToDisplay) publishSwallowed() didExpire } @@ -106,7 +107,7 @@ class ThrottledHandler( private[this] def publishSwallowed(): Unit = { val throttledRecord = new javalog.LogRecord( level, - "(swallowed %d repeating messages)".format(count - maxToDisplay) + "(swallowed %d repeating messages)".format(count.get() - maxToDisplay) ) throttledRecord.setLoggerName(name) doPublish(throttledRecord) @@ -114,7 +115,7 @@ class ThrottledHandler( } private val lastFlushCheck = new AtomicReference(Time.epoch) - private val throttleMap = new mutable.HashMap[String, Throttle] + private val throttleMap = new ConcurrentHashMap[String, Throttle]() @deprecated("Use flushThrottled() instead", "5.3.13") def reset(): Unit = { @@ -125,11 +126,9 @@ class ThrottledHandler( * Force printing any "swallowed" messages. */ def flushThrottled(): Unit = { - synchronized { - val now = Time.now - throttleMap retain { - case (_, throttle) => !throttle.removeIfExpired(now) - } + val now = Time.now + throttleMap.asScala.retain { + case (_, throttle) => !throttle.removeIfExpired(now) } } @@ -141,7 +140,7 @@ class ThrottledHandler( val now = Time.now val last = lastFlushCheck.get - if (now - last > 1.second && lastFlushCheck.compareAndSet(last, now)) { + if (now - last > ThrottledHandler.OneSecond && lastFlushCheck.compareAndSet(last, now)) { flushThrottled() } @@ -150,12 +149,13 @@ class ThrottledHandler( case _ => record.getMessage } @tailrec def tryPublish(): Unit = { - val throttle = synchronized { - throttleMap.getOrElseUpdate( - key, - new Throttle(now, record.getLoggerName(), record.getLevel()) - ) - } + val throttle = throttleMap.computeIfAbsent( + key, + new JFunction[String, Throttle] { + def apply(key: String): Throttle = + new Throttle(now, record.getLoggerName(), record.getLevel()) + } + ) // catch the case where throttle is removed before we had a chance to add if (!throttle.add(record, now)) tryPublish() @@ -164,7 +164,7 @@ class ThrottledHandler( tryPublish() } - private def doPublish(record: javalog.LogRecord) = { + private def doPublish(record: javalog.LogRecord): Unit = { super.publish(record) } } diff --git a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala index e2f0b65de8..6a990588f8 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala @@ -16,24 +16,21 @@ package com.twitter.logging -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -import org.scalatest.{BeforeAndAfter, WordSpec} - import com.twitter.conversions.time._ import com.twitter.util.{TempFolder, Time} -@RunWith(classOf[JUnitRunner]) +import org.scalatest.{BeforeAndAfter, WordSpec} + class ThrottledHandlerTest extends WordSpec with BeforeAndAfter with TempFolder { - private var handler: StringHandler = null + private var handler: StringHandler = _ "ThrottledHandler" should { before { - Logger.clearHandlers + Logger.clearHandlers() handler = new StringHandler(BareFormatter, None) } after { - Logger.clearHandlers + Logger.clearHandlers() } "throttle keyed log messages" in { From 1d6e539ecd41afb844c9c946b071b367b05c0fd2 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 25 Sep 2018 21:10:50 +0000 Subject: [PATCH 017/796] util-core: Scaladoc for Pipes # Problem / Solution We need better docs for Pipes. Here are the better docs for pipes. JIRA Issues: CSL-2701 Differential Revision: https://phabricator.twitter.biz/D218115 --- .../src/main/scala/com/twitter/io/Pipe.scala | 116 +++++++++++++----- 1 file changed, 84 insertions(+), 32 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 8be08abd28..38f0a3e98f 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -6,12 +6,70 @@ import com.twitter.util.{Future, Promise, Return, Time} * A synchronous in-memory pipe that connects [[Reader]] and [[Writer]] in the sense * that a reader's input is the output of a writer. * + * A pipe is structured as a smash of both interfaces, a [[Reader]] and a [[Writer]] such that can + * be passed directly to a consumer or a producer. + * + * {{{ + * def consumer(r: Reader[Buf]): Future[Unit] = ??? + * def producer(w: Writer[Buf]): Future[Unit] = ??? + * + * val p = new Pipe[Buf] + * + * consumer(p) + * producer(p) + * }}} + * * Reads and writes on the pipe are matched one to one and only one outstanding `read` - * or `write` is permitted in the current implementation. That is, the `write` (its - * returned [[Future]]) is resolved when the `read` consumes the written data. + * or `write` is permitted in the current implementation (multiple pending writes or reads + * resolve into [[IllegalStateException]] while leaving the pipe healthy). That is, the `write` + * (its returned [[Future]]) is resolved when the `read` consumes the written data. + * + * Here is, for example, a very typical write-loop that writes into a pipe-backed [[Writer]]: + * + * {{{ + * def writeLoop(w: Writer[Buf], data: List[Buf]): Future[Unit] = data match { + * case h :: t => p.write(h).before(writeLoop(w, t)) + * case Nil => w.close() + * } + * }}} + * + * Reading from a pipe-backed [[Reader]] is no different from working with any other reader: + * + *{{{ + * def readLoop(r: Reader[Buf], process: Buf => Future[Unit]): Future[Unit] = r.read().flatMap { + * case Some(chunk) => process(chunk).before(readLoop(r, process)) + * case None => Future.Done + * } + * }}} + * + * == Thread Safety == * - * It is safe to call `read`, `write`, and `close` concurrently. The individual calls - * are synchronized on the given [[Pipe]]. + * It is safe to call `read`, `write`, `fail`, `discard`, and `close` concurrently. The individual + * calls are synchronized on the given [[Pipe]]. + * + * == Closing or Failing Pipes == + * + * Besides expecting a write or a read, a pipe can be closed or failed. A writer can do both `close` + * and `fail` the pipe, while reader can only fail the pipe via `discard`. + * + * The following behavior is expected with regards to reading from or writing into a closed or a + * failed pipe: + * + * - Writing into a closed pipe yields [[IllegalStateException]] + * - Reading from a closed pipe yields EOF ([[Future.None]]) + * - Reading from or writing into a failed pipe yields a failure it was failed with + * + * It's also worth discussing how pipes are being closed. As closure is always initiated by a + * producer (writer), there is a machinery allowing it to be notified when said closure is observed + * by a consumer (reader). + * + * The following rules should help reasoning about closure signals in pipes: + * + * - Closing a pipe with a pending read resolves said read into EOF and returns a [[Future.Unit]] + * - Closing a pipe with a pending write fails said write with [[IllegalStateException]] and + * returns a future that will be satisfied when a consumer observes the closure (EOF) via read + * - Closing an idle pipe returns a future that will be satisfied when a consumer observes the + * closure (EOF) via read */ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { @@ -22,15 +80,15 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { def read(n: Int): Future[Option[A]] = synchronized { state match { - case State.Failing(exc) => + case State.Failed(exc) => Future.exception(exc) case State.Closing(reof) => - state = State.Eof + state = State.Closed reof.setDone() Future.None - case State.Eof => + case State.Closed => Future.None case State.Idle => @@ -50,22 +108,19 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { Future.value(Some(buf.slice(0, n).asInstanceOf[A])) case State.Reading(_, _) => - Future.exception(new IllegalStateException("read() while Reading")) + Future.exception(new IllegalStateException("read() while read is pending")) } } - def discard(): Unit = synchronized { - val cause = new Reader.ReaderDiscarded() - fail(cause) - } + def discard(): Unit = fail(new Reader.ReaderDiscarded()) def write(buf: A): Future[Unit] = synchronized { state match { - case State.Failing(exc) => + case State.Failed(exc) => Future.exception(exc) - case State.Eof | State.Closing(_) => - Future.exception(new IllegalStateException("write after close")) + case State.Closed | State.Closing(_) => + Future.exception(new IllegalStateException("write() while closed")) case State.Idle => val p = new Promise[Unit]() @@ -86,37 +141,34 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { Future.Done case State.Writing(_, _) => - Future.exception(new IllegalStateException("write while Writing")) + Future.exception(new IllegalStateException("write() while write is pending")) } } def fail(cause: Throwable): Unit = synchronized { - // We fix the state before inspecting it so we enter in a - // good state if setting promises recurses. - val oldState = state - oldState match { - case State.Eof | State.Failing(_) => + state match { + case State.Closed | State.Failed(_) => // do not update state to failing case State.Idle => - state = State.Failing(cause) + state = State.Failed(cause) case State.Closing(reof) => - state = State.Failing(cause) + state = State.Failed(cause) reof.setException(cause) case State.Reading(_, p) => - state = State.Failing(cause) + state = State.Failed(cause) p.setException(cause) case State.Writing(_, p) => - state = State.Failing(cause) + state = State.Failed(cause) p.setException(cause) } } def close(deadline: Time): Future[Unit] = synchronized { state match { - case State.Failing(t) => + case State.Failed(t) => Future.exception(t) - case State.Eof => + case State.Closed => Future.Done case State.Closing(p) => @@ -128,14 +180,14 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { reof case State.Reading(_, p) => - state = State.Eof + state = State.Closed p.update(Return.None) Future.Done case State.Writing(_, p) => val reof = new Promise[Unit]() state = State.Closing(reof) - p.setException(new IllegalStateException("close while write is pending")) + p.setException(new IllegalStateException("close() while write is pending")) reof } } @@ -168,8 +220,8 @@ object Pipe { */ final case class Writing[A <: Buf](buf: A, p: Promise[Unit]) extends State[A] - /** Indicates this was `fail`-ed or `discard`-ed. */ - final case class Failing(exc: Throwable) extends State[Nothing] + /** Indicates the pipe was failed. */ + final case class Failed(exc: Throwable) extends State[Nothing] /** * Indicates a close occurred while `Idle` — no reads or writes were pending. @@ -179,6 +231,6 @@ object Pipe { final case class Closing(reof: Promise[Unit]) extends State[Nothing] /** Indicates the reader has seen the EOF. No more reads or writes are allowed. */ - case object Eof extends State[Nothing] + case object Closed extends State[Nothing] } } From 3d6d7457d34055a9e9c86ffd112cc477fd557e0f Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 26 Sep 2018 19:16:48 +0000 Subject: [PATCH 018/796] util-slf4j-api: Move slf4j-simple to Test Dependencies Only Problem slf4j-simple is listed as a standard dependency of util-slf4j-api which is incorrect. This causes problems for open source users when they specify their own concrete version of the slf4j-api. Solution Move slf4j-simple to the test section only. Differential Revision: https://phabricator.twitter.biz/D220718 --- CHANGELOG.rst | 6 ++++++ build.sbt | 5 ++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e9d374d00a..0d0fc3d37b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,12 @@ Breaking API Changes * util-core: `c.t.io.Writer` now extends `c.t.util.Closable`. `c.t.io.Writer.ClosableWriter` is no longer exist. ``PHAB_ID=D218453`` +Bug Fixes +~~~~~~~~~ + +* util-slf4j-api: Moved slf4j-simple dependency to be a 'test' dependency, instead of a + compile dependency, which was inaccurate. ``PHAB_ID=D220718`` + 18.9.0 ------- diff --git a/build.sbt b/build.sbt index 9bdb0b870f..12cec026d2 100644 --- a/build.sbt +++ b/build.sbt @@ -19,7 +19,6 @@ val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.3.4" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.13.4" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion -val slf4jSimple = "org.slf4j" % "slf4j-simple" % slf4jVersion val defaultProjectSettings = Seq( scalaVersion := "2.12.6", @@ -252,7 +251,7 @@ lazy val utilDoc = Project( ).enablePlugins( SphinxPlugin ).settings( - sharedSettings ++ + sharedSettings ++ noPublishSettings ++ Seq( scalacOptions in doc ++= Seq("-doc-title", "Util", "-doc-version", version.value), includeFilter in Sphinx := ("*.html" | "*.png" | "*.svg" | "*.js" | "*.css" | "*.gif" | "*.txt") @@ -338,7 +337,7 @@ lazy val utilSlf4jApi = Project( name := "util-slf4j-api", libraryDependencies ++= Seq( slf4jApi, - slf4jSimple + "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" ) ).dependsOn(utilCore % "test") From a41dba5408cc429ef3bf2f250972e1846ee38cc5 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 26 Sep 2018 20:14:03 +0000 Subject: [PATCH 019/796] twitter-oss: Prepare OSS libraries for release 18.9.1 Problem We want to release the next versions of our Twitter OSS libraries 18.9.1 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D220746 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0d0fc3d37b..4179ca1f98 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.9.1 +------- + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index a258287748..8564a7c1a1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "18.9.0" +val collUtils = "com.twitter" %% "util-collection" % "18.9.1" ``` # Units diff --git a/build.sbt b/build.sbt index 12cec026d2..d7765147b2 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.10.0-SNAPSHOT" +val releaseVersion = "18.9.1" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 16e1a74a04f6ec87e6f6768d16f4096269e80c0d Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 27 Sep 2018 17:44:41 +0000 Subject: [PATCH 020/796] twitter-oss: Update OSS Libraries Post-Release Back to 18.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 18.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D221275 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8564a7c1a1..bebe09a8fd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index d7765147b2..12cec026d2 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.9.1" +val releaseVersion = "18.10.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From bdb43b1b81cdc0e623cf97c81f527840f5061005 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 27 Sep 2018 17:58:47 +0000 Subject: [PATCH 021/796] util-core: Examples for some Future APIs Problem/Solution: Adding examples for `flatMap`, `map`, `liftToTry`, `lowerFromTry`, and callback methods `onSuccess`, `onFailure`, `ensure`. JIRA Issues: CSL-6674 Differential Revision: https://phabricator.twitter.biz/D220730 --- .../main/scala/com/twitter/util/Future.scala | 98 ++++++++++++++++++- 1 file changed, 94 insertions(+), 4 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 353c9f44bc..a359690819 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1426,6 +1426,19 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * @see The [[https://twitter.github.io/finagle/guide/Futures.html user guide]] * on concurrent programming with Futures. * @see [[Futures]] for Java-friendly APIs. + * + * @define callbacks + * {{{ + * import com.twitter.util.Future + * def callbacks(result: Future[Int]): Future[Int] = + * result.onSuccess { i => + * println(i) + * }.onFailure { e => + * println(e.getMessage) + * }.ensure { + * println("always printed") + * } + * }}} */ abstract class Future[+A] extends Awaitable[A] { self => @@ -1464,6 +1477,13 @@ abstract class Future[+A] extends Awaitable[A] { self => * * @note this should be used for side-effects. * + * For example: + * $callbacks + * {{{ + * val a = Future.value(1) + * callbacks(a) // prints "1" and then "always printed" + * }}} + * * @param f the side-effect to apply when the computation completes. * @see [[respond]] if you need the result of the computation for * usage in the side-effect. @@ -1696,9 +1716,27 @@ abstract class Future[+A] extends Awaitable[A] { self => * If this, the original future, succeeds, run `f` on the result. * * The returned result is a Future that is satisfied when the original future - * and the callback, `f`, are done. + * and the callback, `f`, are done. For example: + * {{{ + * import com.twitter.util.{Await, Future} + * val f: Future[Int] = Future.value(1) + * val newf: Future[Int] = f.flatMap { x => + * Future(x + 10) + * } + * Await.result(newf) // 11 + * }}} + * * If the original future fails, this one will also fail, without executing `f` - * and preserving the failed computation of `this`. + * and preserving the failed computation of `this`. For example: + * {{{ + * import com.twitter.util.{Await, Future} + * val f: Future[Int] = Future.exception(new Exception("boom!")) + * val newf: Future[Int] = f.flatMap { x => + * println("I'm being executed") // won't print + * Future(x + 10) + * } + * Await.result(newf) // java.lang.Exception: boom! + * }}} * * @see [[map]] */ @@ -1752,9 +1790,27 @@ abstract class Future[+A] extends Awaitable[A] { self => * If this, the original future, succeeds, run `f` on the result. * * The returned result is a Future that is satisfied when the original future - * and the callback, `f`, are done. + * and the callback, `f`, are done. For example: + * {{{ + * import com.twitter.util.{Await, Future} + * val f: Future[Int] = Future.value(1) + * val newf: Future[Int] = f.map { x => + * x + 10 + * } + * Await.result(newf) // 11 + * }}} + * * If the original future fails, this one will also fail, without executing `f` - * and preserving the failed computation of `this`. + * and preserving the failed computation of `this`. For example: + * {{{ + * import com.twitter.util.{Await, Future} + * val f: Future[Int] = Future.exception(new Exception("boom!")) + * val newf: Future[Int] = f.map { x => + * println("I'm being executed") // won't print + * x + 10 + * } + * Await.result(newf) // java.lang.Exception: boom! + * }}} * * @see [[flatMap]] for computations that return `Future`s. * @see [[onSuccess]] for side-effecting chained computations. @@ -1777,6 +1833,13 @@ abstract class Future[+A] extends Awaitable[A] { self => * * @note this should be used for side-effects. * + * For example: + * $callbacks + * {{{ + * val a = Future.value(1) + * callbacks(a) // prints "1" and then "always printed" + * }}} + * * @return chained Future * @see [[flatMap]] and [[map]] to produce a new `Future` from the result of * the computation. @@ -1799,6 +1862,13 @@ abstract class Future[+A] extends Awaitable[A] { self => * `future.onFailure { case NonFatal(e) => ... }` when the Throwable * is "fatal". * + * For example: + * $callbacks + * {{{ + * val b = Future.exception(new Exception("boom!")) + * callbacks(b) // prints "boom!" and then "always printed" + * }}} + * * @return chained Future * @see [[handle]] and [[rescue]] to produce a new `Future` from the result of * the computation. @@ -2095,11 +2165,31 @@ abstract class Future[+A] extends Awaitable[A] { self => /** * Returns the result of the computation as a `Future[Try[A]]`. + * For example: + * {{{ + * import com.twitter.util.{Await, Future, Try} + * val fr: Future[Int] = Future(1) + * val ft: Future[Int] = Future.exception(new Exception("boom!")) + * val r: Future[Try[Int]] = fr.liftToTry + * val t: Future[Try[Int]] = ft.liftToTry + * Await.result(r) // Return(1) + * Await.result(t) // Throw(java.lang.Exception: boom!) + * }}} */ def liftToTry: Future[Try[A]] = transform(Future.toValue) /** * Lowers a `Future[Try[T]]` into a `Future[T]`. + * For example: + * {{{ + * import com.twitter.util.{Await, Future, Return, Throw, Try} + * val fr: Future[Try[Int]] = Future(Return(1)) + * val ft: Future[Try[Int]] = Future(Throw(new Exception("boom!"))) + * val r: Future[Int] = fr.lowerFromTry + * val t: Future[Int] = ft.lowerFromTry + * Await.result(r) // 1 + * Await.result(t) // java.lang.Exception: boom! + * }}} */ def lowerFromTry[B](implicit ev: A <:< Try[B]): Future[B] = flatMap(Future.lowerFromTry) From f069921af3a986bb74afca01c0e370dfe2dca4cc Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 27 Sep 2018 20:38:45 +0000 Subject: [PATCH 022/796] util-core: Scaladoc for Reader # Problem / Solution Adding a little more docs for readers. JIRA Issues: CSL-2701 Differential Revision: https://phabricator.twitter.biz/D220113 --- .../main/scala/com/twitter/io/Reader.scala | 82 +++++++++++++++++-- 1 file changed, 73 insertions(+), 9 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 0cb3c420ea..6897021c53 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -7,26 +7,90 @@ import scala.annotation.tailrec import scala.collection.mutable.ListBuffer /** - * A Reader represents a stream of `A`s. + * A reader exposes a pull-based API to model a potentially infinite stream of arbitrary elements. * - * Readers permit at most one outstanding read. + * Given the pull-based API, the consumer is responsible for driving the computation. A very + * typical code pattern to consume a reader is to use a read-loop: * - * @tparam A the type of objects produced by this reader + * {{{ + * def consume[A](r: Reader[A]))(process: A => Future[Unit]): Future[Unit] = + * r.read().flatMap { + * case Some(a) => process(a).before(consume(r)(process)) + * case None => Future.Done // reached the end of the stream; no need to discard + * } + * }}} + * + * One way to reason about the read-loop idiom is to view it as a subscription to a publisher + * (reader) in the Pub/Sub terminology. Perhaps the major difference between readers and traditional + * publishers, is readers only allow one subscriber (read-loop). It's generally safer to assume the + * reader is fully consumed (stream is exhausted) once its read-loop is run. + * + * == Error Handling == + * + * Given the read-loop above, its returned [[Future]] could be used to observe both successful and + * unsuccessful outcomes. + * + * {{{ + * consume(stream)(_ => Future.Done).respond { + * case Return(()) => println("Consumed an entire stream successfully.") + * case Throw(e) => println(s"Encountered an error while consuming a stream $e") + * } + * }}} + * + * @note Once failed, a stream can not be restarted such that all future reads will resolve into a + * failure. There is no need to discard an already failed stream. + * + * == Resource Safety == + * + * One of the important implications of readers, and streams in general, is that they are prone to + * resource leaks unless fully consumed or discarded (failed). Specifically, readers backed by + * network connections MUST be discarded unless already consumed (EOF observed) to prevent + * connection leaks. + * + * @note The read-loop above, for example, exhausts the stream (observes EOF) hence does not have to + * discard it (stream). + * + * == Back Pressure == + * + * The pattern above leverages [[Future]] recursion to exert back-pressure via allowing only one + * outstanding read. It's usually a good idea to structure consumers this way (i.e., the new read + * isn't issued until the current read finishes). This would always ensure a finer grained + * back-pressure in network systems allowing the consumers to artificially slow down the producers + * and not rely solely on transport and IO buffering. + * + * @note Whether or not multiple outstanding reads are allowed on a `Reader` type is an undefined + * behaviour but could be changed in a refinement. + * + * == Cancellations == + * + * If a consumer is no longer interested in the stream, it can discard it. Note a discarded reader + * (or stream) can not be restarted. + * + * {{{ + * def consumeN[A](r: Reader[A], n: Int)(process: A => Future[Unit]): Future[Unit] = + * if (n == 0) Future(r.discard()) + * else r.read().flatMap { + * case Some(a) => process(a).before(consumeN(r, n - 1)(process)) + * case None => Future.Done // reached the end of the stream; no need to discard + * } + * }}} */ trait Reader[+A] { /** - * Asynchronously read at most `n` bytes from the byte stream. The - * returned future represents the results of the read request. If - * the read fails, the Reader is considered failed -- future reads - * will also fail. + * Asynchronously read the next element of this stream. Returned [[Future]] will resolve into + * `Some(e)` when the element is available or into `None` when stream is exhausted. * - * A result of None indicates EOF. + * Stream failures are terminal such that all subsequent reads will resolve in failed [[Future]]s. */ def read(n: Int): Future[Option[A]] /** - * Discard this reader: its output is no longer required. + * Discard this stream as its output is no longer required. This could be used to signal the + * producer of this stream similarly how [[Future.raise]] used to propagate interrupts across + * future chains. + * + * @note Although unnecessary, it's always safe to discard a fully-consumed stream. */ def discard(): Unit } From 50688fe7afcbcc702de24d1bf3fdfed549289132 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 28 Sep 2018 00:29:12 +0000 Subject: [PATCH 023/796] TravisCI: Turn down the Slack notifications Problem The Slack notification on TravisCI builds is fairly noisy. Solution Change to only notify on change of either status. TBR=true Differential Revision: https://phabricator.twitter.biz/D221661 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0b8830f323..2d1a632cb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,8 @@ jdk: notifications: slack: secure: BFX2aB0sq5cMxmxlWIwgHwJOjI5W6WBn3YSPqHYYz6VAK6fzlX2VUKsfyQJTUnqMYox/0qGzPUB4TFhVMH3c78ocNZNj/9tPOPLkMfHjAqobnq/89+hp4Q6fmWCMKVAGTzVNq83OCykYkzITcYWxo0Pyc2eguW6tHaHIgqnbCRY= + on_failure: change + on_success: change before_script: - unset SBT_OPTS # default $SBT_OPTS is irrelevant to sbt launcher From 89f9776aa441624868e3bf61aa4365dd6ac3e83b Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 29 Sep 2018 20:06:29 +0000 Subject: [PATCH 024/796] TravisCI: Remove Build Stages configuration Problem Time is a flat circle. Everything we've ever done or will do, we're going to do over and over again. Build stages rely on the caching of artifacts to S3 in order fro jobs to share state. While this seems like a great idea in theory. The practice has proven to be somewhat untenable. Solution Walk away from build stages. This really only affects the Finagle build as it is the only build which remains a matrix build and thus the only build which potentially runs the dodo step multiple times. Given the complexity around trying to get build stages to work and the cache to be properly created and shared for different scala versions, this is an acceptable amount of overhead. JIRA Issues: CSL-7054 TBR=true Differential Revision: https://phabricator.twitter.biz/D222286 --- .travis.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2d1a632cb5..ec949997f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,20 +3,25 @@ sudo: required language: scala +git: + depth: 3 + quiet: true + env: - JAVA_OPTS="-DSKIP_FLAKY=true -Dsbt.log.noformat=true" # These directories are cached to S3 at the end of the build cache: directories: - - $HOME/.ivy2 - - $HOME/.sbt/boot/scala-$TRAVIS_SCALA_VERSION + - $HOME/.ivy2/cache + - $HOME/.ivy2/local/com.twitter + - $HOME/.sbt - $HOME/.dodo before_cache: - # Tricks to avoid unnecessary cache updates - - find $HOME/.ivy2 -name "ivydata-*.properties" -delete - - find $HOME/.sbt -name "*.lock" -delete + # Cleanup the cached directories to avoid unnecessary cache updates + - find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete + - find $HOME/.sbt -name "*.lock" -delete scala: - 2.11.12 From 0d183b77e43cd2160ec1bfc2c1f348341989a1cc Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 2 Oct 2018 18:56:57 +0000 Subject: [PATCH 025/796] util-core: Scaladoc for Writer # Problem / Solution Adding some scaladocs for `Writer`. JIRA Issues: CSL-2701 Differential Revision: https://phabricator.twitter.biz/D221602 --- .../main/scala/com/twitter/io/Reader.scala | 8 +-- .../main/scala/com/twitter/io/Writer.scala | 65 ++++++++++++++++--- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 6897021c53..2f26e2e791 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -31,9 +31,9 @@ import scala.collection.mutable.ListBuffer * unsuccessful outcomes. * * {{{ - * consume(stream)(_ => Future.Done).respond { + * consume(stream)(processor).respond { * case Return(()) => println("Consumed an entire stream successfully.") - * case Throw(e) => println(s"Encountered an error while consuming a stream $e") + * case Throw(e) => println(s"Encountered an error while consuming a stream: $e") * } * }}} * @@ -53,8 +53,8 @@ import scala.collection.mutable.ListBuffer * == Back Pressure == * * The pattern above leverages [[Future]] recursion to exert back-pressure via allowing only one - * outstanding read. It's usually a good idea to structure consumers this way (i.e., the new read - * isn't issued until the current read finishes). This would always ensure a finer grained + * outstanding read. It's usually a good idea to structure consumers this way (i.e., the next read + * isn't issued until the previous read finishes). This would always ensure a finer grained * back-pressure in network systems allowing the consumers to artificially slow down the producers * and not rely solely on transport and IO buffering. * diff --git a/util-core/src/main/scala/com/twitter/io/Writer.scala b/util-core/src/main/scala/com/twitter/io/Writer.scala index c93d2b32cf..f7c00e495e 100644 --- a/util-core/src/main/scala/com/twitter/io/Writer.scala +++ b/util-core/src/main/scala/com/twitter/io/Writer.scala @@ -4,21 +4,70 @@ import com.twitter.util.{Closable, Future, Time} import java.io.OutputStream /** - * A Writer represents a sink for a stream of `A`s, providing a convenient interface - * for the producer of such streams. + * A writer represents a sink for a stream of arbitrary elements. While [[Reader]] provides an API + * to consume streams, [[Writer]] provides an API to produce streams. + * + * Similar to [[Reader]]s, a very typical way to work with [[Writer]]s is to model a stream producer + * as a write-loop: + * + * {{{ + * def produce[A](w: Writer[A])(generate: () => Option[A]): Future[Unit] = + * generate match { + * case Some(a) => w.write(a).before(produce(w)(generate)) + * case None => w.close() // signal EOF and quit producing + * } + * }}} + * + * == Closures and Failures == + * + * Writers are [[Closable]] and the producer MUST `close` the stream when it finishes or `fail` the + * stream when it encounters a failure and can no longer continue. Streams backed by network + * connections are particularly prone to resource leaks when they aren't cleaned up properly. + * + * Observing stream failures in the producer could be done via a [[Future]] returned form + * a write-loop: + * + * {{{ + * produce(writer)(generator).respond { + * case Return(()) => println("Produced a stream successfully.") + * case Throw(e) => println(s"Could not produce a stream because of a failure: $e") + * } + * }}} + * + * @note Encountering a stream failure would terminate the write-loop given the [[Future]] + * recursion semantics. + * + * @note Once failed or closed, a stream can not be restarted such that all future writes will + * resolve into a failure. + * + * @note Closing an already failed stream does not have an effect. + * + * == Back Pressure == + * + * By analogy with read-loops (see [[Reader]] API), write-loops leverage [[Future]] recursion to + * exert back-pressure: the next write isn't issued until the previous write finishes. This will + * always ensure a finer grained back-pressure in network systems allowing the producers to + * adjust the flow rate based on consumer's speed and not on IO buffering. + * + * @note Whether or not multiple pending writes are allowed on a `Writer` type is an undefined + * behaviour but could be changed in a refinement. */ trait Writer[-A] extends Closable { /** - * Write a chunk. The returned future is completed when the chunk has been - * fully read by the sink. + * Write an `element` into this stream. Although undefined by this contract, a trustworthy + * implementation (such as [[Pipe]]) would do its best to resolve the returned [[Future]] only + * when a consumer observes a written `element`. * - * Only one outstanding `write` is permitted, thus backpressure is asserted. + * The write can also resolve into a failure (failed [[Future]]). */ - def write(buf: A): Future[Unit] + def write(element: A): Future[Unit] /** - * Indicate that the producer of the bytestream has failed. No further writes are allowed. + * Fail this stream with a given `cause`. No further writes are allowed, but if happen, will + * resolve into a [[Future]] failed with `cause`. + * + * @note Failing an already closed stream does not have an effect. */ def fail(cause: Throwable): Unit } @@ -36,7 +85,7 @@ object Writer { val FailingWriter: Writer[Buf] = fail[Buf] def fail[A]: Writer[A] = new Writer[A] { - def write(buf: A): Future[Unit] = Future.exception(new IllegalStateException("NullWriter")) + def write(element: A): Future[Unit] = Future.exception(new IllegalStateException("NullWriter")) def fail(cause: Throwable): Unit = () def close(deadline: Time): Future[Unit] = Future.Done } From 3245a8e1a98bd5eb308f366678528879d7140f5e Mon Sep 17 00:00:00 2001 From: Flavio Brasil Date: Tue, 2 Oct 2018 19:03:08 +0000 Subject: [PATCH 026/796] util-core: Avoid CAS operations in Futures **Problem** Some of the compare and swap (CAS) operations done by Future could be avoided. CAS operations are the main source of overhead in our Finagle stack. **Solution** 1. Introduce `Future::transformTry`, which applies the transformation directly using a `Try` instead of transforming into a `Future` since the current implementation requires additional CAS operations 2. Change methods to use `transformTry` instead of `transform` where possible 3. Change `Promise.interrupts` to create the promise and set the `Transforming` state without performing a CAS 4. Set the state directly in `Promise.compress` since a CAS is unnecessary. The field is volatile so the write is safe Differential Revision: https://phabricator.twitter.biz/D209541 --- .../scala/com/twitter/util/ConstFuture.scala | 29 ++++++- .../main/scala/com/twitter/util/Future.scala | 76 ++++++++++------- .../scala/com/twitter/util/NoFuture.scala | 1 + .../main/scala/com/twitter/util/Promise.scala | 82 ++++++++++++++----- .../src/main/scala/com/twitter/util/Try.scala | 10 +++ .../TwitterFuturesCapturePoints.xml | 4 +- 6 files changed, 145 insertions(+), 57 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala index a43fa82b16..bb8bac1dc7 100644 --- a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala @@ -22,7 +22,8 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { Scheduler.submit(new Runnable { def run(): Unit = { val current = Local.save() - Local.restore(saved) + if (current ne saved) + Local.restore(saved) try k(result) catch Monitor.catcher finally Local.restore(current) @@ -33,6 +34,29 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { def raise(interrupt: Throwable): Unit = () + protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { + val p = new Promise[B] + // see the note on `respond` for an explanation of why `Scheduler` is used. + val saved = Local.save() + Scheduler.submit(new Runnable { + def run(): Unit = { + val current = Local.save() + if (current ne saved) + Local.restore(saved) + val computed = try f(result) + catch { + case e: NonLocalReturnControl[_] => Throw(new FutureNonLocalReturnControl(e)) + case scala.util.control.NonFatal(e) => Throw(e) + case t: Throwable => + Monitor.handle(t) + throw t + } finally Local.restore(current) + p.update(computed) + } + }) + p + } + def transform[B](f: Try[A] => Future[B]): Future[B] = { val p = new Promise[B] // see the note on `respond` for an explanation of why `Scheduler` is used. @@ -40,7 +64,8 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { Scheduler.submit(new Runnable { def run(): Unit = { val current = Local.save() - Local.restore(saved) + if (current ne saved) + Local.restore(saved) val computed = try f(result) catch { case e: NonLocalReturnControl[_] => Future.exception(new FutureNonLocalReturnControl(e)) diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index a359690819..56500b6627 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -49,39 +49,36 @@ object Future { private val SomeReturnUnit = Some(Return.Unit) private val NotApplied: Future[Nothing] = new NoFuture private val AlwaysNotApplied: Any => Future[Nothing] = scala.Function.const(NotApplied) - private val toUnit: Try[Any] => Future[Unit] = { - case Return(_) => Future.Unit - case t => Future.const(t.asInstanceOf[Try[Unit]]) + private val tryToUnit: Try[Any] => Try[Unit] = { + case t: Return[_] => Try.Unit + case t => t.asInstanceOf[Try[Unit]] } - private val toVoid: Try[Any] => Future[Void] = { - case Return(_) => Future.Void - case t => Future.const(t.asInstanceOf[Try[Void]]) + private val tryToVoid: Try[Any] => Try[Void] = { + case t: Return[_] => Try.Void + case t => t.asInstanceOf[Try[Void]] } private val AlwaysMasked: PartialFunction[Throwable, Boolean] = { case _ => true } private val toTuple2Instance: (Any, Any) => (Any, Any) = Tuple2.apply private def toTuple2[A, B]: (A, B) => (A, B) = toTuple2Instance.asInstanceOf[(A, B) => (A, B)] - private val toValueInstance: Any => Future[Any] = Future.value - private def toValue[T]: T => Future[T] = toValueInstance.asInstanceOf[T => Future[T]] + private val liftToTryInstance: Any => Try[Any] = Return(_) + private def liftToTry[T]: T => Try[T] = liftToTryInstance.asInstanceOf[T => Try[T]] - private val lowerFromTryInstance: Any => Future[Any] = { - case t: Try[_] => Future.const(t) - } - - private def lowerFromTry[A, T]: A => Future[T] = - lowerFromTryInstance.asInstanceOf[A => Future[T]] + private val flattenTryInstance: Try[Try[Any]] => Try[Any] = _.flatten + private def flattenTry[A, T](implicit ev: A => Try[T]): Try[A] => Try[T] = + flattenTryInstance.asInstanceOf[Try[A] => Try[T]] - private val toTxInstance: Try[Any] => Future[Tx[Try[Any]]] = + private val toTxInstance: Try[Any] => Try[Tx[Try[Any]]] = res => { val tx = new Tx[Try[Any]] { def ack(): Future[Tx.Result[Try[Any]]] = Future.value(Tx.Commit(res)) def nack(): Unit = () } - Future.value(tx) + Return(tx) } - private def toTx[A]: Try[A] => Future[Tx[Try[A]]] = - toTxInstance.asInstanceOf[Try[A] => Future[Tx[Try[A]]]] + private def toTx[A]: Try[A] => Try[Tx[Try[A]]] = + toTxInstance.asInstanceOf[Try[A] => Try[Tx[Try[A]]]] private val emptySeqInstance: Future[Seq[Any]] = Future.value(Seq.empty) private def emptySeq[A]: Future[Seq[A]] = emptySeqInstance.asInstanceOf[Future[Seq[A]]] @@ -1700,7 +1697,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * When this future completes, run `f` on that completed result * whether or not this computation was successful. * - * The returned `Future` will be satisfied when this, + * The returned `Future` will be satisfied when `this`, * the original future, and `f` are done. * * @see [[respond]] for purely side-effecting callbacks. @@ -1712,6 +1709,27 @@ abstract class Future[+A] extends Awaitable[A] { self => */ def transform[B](f: Try[A] => Future[B]): Future[B] + /** + * When this future completes, run `f` on that completed result + * whether or not this computation was successful. + * + * This method is similar to `transform`, but the transformation is + * applied without introducing an intermediate future, which leads + * to better performance. + * + * The returned `Future` will be satisfied when `this`, + * the original future, is done. + * + * @see [[respond]] for purely side-effecting callbacks. + * @see [[map]] and [[flatMap]] for dealing strictly with successful + * computations. + * @see [[handle]] and [[rescue]] for dealing strictly with exceptional + * computations. + * @see [[transformedBy]] for a Java friendly API. + * @see [[transform]] for transformations that return `Future`. + */ + protected def transformTry[B](f: Try[A] => Try[B]): Future[B] + /** * If this, the original future, succeeds, run `f` on the result. * @@ -1816,14 +1834,10 @@ abstract class Future[+A] extends Awaitable[A] { self => * @see [[onSuccess]] for side-effecting chained computations. */ def map[B](f: A => B): Future[B] = - transform { - case Return(r) => Future { f(r) } - case t: Throw[_] => Future.const[B](t.cast[B]) - } + transformTry(_.map(f)) - def filter(p: A => Boolean): Future[A] = transform { x: Try[A] => - Future.const(x.filter(p)) - } + def filter(p: A => Boolean): Future[A] = + transformTry(_.filter(p)) def withFilter(p: A => Boolean): Future[A] = filter(p) @@ -2009,14 +2023,14 @@ abstract class Future[+A] extends Awaitable[A] { self => * * @note failed futures will remain as is. */ - def unit: Future[Unit] = transform(Future.toUnit) + def unit: Future[Unit] = transformTry(Future.tryToUnit) /** * Convert this `Future[A]` to a `Future[Void]` by discarding the result. * * @note failed futures will remain as is. */ - def voided: Future[Void] = transform(Future.toVoid) + def voided: Future[Void] = transformTry(Future.tryToVoid) /** * Send updates from this Future to the other. @@ -2047,7 +2061,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * The offer is activated when the future is satisfied. */ def toOffer: Offer[Try[A]] = new Offer[Try[A]] { - def prepare(): Future[Tx[Try[A]]] = transform(Future.toTx) + def prepare(): Future[Tx[Try[A]]] = transformTry(Future.toTx) } /** @@ -2176,7 +2190,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * Await.result(t) // Throw(java.lang.Exception: boom!) * }}} */ - def liftToTry: Future[Try[A]] = transform(Future.toValue) + def liftToTry: Future[Try[A]] = transformTry(Future.liftToTry) /** * Lowers a `Future[Try[T]]` into a `Future[T]`. @@ -2192,7 +2206,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * }}} */ def lowerFromTry[B](implicit ev: A <:< Try[B]): Future[B] = - flatMap(Future.lowerFromTry) + transformTry(Future.flattenTry) /** * Makes a derivative `Future` which will be satisfied with the result diff --git a/util-core/src/main/scala/com/twitter/util/NoFuture.scala b/util-core/src/main/scala/com/twitter/util/NoFuture.scala index c498552ca5..5cf3dfba81 100644 --- a/util-core/src/main/scala/com/twitter/util/NoFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/NoFuture.scala @@ -9,6 +9,7 @@ package com.twitter.util class NoFuture extends Future[Nothing] { def respond(k: Try[Nothing] => Unit): Future[Nothing] = this def transform[B](f: Try[Nothing] => Future[B]): Future[B] = this + protected def transformTry[B](f: Try[Nothing] => Try[B]): Future[B] = this def raise(interrupt: Throwable): Unit = () diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index a9a8afeeaa..457178c278 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -195,27 +195,48 @@ object Promise { private class Monitored[A](saved: Local.Context, k: Try[A] => Unit) extends K[A] { def apply(result: Try[A]): Unit = { val current = Local.save() - Local.restore(saved) + if (current ne saved) + Local.restore(saved) try k(result) catch Monitor.catcher finally Local.restore(current) } } + private abstract class Transformer[A, B]( + saved: Local.Context + ) extends K[A] { + + protected[this] def k(r: Try[A]): Unit + + def apply(result: Try[A]): Unit = { + val current = Local.save() + if (current ne saved) + Local.restore(saved) + try k(result) + catch { + case t: Throwable => + Monitor.handle(t) + throw t + } finally Local.restore(current) + } + } + /** * A transforming continuation. * * @param saved The saved local context of the invocation site - * @param promise The Promise for the transformed value * @param f The closure to invoke to produce the Future of the transformed value. + * @param promise The Promise for the transformed value */ - private class Transformer[A, B]( + private final class FutureTransformer[A, B]( saved: Local.Context, - promise: Promise[B], - f: Try[A] => Future[B] - ) extends K[A] { + f: Try[A] => Future[B], + promise: Promise[B] + ) extends Transformer[A, B](saved) { - private[this] def k(r: Try[A]): Unit = { + protected[this] def k(r: Try[A]): Unit = + // The promise can be fulfilled only by the transformer, so it's safe to use `become` here promise.become( try f(r) catch { @@ -223,17 +244,23 @@ object Promise { case NonFatal(e) => Future.exception(e) } ) - } + } - def apply(result: Try[A]): Unit = { - val current = Local.save() - Local.restore(saved) - try k(result) - catch { - case t: Throwable => - Monitor.handle(t) - throw t - } finally Local.restore(current) + private final class TryTransformer[A, B]( + saved: Local.Context, + f: Try[A] => Try[B], + promise: Promise[B] + ) extends Transformer[A, B](saved) { + + protected[this] def k(r: Try[A]): Unit = { + // The promise can be fulfilled only by the transformer, so it's safe to use `update` here + promise.update( + try f(r) + catch { + case e: NonLocalReturnControl[_] => Throw(new FutureNonLocalReturnControl(e)) + case NonFatal(e) => Throw(e) + } + ) } } @@ -304,7 +331,15 @@ object Promise { def transform[B](f: Try[A] => Future[B]): Future[B] = { val promise = interrupts[B](this) - continue(new Transformer(Local.save(), promise, f)) + continue(new FutureTransformer(Local.save(), f, promise)) + + promise + } + + protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { + val promise = interrupts[B](this) + + continue(new TryTransformer(Local.save(), f, promise)) promise } @@ -327,9 +362,7 @@ object Promise { * @see [[interrupts(Future, Future)]] * @see [[interrupts(Future*)]] */ - def interrupts[A](f: Future[_]): Promise[A] = new Promise[A] { - forwardInterruptsTo(f) - } + def interrupts[A](f: Future[_]): Promise[A] = new Promise[A](f) /** * Create a promise that interrupts `a` and `b` futures. In particular: @@ -428,6 +461,11 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ @volatile private[this] var state: Any = WaitQueue.empty[A] private def theState(): Any = state + private[util] def this(forwardInterrupts: Future[_]) { + this() + this.state = new Transforming[A](WaitQueue.empty, forwardInterrupts) + } + def this(handleInterrupt: PartialFunction[Throwable, Unit]) { this() this.state = new Interruptible[A](WaitQueue.empty, handleInterrupt) @@ -785,7 +823,7 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ val target = p.compress() // due to the assumptions stated above regarding when this can be called, // there should never be a `cas` fail. - cas(p, target) + state = target target case _ => this diff --git a/util-core/src/main/scala/com/twitter/util/Try.scala b/util-core/src/main/scala/com/twitter/util/Try.scala index b9c898d350..5c39fb471e 100644 --- a/util-core/src/main/scala/com/twitter/util/Try.scala +++ b/util-core/src/main/scala/com/twitter/util/Try.scala @@ -12,6 +12,16 @@ import scala.util.{Success, Failure} object Try { case class PredicateDoesNotObtain() extends Exception() + /** + * A constant `Try` that returns `Unit`. + */ + val Unit: Try[Unit] = Try(()) + + /** + * A constant `Try` that returns `Void`. + */ + val Void: Try[Void] = Try(null: Void) + def apply[R](r: => R): Try[R] = { try { Return(r) } catch { case NonFatal(e) => Throw(e) diff --git a/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml b/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml index 155a33a89a..3100e4daa1 100644 --- a/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml +++ b/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml @@ -30,7 +30,7 @@ class-name="com.twitter.util.Promise" method-name="transform" capture-key-expression="f" - insert-class-name="com.twitter.util.Promise$Transformer" + insert-class-name="com.twitter.util.Promise$FutureTransformer" insert-method-name="liftedTree1$1" insert-key-expression="this.f" /> @@ -62,7 +62,7 @@ class-name="com.twitter.util.Future" method-name="map" capture-key-expression="f" - insert-class-name="com.twitter.util.Future$$anonfun$map$1$$anonfun$apply$3" + insert-class-name="com.twitter.util.Future$$anonfun$map$1" insert-method-name="apply" insert-key-expression="f" /> From 435c079b1660ccde24da7c64c90db1ac6e625ae6 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Tue, 2 Oct 2018 20:59:49 +0000 Subject: [PATCH 027/796] util-benchmark: benchmark for Broker Problem / Solution We want to understand the performance of `Broker`. Differential Revision: https://phabricator.twitter.biz/D218025 --- .../twitter/concurrent/BrokerBenchmark.scala | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 util-benchmark/src/main/scala/com/twitter/concurrent/BrokerBenchmark.scala diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/BrokerBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/BrokerBenchmark.scala new file mode 100644 index 0000000000..8dd4ca4782 --- /dev/null +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/BrokerBenchmark.scala @@ -0,0 +1,51 @@ +package com.twitter.concurrent + +import com.twitter.util.StdBenchAnnotations +import org.openjdk.jmh.annotations.{Benchmark, Scope, State} +import org.openjdk.jmh.infra.Blackhole + +@State(Scope.Benchmark) +class BrokerBenchmark extends StdBenchAnnotations { + // It's safe to share brokers across iterations because each benchmark pairs + // a send with a recv. + private[this] val q = new Broker[Int] + private[this] val p = new Broker[Int] + + @Benchmark + def sendAndRecv(blackhole: Blackhole): Unit = { + val f = q.send(1).sync() + blackhole.consume(f) + val g = q.recv.sync() + blackhole.consume(g) + } + + @Benchmark + def recvAndSend(blackhole: Blackhole): Unit = { + val g = q.recv.sync() + blackhole.consume(g) + val f = q.send(2).sync() + blackhole.consume(f) + } + + // Especially notable is that, when selecting over multiple brokers (as we do + // below), both send and recv operations are guaranteed to affect the same + // broker. Offer.select commits to the communication that can proceed, which + // means that if we attempt to send to multiple brokers, only one broker will + // be modified, the state of the other brokers is unaffected. + + @Benchmark + def selectRecvAndSend(blackhole: Blackhole): Unit = { + val f = Offer.select(q.recv, p.recv) + blackhole.consume(f) + val g = Offer.select(q.send(3), p.send(3)) + blackhole.consume(g) + } + + @Benchmark + def selectSendAndRecv(blackhole: Blackhole): Unit = { + val g = Offer.select(q.send(3), p.send(3)) + blackhole.consume(g) + val f = Offer.select(q.recv, p.recv) + blackhole.consume(f) + } +} From 0daac8d785b02bc819b1464c695f17a28ffb3384 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Sat, 6 Oct 2018 05:20:14 +0000 Subject: [PATCH 028/796] util-core: minute granularity for inTimeUnit Problem / Solution Minute granularity for inTimeUnit so that Durations that can be expressed in terms of minutes, e.g.: 1.minute.inTimeUnit == (60, SECONDS) // Before 1.minute.inTimeUnit == (1, MINUTES) // After Differential Revision: https://phabricator.twitter.biz/D225115 --- CHANGELOG.rst | 6 ++++++ .../main/scala/com/twitter/util/Time.scala | 19 ++++++++----------- .../scala/com/twitter/util/DurationTest.scala | 1 + 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4179ca1f98..2e2eb743b7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-core: `c.t.u.Duration.inTimeUnit` can now return + `j.u.c.TimeUnit.MINUTES`. ``PHAB_ID=D225115`` + 18.9.1 ------- diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index 20a90d7e0f..f812e42713 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -156,20 +156,17 @@ trait TimeLike[This <: TimeLike[This]] extends Ordered[This] { self: This => def inMillis: Long = inMilliseconds // (Backwards compatibility) /** - * Returns a value/`TimeUnit` pair; attempting to return coarser - * grained values if possible (specifically: `TimeUnit.SECONDS` or - * `TimeUnit.MILLISECONDS`) before resorting to the default + * Returns a value/`TimeUnit` pair; attempting to return coarser grained + * values if possible (specifically: `TimeUnit.MINUTES`, `TimeUnit.SECONDS` + * or `TimeUnit.MILLISECONDS`) before resorting to the default * `TimeUnit.NANOSECONDS`. */ - def inTimeUnit: (Long, TimeUnit) = { + def inTimeUnit: (Long, TimeUnit) = inNanoseconds match { // allow for APIs that may treat TimeUnit differently if measured in very tiny units. - if (inNanoseconds % Duration.NanosPerSecond == 0) { - (inSeconds, TimeUnit.SECONDS) - } else if (inNanoseconds % Duration.NanosPerMillisecond == 0) { - (inMilliseconds, TimeUnit.MILLISECONDS) - } else { - (inNanoseconds, TimeUnit.NANOSECONDS) - } + case ns if ns % Duration.NanosPerMinute == 0 => (inMinutes, TimeUnit.MINUTES) + case ns if ns % Duration.NanosPerSecond == 0 => (inSeconds, TimeUnit.SECONDS) + case ns if ns % Duration.NanosPerMillisecond == 0 => (inMilliseconds, TimeUnit.MILLISECONDS) + case ns => (ns, TimeUnit.NANOSECONDS) } /** diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 22e551c230..477bbfcb88 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -187,6 +187,7 @@ class DurationTest extends { val ops = Duration } with TimeLikeSpec[Duration] { assert(23.microseconds.inTimeUnit == ((23000, TimeUnit.NANOSECONDS))) assert(23.milliseconds.inTimeUnit == ((23, TimeUnit.MILLISECONDS))) assert(23.seconds.inTimeUnit == ((23, TimeUnit.SECONDS))) + assert(60.seconds.inTimeUnit == ((1, TimeUnit.MINUTES))) } "inUnit" in { From 6f587b35acb19b9e526ff6f0111f9512d091175a Mon Sep 17 00:00:00 2001 From: Simon Adameit Date: Wed, 10 Oct 2018 18:51:06 +0000 Subject: [PATCH 029/796] util-jvm: Use newer jdk management api when the old one is not available Problem Newer jdks do not expose `sun.management.ManagementFactory`. The code to branch between `sun.management.ManagementFactoryHelper` for jdk7 and `sun.management.ManagementFactory` for earlier jdks only caught `NoSuchFieldException`. This lead to the `NilJvm` being used on newer jdks that do not expose `sun.management.ManagementFactory`. Solution Catch `Throwable` as is done elsewhere in the same codebase and switch to the `sun.management.ManagementFactory` Result The Hotspot class can be instantiated for newer jdks. Signed-off-by: Bryce Anderson JIRA Issues: CSL-7091 Differential Revision: https://phabricator.twitter.biz/D226675 --- util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index e4b91e66d8..d7b0259033 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -9,6 +9,7 @@ import javax.management.openmbean.CompositeDataSupport import javax.management.{ObjectName, RuntimeMBeanException} import scala.collection.JavaConverters._ import scala.language.reflectiveCalls +import scala.util.control.NonFatal class Hotspot extends Jvm { private[this] val epoch = @@ -32,7 +33,7 @@ class Hotspot extends Jvm { // jdk5/6 have jvm field in ManagementFactory class Class.forName("sun.management.ManagementFactory").getDeclaredField("jvm") } catch { - case _: NoSuchFieldException => + case NonFatal(_) => // jdk7 moves jvm field to ManagementFactoryHelper class Class.forName("sun.management.ManagementFactoryHelper").getDeclaredField("jvm") } From 089d42c402bcff50779d238d6e39302969b53efe Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 10 Oct 2018 19:41:19 +0000 Subject: [PATCH 030/796] util-core: Avoid using conversions within util-core Problem In the interest of having finer grained pants build targets the source code within util-core should not have cycles. Solution One step is to avoid using `c.t.conversions` from the other util-core packages. JIRA Issues: CSL-7094 Differential Revision: https://phabricator.twitter.biz/D226751 --- .../main/scala/com/twitter/concurrent/AsyncMeter.scala | 9 ++++----- .../main/scala/com/twitter/io/exp/ActivitySource.scala | 5 +++-- .../scala/com/twitter/io/exp/MinimumThroughput.scala | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala index b01ee91873..807352c736 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala @@ -1,6 +1,5 @@ package com.twitter.concurrent -import com.twitter.conversions.time._ import com.twitter.util._ import java.util.concurrent.{ ArrayBlockingQueue, @@ -25,7 +24,7 @@ private[concurrent] object Period { } object AsyncMeter { - private[concurrent] val MinimumInterval = 1.millisecond + private[concurrent] val MinimumInterval = Duration.fromMilliseconds(1) /** * Creates an [[AsyncMeter]] that allows smoothed out `permits` per @@ -34,7 +33,7 @@ object AsyncMeter { * This is equivalent to `AsyncMeter.newMeter(permits, 1.second, maxWaiters)`. */ def perSecond(permits: Int, maxWaiters: Int)(implicit timer: Timer): AsyncMeter = - newMeter(permits, 1.second, maxWaiters) + newMeter(permits, Duration.fromSeconds(1), maxWaiters) /** * Creates an [[AsyncMeter]] that allows smoothed out `permits` per second, @@ -58,7 +57,7 @@ object AsyncMeter { * second, but the burst should be smoothed out after that. */ def perSecondLimited(permits: Int, maxWaiters: Int)(implicit timer: Timer): AsyncMeter = - newMeter(1, 1.second / permits, maxWaiters) + newMeter(1, Duration.fromSeconds(1) / permits, maxWaiters) /** * Creates an [[AsyncMeter]] that has a maximum burst size of `burstSize` over @@ -240,7 +239,7 @@ class AsyncMeter private ( // guarantees that satisfying the thread is not racy--we also use // Promise#setValue or Promise#setException to ensure that if there's a // race, it will fail loudly. - val p = Promise[Unit] + val p = Promise[Unit]() val tup = (p, permits) if (q.offer(tup)) { diff --git a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala b/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala index db9e4a6ffa..7287fdabd5 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala +++ b/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala @@ -1,6 +1,5 @@ package com.twitter.io.exp -import com.twitter.conversions.time._ import com.twitter.io.{InputStreamReader, Buf, Reader} import com.twitter.util._ import java.io.{FileInputStream, File} @@ -40,7 +39,9 @@ object ActivitySource { * An ActivitySource for observing file contents. Once observed, * each file will be polled once per period. */ - def forFiles(period: Duration = 1.minute)(implicit timer: Timer): ActivitySource[Buf] = + def forFiles( + period: Duration = Duration.fromSeconds(60) + )(implicit timer: Timer): ActivitySource[Buf] = new CachingActivitySource(new FilePollingActivitySource(period)(timer)) /** diff --git a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala b/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala index 28189d27d9..273c7e6d37 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala +++ b/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala @@ -1,6 +1,5 @@ package com.twitter.io.exp -import com.twitter.conversions.time._ import com.twitter.io.{Writer, Buf, Reader} import com.twitter.util._ import scala.util.control.NoStackTrace @@ -13,7 +12,7 @@ object MinimumThroughput { private val BpsNoElapsed = -1d - private val MinDeadline = 1.second + private val MinDeadline = Duration.fromSeconds(1) /** internal marker used to raise on read/write timeouts. */ private case object MinThroughputTimeoutException extends Exception with NoStackTrace @@ -54,7 +53,7 @@ object MinimumThroughput { } def newBelowThroughputException: BelowThroughputException = - new BelowThroughputException(elapsed, bps, minBps) + BelowThroughputException(elapsed, bps, minBps) } From 5ef6a0dca89dad4488ce33cd1ea5a2f8eab1bb7a Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 11 Oct 2018 00:42:22 +0000 Subject: [PATCH 031/796] source: Replace Reader.Writable with Pipe # Problem / Solution `Reader.Writable` is deprecated. Let's remove it and replace with `Pipe`. JIRA Issues: CSL-6977 Differential Revision: https://phabricator.twitter.biz/D226603 --- CHANGELOG.rst | 6 +++ .../src/main/java/com/twitter/io/Readers.java | 7 ---- .../main/scala/com/twitter/io/Reader.scala | 38 ------------------- .../com/twitter/io/ReaderCompilationTest.java | 4 +- 4 files changed, 8 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2e2eb743b7..f3b46f31c6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: `c.t.io.Reader.Writable` and `c.t.Reader.writable()` are removed. Use `c.t.io.Pipe` +instead. ``PHAB_ID=D226603`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index 7c54857478..db51d879c6 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -75,13 +75,6 @@ public static Future copyMany(AsyncStream> readers, Write return Reader$.MODULE$.copyMany(readers, w, readSize); } - /** - * See {@code com.twitter.io.Pipe}. - */ - public static Pipe writable() { - return new Pipe<>(); - } - /** * See {@code com.twitter.io.Reader.fromFile()}. */ diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 2f26e2e791..b791d9a4de 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -189,44 +189,6 @@ object Reader { class ReaderDiscarded extends Exception("This writer's reader has been discarded") - /** - * A [[Reader]] that is linked with a [[Writer]] and `close`-ing - * is synchronous. - * - * Just as with [[Reader readers]] and [[Writer writers]], - * only one outstanding `read` or `write` is permitted. - * - * For a proper `close`, it should only be done when - * no writes are outstanding: - * {{{ - * val rw = Reader.writable() - * ... - * rw.write(buf).before(rw.close()) - * }}} - * - * If a producer is interested in knowing when all writes - * have been read and the reader has seen the EOF, it can - * wait until the future returned by `close()` is satisfied: - * {{{ - * val rw = Reader.writable() - * ... - * rw.close().ensure { - * println("party on! ♪┏(・o･)┛♪ the Reader has seen the EOF") - * } - * }}} - */ - @deprecated("Use Pipe[A] instead", "2018-8-7") - type Writable[A <: Buf] = Pipe[A] - - /** - * Create a new [[Writable]] which is a [[Reader]] that is linked - * with a [[Writer]]. - * - * @see Readers.writable() for a Java API. - */ - @deprecated("Use Pipe() instead", "2018-8-7") - def writable(): Pipe[Buf] = new Pipe[Buf] - /** * Create a new [[Reader]] for a `File`. * diff --git a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java b/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java index 2a316c76e6..b8c2d90a25 100644 --- a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java +++ b/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java @@ -33,8 +33,8 @@ public void testConcat() { @Test - public void testWritable() { - Pipe w = Readers.writable(); + public void testPipe() { + Pipe w = new Pipe<>(); w.close(); } From bff967edd4c42dd8e8482a3a3709f3d22e44816a Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 11 Oct 2018 17:07:19 +0000 Subject: [PATCH 032/796] util-core: Add contramap for Writer Problem: We want to make Writer API polymorphic regarding the type it operates on. Solution: Writer becomes a contravariant functor by adding `contramap`: if a function `f` which transforms `B` into `A` is given, and the `Writer[A]` is implemented, we get the `Writer[B]` by calling `contramap`. JIRA Issues: CSL-6686 Differential Revision: https://phabricator.twitter.biz/D225686 --- CHANGELOG.rst | 6 ++++++ .../main/scala/com/twitter/io/Writer.scala | 20 ++++++++++++++++++- .../scala/com/twitter/io/WriterTest.scala | 7 +++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f3b46f31c6..7710c06f4a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -34,6 +34,12 @@ Bug Fixes * util-slf4j-api: Moved slf4j-simple dependency to be a 'test' dependency, instead of a compile dependency, which was inaccurate. ``PHAB_ID=D220718`` +New Features +~~~~~~~~~~~~ + +* util-core: Added a `contramap` function into `c.t.io.Writer`, `Writer` is now a contravariant + functor. Added the `AbstractWriter` for Java compatibility ``PHAB_ID=D225686`` + 18.9.0 ------- diff --git a/util-core/src/main/scala/com/twitter/io/Writer.scala b/util-core/src/main/scala/com/twitter/io/Writer.scala index f7c00e495e..eab9e11685 100644 --- a/util-core/src/main/scala/com/twitter/io/Writer.scala +++ b/util-core/src/main/scala/com/twitter/io/Writer.scala @@ -52,7 +52,7 @@ import java.io.OutputStream * @note Whether or not multiple pending writes are allowed on a `Writer` type is an undefined * behaviour but could be changed in a refinement. */ -trait Writer[-A] extends Closable { +trait Writer[-A] extends Closable { self => /** * Write an `element` into this stream. Although undefined by this contract, a trustworthy @@ -70,8 +70,26 @@ trait Writer[-A] extends Closable { * @note Failing an already closed stream does not have an effect. */ def fail(cause: Throwable): Unit + + /** + * Given f, a function from B into A, creates an Writer[B] whose `fail` and `close` functions + * are equivalent to Writer[A]'s. Writer[B]'s `write` function is equivalent to: + * {{{ + * def write(element: B) = Writer[A].write(f(element)) + * }}} + */ + final def contramap[B](f: B => A): Writer[B] = new Writer[B] { + def write(element: B): Future[Unit] = self.write(f(element)) + def fail(cause: Throwable): Unit = self.fail(cause) + def close(deadline: Time): Future[Unit] = self.close(deadline) + } } +/** + * Abstract `Writer` class for Java compatibility. + */ +abstract class AbstractWriter[-A] extends Writer[A] + /** * @see Writers for Java friendly APIs. */ diff --git a/util-core/src/test/scala/com/twitter/io/WriterTest.scala b/util-core/src/test/scala/com/twitter/io/WriterTest.scala index cb986aa4c5..29a2043e9a 100644 --- a/util-core/src/test/scala/com/twitter/io/WriterTest.scala +++ b/util-core/src/test/scala/com/twitter/io/WriterTest.scala @@ -21,6 +21,13 @@ class WriterTest extends FunSuite with Matchers { intercept[Exception] { await(w.write(Buf.Empty)) } } + test("fromOutputStream - contramap") { + val w = Writer.fromOutputStream(new ByteArrayOutputStream) + val nw = w.contramap[String](s => Buf.Utf8(s)) + await(nw.write("hi")) + await(nw.close()) + } + test("fromOutputStream - error") { val os = new OutputStream { def write(b: Int): Unit = () From 2f8ee9048431a7cdf10333390b0eb2e1c2df08cd Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Thu, 11 Oct 2018 17:13:42 +0000 Subject: [PATCH 033/796] util-core: Move TempFolder from c.t.util to c.t.io Problem In the interest of having finer grained pants build targets the source code within util-core should not have cycles. Solution One step is to move the misplaced `TempFolder` into the `c.t.io` package. This avoids the dependency from `c.t.util` on `c.t.io`. JIRA Issues: CSL-7094 TBR=true Differential Revision: https://phabricator.twitter.biz/D226940 --- CHANGELOG.rst | 4 +++- .../scala/com/twitter/{util => io}/TempFolder.scala | 10 +++------- .../src/test/scala/com/twitter/io/FilesTest.scala | 3 --- .../scala/com/twitter/logging/FileHandlerTest.scala | 5 +++-- .../test/scala/com/twitter/logging/LoggerTest.scala | 2 +- .../com/twitter/logging/ThrottledHandlerTest.scala | 3 ++- .../twitter/util/logging/SerializableLoggingTest.scala | 2 +- 7 files changed, 13 insertions(+), 16 deletions(-) rename util-core/src/main/scala/com/twitter/{util => io}/TempFolder.scala (88%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7710c06f4a..b4cac48d92 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,7 +11,9 @@ Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ * util-core: `c.t.io.Reader.Writable` and `c.t.Reader.writable()` are removed. Use `c.t.io.Pipe` -instead. ``PHAB_ID=D226603`` + instead. ``PHAB_ID=D226603`` + +* util-core: `c.t.util.TempFolder` has been moved to `c.t.io.TempFolder`. ``PHAB_ID=D226940`` Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/util/TempFolder.scala b/util-core/src/main/scala/com/twitter/io/TempFolder.scala similarity index 88% rename from util-core/src/main/scala/com/twitter/util/TempFolder.scala rename to util-core/src/main/scala/com/twitter/io/TempFolder.scala index 28174fdb5b..ceb977ca9b 100644 --- a/util-core/src/main/scala/com/twitter/util/TempFolder.scala +++ b/util-core/src/main/scala/com/twitter/io/TempFolder.scala @@ -14,18 +14,14 @@ * limitations under the License. */ -package com.twitter.util +package com.twitter.io import java.io.File -import com.twitter.io.Files - /** * Test mixin that creates a temporary thread-local folder for a block of code to execute in. * The folder is recursively deleted after the test. * - * Note, the [[com.twitter.util.io]] package would be a better home for this trait. - * * Note that multiple uses of TempFolder cannot be nested, because the temporary directory * is effectively a thread-local global. */ @@ -59,11 +55,11 @@ trait TempFolder { * @return The current thread's active temporary folder. * @throws RuntimeException if not running within a withTempFolder block */ - def folderName = { _folderName.get.getPath } + def folderName: String = { _folderName.get.getPath } /** * @return The canonical path of the current thread's active temporary folder. * @throws RuntimeException if not running within a withTempFolder block */ - def canonicalFolderName = { _folderName.get.getCanonicalPath } + def canonicalFolderName: String = { _folderName.get.getCanonicalPath } } diff --git a/util-core/src/test/scala/com/twitter/io/FilesTest.scala b/util-core/src/test/scala/com/twitter/io/FilesTest.scala index 5f7a62eb8d..0ce3e1657f 100644 --- a/util-core/src/test/scala/com/twitter/io/FilesTest.scala +++ b/util-core/src/test/scala/com/twitter/io/FilesTest.scala @@ -1,13 +1,10 @@ package com.twitter.io import java.io.File - import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner -import com.twitter.util.TempFolder - @RunWith(classOf[JUnitRunner]) class FilesTest extends WordSpec with TempFolder { "Files" should { diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index a617744658..1f6f5b0f5f 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -18,13 +18,14 @@ package com.twitter.logging import java.io._ import java.util.{Calendar, Date, logging => javalog} - import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner import com.twitter.conversions.storage._ import com.twitter.conversions.time._ -import com.twitter.util.{TempFolder, Time} +import com.twitter.io.TempFolder +import com.twitter.util.Time + @RunWith(classOf[JUnitRunner]) class FileHandlerTest extends WordSpec with TempFolder { def reader(filename: String) = { diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index c199c23288..27f2a20668 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -17,7 +17,7 @@ package com.twitter.logging import com.twitter.conversions.time._ -import com.twitter.util.TempFolder +import com.twitter.io.TempFolder import java.net.InetSocketAddress import java.util.concurrent.{Callable, CountDownLatch, Executors, Future, TimeUnit} import java.util.{logging => javalog} diff --git a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala index 6a990588f8..51922db563 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala @@ -17,7 +17,8 @@ package com.twitter.logging import com.twitter.conversions.time._ -import com.twitter.util.{TempFolder, Time} +import com.twitter.io.TempFolder +import com.twitter.util.Time import org.scalatest.{BeforeAndAfter, WordSpec} class ThrottledHandlerTest extends WordSpec with BeforeAndAfter with TempFolder { diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/SerializableLoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/SerializableLoggingTest.scala index ba77802a1e..32460dbba3 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/SerializableLoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/SerializableLoggingTest.scala @@ -1,6 +1,6 @@ package com.twitter.util.logging -import com.twitter.util.TempFolder +import com.twitter.io.TempFolder import java.io._ import org.scalatest.FunSuite From 0c83ebc0ae1d6d62fabf80a48bca501c6457440f Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Thu, 11 Oct 2018 23:05:40 +0000 Subject: [PATCH 034/796] util-core: Remove forwarding types on c.t.util package object Problem In the interest of having finer grained pants build targets the source code within util-core should not have cycles. Solution Remove the forwarding types `c.t.util.TimeConversions` and `c.t.util.StorageUnitConversions`. Use `c.t.conversions.time` and `c.t.conversions.storage` directly as replacements. Result Removes the cycle between the `c.t.util` and `c.t.conversions` packages. JIRA Issues: CSL-7094 Differential Revision: https://phabricator.twitter.biz/D227363 --- CHANGELOG.rst | 4 ++++ .../scala/com/twitter/cache/Refresh.scala | 2 +- .../scala/com/twitter/cache/RefreshTest.scala | 13 ++++------- .../main/scala/com/twitter/util/package.scala | 23 ------------------- .../scala/com/twitter/util/ClosableTest.scala | 2 +- .../scala/com/twitter/util/TimeTest.scala | 2 +- 6 files changed, 12 insertions(+), 34 deletions(-) delete mode 100644 util-core/src/main/scala/com/twitter/util/package.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b4cac48d92..9626d3cd4d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,10 @@ Breaking API Changes * util-core: `c.t.util.TempFolder` has been moved to `c.t.io.TempFolder`. ``PHAB_ID=D226940`` +* util-core: Removed the forwarding types `c.t.util.TimeConversions` and + `c.t.util.StorageUnitConversions`. Use `c.t.conversions.time` and + `c.t.conversions.storage` directly. ``PHAB_ID=D227363`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-cache/src/main/scala/com/twitter/cache/Refresh.scala b/util-cache/src/main/scala/com/twitter/cache/Refresh.scala index 98fe27a010..3c12ffe8bc 100644 --- a/util-cache/src/main/scala/com/twitter/cache/Refresh.scala +++ b/util-cache/src/main/scala/com/twitter/cache/Refresh.scala @@ -17,7 +17,7 @@ import java.util.concurrent.atomic.AtomicReference * def getData(): Future[T] = { ... } * * can be memoized with a TTL of 1 hour as follows: - * import com.twitter.util.TimeConversions._ + * import com.twitter.conversions.time._ * import com.twitter.cache.Refresh * val getData: () => Future[T] = Refresh.every(1.hour) { ... } */ diff --git a/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala b/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala index 4bb33121d6..963a363b95 100644 --- a/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala @@ -1,14 +1,11 @@ package com.twitter.cache -import com.twitter.util.{Await, Future, Time, Promise} +import com.twitter.conversions.time._ +import com.twitter.util.{Await, Future, Promise, Time} import org.mockito.Mockito._ -import com.twitter.util.TimeConversions._ -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner import org.scalatest._ import org.scalatest.mockito.MockitoSugar -@RunWith(classOf[JUnitRunner]) class RefreshTest extends FunSuite with MockitoSugar { class Ctx { @@ -73,7 +70,7 @@ class RefreshTest extends FunSuite with MockitoSugar { val ctx = new Ctx import ctx._ - val promise = Promise[Int] + val promise = Promise[Int]() reset(provider) when(provider()) .thenReturn(promise) @@ -93,7 +90,7 @@ class RefreshTest extends FunSuite with MockitoSugar { val ctx = new Ctx import ctx._ - val promise = Promise[Int] + val promise = Promise[Int]() reset(provider) when(provider()).thenReturn(promise) @@ -108,7 +105,7 @@ class RefreshTest extends FunSuite with MockitoSugar { verify(provider, times(1))() reset(provider) - val promise2 = Promise[Int] + val promise2 = Promise[Int]() when(provider()).thenReturn(promise2) val result3 = memoizedFuture() diff --git a/util-core/src/main/scala/com/twitter/util/package.scala b/util-core/src/main/scala/com/twitter/util/package.scala deleted file mode 100644 index 4adb0b95ab..0000000000 --- a/util-core/src/main/scala/com/twitter/util/package.scala +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2010 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter - -package object util { - // backward compat for people who imported "com.twitter.util.TimeConversions._" - val TimeConversions = com.twitter.conversions.time - val StorageUnitConversions = com.twitter.conversions.storage -} diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index e74b9ef3b7..f6ae8fd72c 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.util.TimeConversions.intToTimeableNumber +import com.twitter.conversions.time.intToTimeableNumber import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 60e17819ad..821324436d 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -10,7 +10,7 @@ import org.scalatest.concurrent.{Eventually, IntegrationPatience} import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks -import com.twitter.util.TimeConversions._ +import com.twitter.conversions.time._ trait TimeLikeSpec[T <: TimeLike[T]] extends WordSpec with GeneratorDrivenPropertyChecks { val ops: TimeLikeOps[T] From 719f41a65b723c21d03348d02989c93e293beb42 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 12 Oct 2018 21:08:34 +0000 Subject: [PATCH 035/796] util-core: Reader.fromBuf, Reader.fromFile, Reader.fromInputStream now take chunk size # Problem / Solution We're preparing the Reader API to drop the "number of bytes" argument in for its `read` method. This change moves this argument to a constructor of a `BufReader` and an `InputStreamReader`. JIRA Issues: CSL-6689 TBR=true Differential Revision: https://phabricator.twitter.biz/D203154 --- CHANGELOG.rst | 8 +++ .../src/main/java/com/twitter/io/Readers.java | 12 ++-- .../main/scala/com/twitter/io/BufReader.scala | 36 ++++++---- .../com/twitter/io/InputStreamReader.scala | 27 ++++---- .../main/scala/com/twitter/io/Reader.scala | 66 ++++++++++++++----- .../com/twitter/io/ReaderCompilationTest.java | 7 +- .../twitter/concurrent/AsyncStreamTest.scala | 2 +- .../scala/com/twitter/io/BufReaderTest.scala | 25 +++---- .../twitter/io/InputStreamReaderTest.scala | 4 +- .../scala/com/twitter/io/ReaderTest.scala | 14 ++-- .../io/exp/MinimumThroughputTest.scala | 23 ++----- 11 files changed, 134 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9626d3cd4d..82c81f83d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,14 @@ Breaking API Changes `c.t.util.StorageUnitConversions`. Use `c.t.conversions.time` and `c.t.conversions.storage` directly. ``PHAB_ID=D227363`` +New Features +~~~~~~~~~~~~ + +* util-core: `c.t.io.Reader.fromBuf` (`BufReader`), `c.t.io.Reader.fromFile`, + `c.t.io.Reader.fromInputStream` (`InputStreamReader`) now take an additional parameter, + `chunkSize`, the upper bound of the number of bytes that a given reader emits at each read. + ``PHAB_ID=D203154`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index db51d879c6..99d1bfad15 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -29,8 +29,8 @@ public static Reader newEmptyReader() { /** * See {@code com.twitter.io.Reader.fromBuf}. */ - public static Reader newBufReader(Buf buf) { - return Reader$.MODULE$.fromBuf(buf); + public static Reader newBufReader(Buf buf, int chunkSize) { + return Reader$.MODULE$.fromBuf(buf, chunkSize); } /** @@ -78,15 +78,15 @@ public static Future copyMany(AsyncStream> readers, Write /** * See {@code com.twitter.io.Reader.fromFile()}. */ - public static Reader newFileReader(File f) throws FileNotFoundException { - return Reader$.MODULE$.fromFile(f); + public static Reader newFileReader(File f, int chunkSize) throws FileNotFoundException { + return Reader$.MODULE$.fromFile(f, chunkSize); } /** * See {@code com.twitter.io.Reader.fromStream()}. */ - public static Reader newInputStreamReader(InputStream in) { - return Reader$.MODULE$.fromStream(in); + public static Reader newInputStreamReader(InputStream in, int chunkSize) { + return Reader$.MODULE$.fromStream(in, chunkSize); } } diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index 9948eb39d9..26c33962c5 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -3,30 +3,40 @@ package com.twitter.io import com.twitter.util.{Future, Return, Try, Throw} /** - * Construct a Reader from a Buf. + * Construct a [[Reader]] from a [[Buf]]. */ -private[io] class BufReader(buf: Buf) extends Reader[Buf] { - @volatile private[this] var state: Try[Buf] = Return(buf) +private[io] final class BufReader(buf: Buf, chunkSize: Int) extends Reader[Buf] { + private[this] var state: Try[Buf] = Return(buf) - def read(n: Int): Future[Option[Buf]] = synchronized { + final def read(n: Int): Future[Option[Buf]] = synchronized { state match { - case Return(buf) => - if (buf.isEmpty) Future.None - else { - val f = Future.value(Some(buf.slice(0, n))) - state = Return(buf.slice(n, Int.MaxValue)) - f - } + case Return(Buf.Empty) => Future.None + case Return(b) => + state = Return(b.slice(chunkSize, Int.MaxValue)) + Future.value(Some(b.slice(0, chunkSize))) + case Throw(exc) => Future.exception(exc) } } - def discard(): Unit = synchronized { + final def discard(): Unit = synchronized { state = Throw(new Reader.ReaderDiscarded) } } object BufReader { + + /** + * Creates a [[BufReader]] out of a given `buf`. The resulting [[Reader]] emits chunks of at + * most `Int.MaxValue` bytes. + */ def apply(buf: Buf): Reader[Buf] = - if (buf.isEmpty) Reader.Null else new BufReader(buf) + apply(buf, Int.MaxValue) + + /** + * Creates a [[BufReader]] out of a given `buf`. The result [[Reader]] emits chunks of at most + * `chunkSize`. + */ + def apply(buf: Buf, chunkSize: Int): Reader[Buf] = + if (buf.isEmpty) Reader.Null else new BufReader(buf, chunkSize) } diff --git a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala index f47bf74b06..056b16358c 100644 --- a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala +++ b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala @@ -10,15 +10,22 @@ import java.io.InputStream * The given `InputStream` will be closed when [[Reader.read]] * reaches the EOF or a call to [[discard()]] or [[close()]]. */ -class InputStreamReader private[io] (inputStream: InputStream, maxBufferSize: Int, pool: FuturePool) - extends Reader[Buf] +class InputStreamReader private[io] ( + inputStream: InputStream, + chunkSize: Int, + pool: FuturePool +) extends Reader[Buf] with Closable with CloseAwaitably { private[this] val mutex = new AsyncMutex() @volatile private[this] var discarded = false - def this(inputStream: InputStream, maxBufferSize: Int) = - this(inputStream, maxBufferSize, FuturePool.interruptibleUnboundedPool) + /** + * Constructs an [[InputStreamReader]] out of a given `inputStream`. The resulting [[Reader]] + * emits chunks of at most `chunkSize`. + */ + def this(inputStream: InputStream, chunkSize: Int) = + this(inputStream, chunkSize, FuturePool.interruptibleUnboundedPool) /** * Asynchronously read at most min(`n`, `maxBufferSize`) bytes from @@ -39,9 +46,8 @@ class InputStreamReader private[io] (inputStream: InputStream, maxBufferSize: In try { if (discarded) throw new Reader.ReaderDiscarded() - val size = math.min(n, maxBufferSize) - val buffer = new Array[Byte](size) - val c = inputStream.read(buffer, 0, size) + val buffer = new Array[Byte](chunkSize) + val c = inputStream.read(buffer, 0, chunkSize) if (c == -1) { pool { inputStream.close() } None @@ -64,8 +70,7 @@ class InputStreamReader private[io] (inputStream: InputStream, maxBufferSize: In * * This closes the underlying `InputStream`. */ - def discard(): Unit = - close() + def discard(): Unit = close() /** * Discards this [[Reader]] and closes the underlying `InputStream` @@ -86,8 +91,8 @@ object InputStreamReader { */ def apply( inputStream: InputStream, - maxBufferSize: Int = DefaultMaxBufferSize + chunkSize: Int = DefaultMaxBufferSize ): InputStreamReader = - new InputStreamReader(inputStream, maxBufferSize) + new InputStreamReader(inputStream, chunkSize) } diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index b791d9a4de..f4064bfea1 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -2,7 +2,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.util._ -import java.io.{File, FileInputStream, FileNotFoundException, InputStream} +import java.io.{File, FileInputStream, InputStream} import scala.annotation.tailrec import scala.collection.mutable.ListBuffer @@ -97,6 +97,8 @@ trait Reader[+A] { object Reader { + class ReaderDiscarded extends Exception("This writer's reader has been discarded") + val Null: Reader[Nothing] = new Reader[Nothing] { def read(n: Int): Future[Option[Nothing]] = Future.None def discard(): Unit = () @@ -183,35 +185,69 @@ object Reader { new Framed(r, new ChunkedFramer(chunkSize)) /** - * Reader from a Buf. + * Create a new [[Reader]] from a given [[Buf]]. The output of a returned reader is chunked by + * a least `chunkSize` (bytes). + * + * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. */ - def fromBuf(buf: Buf): Reader[Buf] = BufReader(buf) + def fromBuf(buf: Buf): Reader[Buf] = fromBuf(buf, Int.MaxValue) - class ReaderDiscarded extends Exception("This writer's reader has been discarded") + /** + * Create a new [[Reader]] from a given [[Buf]]. The output of a returned reader is chunked by + * at most `chunkSize` (bytes). + * + * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. + */ + def fromBuf(buf: Buf, chunkSize: Int): Reader[Buf] = BufReader(buf, chunkSize) /** - * Create a new [[Reader]] for a `File`. + * Create a new [[Reader]] from a given [[File]]. The output of a returned reader is chunked by + * at most `chunkSize` (bytes). * - * The resources held by the returned [[Reader]] are released - * on reading of EOF and [[Reader.discard()]]. + * The resources held by the returned [[Reader]] are released on reading of EOF and + * [[Reader.discard()]]. * + * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. * @see `Readers.fromFile` for a Java API */ - @throws(classOf[FileNotFoundException]) - @throws(classOf[SecurityException]) - def fromFile(f: File): Reader[Buf] = - fromStream(new FileInputStream(f)) + def fromFile(f: File): Reader[Buf] = fromFile(f, InputStreamReader.DefaultMaxBufferSize) /** - * Wrap `InputStream` with a [[Reader]]. + * Create a new [[Reader]] from a given [[File]]. The output of a returned reader is chunked by + * at most `chunkSize` (bytes). * - * Note that the given `InputStream` will be closed - * on reading of EOF and [[Reader.discard()]]. + * The resources held by the returned [[Reader]] are released on reading of EOF and + * [[Reader.discard()]]. * + * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. + * @see `Readers.fromFile` for a Java API + */ + def fromFile(f: File, chunkSize: Int): Reader[Buf] = fromStream(new FileInputStream(f), chunkSize) + + /** + * Create a new [[Reader]] from a given [[InputStream]]. The output of a returned reader is + * chunked by at most `chunkSize` (bytes). + * + * The resources held by the returned [[Reader]] are released on reading of EOF and + * [[Reader.discard()]]. + * + * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. * @see `Readers.fromStream` for a Java API */ def fromStream(s: InputStream): Reader[Buf] = - InputStreamReader(s) + fromStream(s, InputStreamReader.DefaultMaxBufferSize) + + /** + * Create a new [[Reader]] from a given [[InputStream]]. The output of a returned reader is + * chunked by at most `chunkSize` (bytes). + * + * The resources held by the returned [[Reader]] are released on reading of EOF and + * [[Reader.discard()]]. + * + * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. + * @see `Readers.fromStream` for a Java API + */ + def fromStream(s: InputStream, chunkSize: Int): Reader[Buf] = InputStreamReader(s, chunkSize) /** * Allow [[AsyncStream]] to be consumed as a [[Reader]] diff --git a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java b/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java index b8c2d90a25..2a86332ffb 100644 --- a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java +++ b/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java @@ -18,7 +18,7 @@ public void testNull() { @Test public void testNewBufReader() { - Readers.newBufReader(Bufs.EMPTY); + Readers.newBufReader(Bufs.EMPTY, 1024); } @Test @@ -41,7 +41,7 @@ public void testPipe() { @Test public void testNewFileReader() { try { - Readers.newFileReader(new File("a")); + Readers.newFileReader(new File("a") ,1024); } catch (Exception x) { // ok } @@ -49,7 +49,6 @@ public void testNewFileReader() { @Test public void testNewInputStreamReader() throws Exception { - Readers.newInputStreamReader(new ByteArrayInputStream(new byte[0])); + Readers.newInputStreamReader(new ByteArrayInputStream(new byte[0]), 1024); } - } diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 649aa4cfed..8cfd285f27 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -625,7 +625,7 @@ class AsyncStreamTest extends FunSuite with GeneratorDrivenPropertyChecks { test("fromReader") { forAll { l: List[Byte] => val buf = Buf.ByteArray.Owned(l.toArray) - val as = AsyncStream.fromReader(Reader.fromBuf(buf), chunkSize = 1) + val as = AsyncStream.fromReader(Reader.fromBuf(buf, 1), chunkSize = 1) assert(toSeq(as).map(b => Buf.ByteArray.Owned.extract(b).head) == l) } diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index 4e5ffb0765..fdd69dd5ad 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -1,31 +1,34 @@ package com.twitter.io -import com.twitter.util.Await +import com.twitter.conversions.time._ +import com.twitter.util.{Await, Future} import org.junit.runner.RunWith -import org.scalacheck.Prop import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner import org.scalatest.prop.Checkers @RunWith(classOf[JUnitRunner]) class BufReaderTest extends FunSuite with Checkers { - import Prop.{forAll, throws} + + private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) test("BufReader") { - check(forAll { (bytes: String) => + check { bytes: String => val buf = Buf.Utf8(bytes) - val r = BufReader(buf) - Await.result(Reader.readAll(r)) == buf - }) + val r = Reader.fromBuf(buf, 8) + await(Reader.readAll(r)) == buf + } } test("BufReader - discard") { - check(forAll { (bytes: String, n: Int) => - val r = BufReader(Buf.Utf8(bytes)) + check { (bytes: String, n: Int) => + val r = Reader.fromBuf(Buf.Utf8(bytes), n) r.discard() + n < 0 || bytes.length == 0 || - throws(classOf[Reader.ReaderDiscarded])({ Await.result(r.read(n)) }) - }) + Await + .ready(r.read(n), 5.seconds).poll.exists(_.throwable.isInstanceOf[Reader.ReaderDiscarded]) + } } } diff --git a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala index 9a25539889..4a28e46555 100644 --- a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala @@ -34,7 +34,7 @@ class InputStreamReaderTest extends FunSuite { test("InputStreamReader - read positive bytes") { val a = arr(0, 25) val s = new ByteArrayInputStream(a) - val r = new InputStreamReader(s, 4096) + val r = new InputStreamReader(s, 10) val f1 = r.read(10) assert(Await.result(f1, 5.seconds) == Some(buf(0, 10))) @@ -90,7 +90,7 @@ class InputStreamReaderTest extends FunSuite { test("InputStreamReader - discard") { val a = arr(0, 25) val s = new ByteArrayInputStream(a) - val r = new InputStreamReader(s, 4096) + val r = new InputStreamReader(s, 10) val f1 = r.read(10) assert(Await.result(f1, 5.seconds) == Some(buf(0, 10))) diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index b499e2e2d9..4df9ec256a 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -84,7 +84,7 @@ class ReaderTest forAll(stringAndChunk) { case (s, i) => - val r = Reader.chunked(Reader.fromBuf(Buf.Utf8(s)), i) + val r = Reader.chunked(Reader.fromBuf(Buf.Utf8(s), 32), i) def readLoop(): Unit = await(r.read(Int.MaxValue)) match { case Some(b) => @@ -99,9 +99,7 @@ class ReaderTest test("Reader.concat") { forAll { ss: List[String] => - val readers = ss.map { s => - BufReader(Buf.Utf8(s)) - } + val readers = ss.map(s => Reader.fromBuf(Buf.Utf8(s), 16)) val buf = Reader.readAll(Reader.concat(AsyncStream.fromSeq(readers))) assert(await(buf) == Buf.Utf8(ss.mkString)) } @@ -156,7 +154,7 @@ class ReaderTest test("Reader.fromStream closes resources on EOF read") { val in = spy(new ByteArrayInputStream(arr(0, 10))) - val r = Reader.fromStream(in) + val r = Reader.fromStream(in, 4) val f = Reader.readAll(r) assert(await(f) == buf(0, 10)) eventually { @@ -166,11 +164,9 @@ class ReaderTest test("Reader.fromStream closes resources on discard") { val in = spy(new ByteArrayInputStream(arr(0, 10))) - val r = Reader.fromStream(in) + val r = Reader.fromStream(in, 4) r.discard() - eventually { - verify(in).close() - } + eventually { verify(in).close() } } test("Reader.fromAsyncStream completes when stream is empty") { diff --git a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala b/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala index 55c48e7fb0..f53b502bfb 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala +++ b/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala @@ -19,30 +19,17 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { } test("Reader - faster than min") { - val buf = Buf.UsAscii("soylent green is made of...") // 27 bytes - val reader = MinimumThroughput.reader(BufReader(buf), 0d, Timer.Nil) + val buf = Buf.UsAscii("soylent green is") // 16 bytes + val reader = MinimumThroughput.reader(Reader.fromBuf(buf, 8), 0d, Timer.Nil) // read from the beginning - Await.result(reader.read(13)) match { - case Some(b) => assert(b == Buf.UsAscii("soylent green")) - case _ => fail() - } - - // a no-op read Await.result(reader.read(0)) match { - case Some(b) => assert(b == Buf.Empty) - case _ => fail() - } - - // read to the end - Await.result(reader.read(20)) match { - case Some(b) => assert(b == Buf.UsAscii(" is made of...")) + case Some(b) => assert(b == buf.slice(0, 8)) case _ => fail() } - // read past the end - Await.result(reader.read(20)) match { - case None => + Await.result(reader.read(0)) match { + case Some(b) => assert(b == buf.slice(8, 16)) case _ => fail() } } From 34f18ff9ff12d0ff6f8e9ec54396273d3feecae0 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Sat, 13 Oct 2018 00:37:38 +0000 Subject: [PATCH 036/796] util-core: Add onClose into Writer Problem/Solution: It would be useful to have an `onClose` in Writer to signal layers above about the stream's closure. JIRA Issues: CSL-6814 Differential Revision: https://phabricator.twitter.biz/D226319 --- CHANGELOG.rst | 3 ++ .../com/twitter/io/OutputStreamWriter.scala | 2 + .../src/main/scala/com/twitter/io/Pipe.scala | 42 +++++++++---------- .../main/scala/com/twitter/io/Writer.scala | 8 ++++ .../twitter/io/exp/MinimumThroughput.scala | 2 + .../test/scala/com/twitter/io/PipeTest.scala | 17 ++++++++ .../io/exp/MinimumThroughputTest.scala | 2 + 7 files changed, 55 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 82c81f83d0..8bc0b8ce0e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -42,6 +42,9 @@ Breaking API Changes * util-core: `c.t.io.Writer` now extends `c.t.util.Closable`. `c.t.io.Writer.ClosableWriter` is no longer exist. ``PHAB_ID=D218453`` +* util-core: Add `onClose` into `c.t.io.Writer`, it exposes a `Future` that is satisfied when + the stream is closed. ``PHAB_ID=D226319`` + Bug Fixes ~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala b/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala index 1f504d401c..90d5e95024 100644 --- a/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala +++ b/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala @@ -58,6 +58,8 @@ private[io] class OutputStreamWriter(out: OutputStream, bufsize: Int) extends Wr if (done.updateIfEmpty(Throw(CloseExc))) FuturePool.unboundedPool { out.close() } else Future.Done + + def onClose: Future[Unit] = done } private object OutputStreamWriter { diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 38f0a3e98f..bacc230fef 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -78,14 +78,17 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { // thread-safety provided by synchronization on `this` private[this] var state: State[A] = State.Idle + // satisfied when a `read` observes the EOF (after calling close()) + private[this] val closep: Promise[Unit] = new Promise[Unit]() + def read(n: Int): Future[Option[A]] = synchronized { state match { case State.Failed(exc) => Future.exception(exc) - case State.Closing(reof) => + case State.Closing => state = State.Closed - reof.setDone() + closep.setDone() Future.None case State.Closed => @@ -119,7 +122,7 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { case State.Failed(exc) => Future.exception(exc) - case State.Closed | State.Closing(_) => + case State.Closed | State.Closing => Future.exception(new IllegalStateException("write() while closed")) case State.Idle => @@ -149,17 +152,17 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { state match { case State.Closed | State.Failed(_) => // do not update state to failing - case State.Idle => + case State.Idle | State.Closing => state = State.Failed(cause) - case State.Closing(reof) => - state = State.Failed(cause) - reof.setException(cause) + closep.setException(cause) case State.Reading(_, p) => state = State.Failed(cause) p.setException(cause) + closep.setException(cause) case State.Writing(_, p) => state = State.Failed(cause) p.setException(cause) + closep.setException(cause) } } @@ -171,27 +174,28 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { case State.Closed => Future.Done - case State.Closing(p) => - p + case State.Closing => + closep case State.Idle => - val reof = new Promise[Unit]() - state = State.Closing(reof) - reof + state = State.Closing + closep case State.Reading(_, p) => state = State.Closed p.update(Return.None) + closep.setDone() Future.Done case State.Writing(_, p) => - val reof = new Promise[Unit]() - state = State.Closing(reof) + state = State.Closing p.setException(new IllegalStateException("close() while write is pending")) - reof + closep } } + def onClose: Future[Unit] = closep + override def toString: String = synchronized(s"Pipe(state=$state)") } @@ -223,12 +227,8 @@ object Pipe { /** Indicates the pipe was failed. */ final case class Failed(exc: Throwable) extends State[Nothing] - /** - * Indicates a close occurred while `Idle` — no reads or writes were pending. - * - * @param reof satisfied when a `read` sees the EOF. - */ - final case class Closing(reof: Promise[Unit]) extends State[Nothing] + /** Indicates a close occurred while `Idle` — no reads or writes were pending.*/ + case object Closing extends State[Nothing] /** Indicates the reader has seen the EOF. No more reads or writes are allowed. */ case object Closed extends State[Nothing] diff --git a/util-core/src/main/scala/com/twitter/io/Writer.scala b/util-core/src/main/scala/com/twitter/io/Writer.scala index eab9e11685..e64224273d 100644 --- a/util-core/src/main/scala/com/twitter/io/Writer.scala +++ b/util-core/src/main/scala/com/twitter/io/Writer.scala @@ -71,6 +71,12 @@ trait Writer[-A] extends Closable { self => */ def fail(cause: Throwable): Unit + /** + * Expose a `Future` that is satisfied when the stream is closed. It may contain an error + * if the Writer is failed. + */ + def onClose: Future[Unit] + /** * Given f, a function from B into A, creates an Writer[B] whose `fail` and `close` functions * are equivalent to Writer[A]'s. Writer[B]'s `write` function is equivalent to: @@ -82,6 +88,7 @@ trait Writer[-A] extends Closable { self => def write(element: B): Future[Unit] = self.write(f(element)) def fail(cause: Throwable): Unit = self.fail(cause) def close(deadline: Time): Future[Unit] = self.close(deadline) + def onClose: Future[Unit] = self.onClose } } @@ -106,6 +113,7 @@ object Writer { def write(element: A): Future[Unit] = Future.exception(new IllegalStateException("NullWriter")) def fail(cause: Throwable): Unit = () def close(deadline: Time): Future[Unit] = Future.Done + def onClose: Future[Unit] = Future.never } val BufferSize: Int = 4096 diff --git a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala b/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala index 273c7e6d37..6e63ae6695 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala +++ b/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala @@ -130,6 +130,8 @@ object MinimumThroughput { } def close(deadline: Time): Future[Unit] = writer.close(deadline) + + def onClose: Future[Unit] = writer.onClose } /** diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index 0e47fafc9e..5a046b7595 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -115,10 +115,14 @@ class PipeTest extends FunSuite with Matchers { test("fail while reading") { val rw = new Pipe[Buf] + var closed = false + rw.onClose.ensure { closed = true } val rf = rw.read(6) assert(!rf.isDefined) + assert(!closed) val exc = new Exception rw.fail(exc) + assert(closed) assert(rf.isDefined) val exc1 = intercept[Exception] { await(rf) } assert(exc eq exc1) @@ -134,19 +138,26 @@ class PipeTest extends FunSuite with Matchers { test("discard") { val rw = new Pipe[Buf] + var closed = false + rw.onClose.ensure { closed = true } rw.discard() val rf = rw.read(10) assert(rf.isDefined) + assert(closed) intercept[Reader.ReaderDiscarded] { await(rf) } } test("close") { val rw = new Pipe[Buf] + var closed = false + rw.onClose.ensure { closed = true } val wf = rw.write(buf(0, 6)) before rw.close() assert(!wf.isDefined) + assert(!closed) assert(await(rw.read(6)) == Some(buf(0, 6))) assert(!wf.isDefined) assertReadEofAndClosed(rw) + assert(closed) } test("write then reads then close") { @@ -234,6 +245,8 @@ class PipeTest extends FunSuite with Matchers { test("write while write pending") { val rw = new Pipe[Buf] + var closed = false + rw.onClose.ensure { closed = true } val wf = rw.write(buf(0, 1)) assert(!wf.isDone) @@ -243,6 +256,7 @@ class PipeTest extends FunSuite with Matchers { // the extraneous write should not mess with the 1st one. assertRead(rw, 0, 1) + assert(!closed) } test("read after fail") { @@ -273,11 +287,14 @@ class PipeTest extends FunSuite with Matchers { test("read while reading") { val rw = new Pipe[Buf] + var closed = false + rw.onClose.ensure { closed = true } val rf = rw.read(1) intercept[IllegalStateException] { await(rw.read(1)) } assert(!rf.isDefined) + assert(!closed) } test("discard with pending read") { diff --git a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala b/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala index f53b502bfb..f6f4bc96f7 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala +++ b/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala @@ -176,6 +176,8 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { } def close(deadline: Time): Future[Unit] = Future.Done + + def onClose: Future[Unit] = Future.never } val writer = MinimumThroughput.writer( From eb56ab962e478ce46e81eb19c1bc6ce5da3dc2c6 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 15 Oct 2018 18:53:26 +0000 Subject: [PATCH 037/796] util-core: Do not run promises under a lock in Pipe # Problem / Solution Satisfying promises within a `synchronized` block is vulnerable to deadlocks. Let's not do it. JIRA Issues: CSL-7016 Differential Revision: https://phabricator.twitter.biz/D226931 --- .../scala/com/twitter/io/PipeBenchmark.scala | 27 +++ .../src/main/scala/com/twitter/io/Pipe.scala | 221 ++++++++++-------- 2 files changed, 153 insertions(+), 95 deletions(-) create mode 100644 util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala diff --git a/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala new file mode 100644 index 0000000000..f620c131f7 --- /dev/null +++ b/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala @@ -0,0 +1,27 @@ +package com.twitter.io + +import com.twitter.util.{Future, StdBenchAnnotations} +import org.openjdk.jmh.annotations.{Benchmark, Measurement, Scope, State, Warmup} + +// sbt 'project util-benchmark' 'jmh:run PipeBenchmark' +@State(Scope.Benchmark) +@Warmup(iterations = 2) +@Measurement(iterations = 2) +class PipeBenchmark extends StdBenchAnnotations { + + private val foo = Buf.Utf8("foo") + + @Benchmark + def writeAndRead(): Future[Option[Buf]] = { + val p = new Pipe[Buf] + p.write(foo) + p.read(Int.MaxValue) + } + + @Benchmark + def readAndWrite(): Future[Unit] = { + val p = new Pipe[Buf] + p.read(Int.MaxValue) + p.write(foo) + } +} diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index bacc230fef..7eb2c71ce1 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -73,6 +73,18 @@ import com.twitter.util.{Future, Promise, Return, Time} */ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { + // Implementation Notes: + // + // - The the only mutable state of this pipe is `state` and the access to it is synchronized + // on `this`. + // + // - We do not run promises under a lock (`synchronized`) as it could lead to a deadlock if + // there are interleaved queue operations in the waiter closure. + // + // - Although promises are run without a lock, synchronized `state` transitions guarantee + // a promise that needs to be run is now owned exclusively by a caller, hence no concurrent + // updates will be racing with it. + import Pipe._ // thread-safety provided by synchronization on `this` @@ -81,117 +93,136 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { // satisfied when a `read` observes the EOF (after calling close()) private[this] val closep: Promise[Unit] = new Promise[Unit]() - def read(n: Int): Future[Option[A]] = synchronized { - state match { - case State.Failed(exc) => - Future.exception(exc) - - case State.Closing => - state = State.Closed - closep.setDone() - Future.None - - case State.Closed => - Future.None - - case State.Idle => - val p = new Promise[Option[A]] - state = State.Reading(n, p) - p - - case State.Writing(buf, p) if buf.length <= n => - // pending write can fully fit into this requested read - state = State.Idle - p.setDone() - Future.value(Some(buf)) - - case State.Writing(buf, p) => - // pending write is larger than the requested read - state = State.Writing(buf.slice(n, buf.length).asInstanceOf[A], p) - Future.value(Some(buf.slice(0, n).asInstanceOf[A])) - - case State.Reading(_, _) => - Future.exception(new IllegalStateException("read() while read is pending")) + def read(n: Int): Future[Option[A]] = { + val (waiter, result) = synchronized { + state match { + case State.Failed(exc) => + (null, Future.exception(exc)) + + case State.Closing => + state = State.Closed + (closep, Future.None) + + case State.Closed => + (null, Future.None) + + case State.Idle => + val p = new Promise[Option[A]] + state = State.Reading(n, p) + (null, p) + + case State.Writing(buf, p) if buf.length <= n => + // pending write can fully fit into this requested read + state = State.Idle + (p, Future.value(Some(buf))) + + case State.Writing(buf, p) => + // pending write is larger than the requested read + state = State.Writing(buf.slice(n, buf.length).asInstanceOf[A], p) + (null, Future.value(Some(buf.slice(0, n).asInstanceOf[A]))) + + case State.Reading(_, _) => + (null, Future.exception(new IllegalStateException("read() while read is pending"))) + } } + + if (waiter != null) waiter.setDone() + result } def discard(): Unit = fail(new Reader.ReaderDiscarded()) - def write(buf: A): Future[Unit] = synchronized { - state match { - case State.Failed(exc) => - Future.exception(exc) - - case State.Closed | State.Closing => - Future.exception(new IllegalStateException("write() while closed")) - - case State.Idle => - val p = new Promise[Unit]() - state = State.Writing(buf, p) - p - - case State.Reading(n, p) if n < buf.length => - // pending reader doesn't have enough space for this write - val nextp = new Promise[Unit]() - state = State.Writing(buf.slice(n, buf.length).asInstanceOf[A], nextp) - p.setValue(Some(buf.slice(0, n).asInstanceOf[A])) - nextp - - case State.Reading(n, p) => - // pending reader has enough space for the full write - state = State.Idle - p.setValue(Some(buf)) - Future.Done - - case State.Writing(_, _) => - Future.exception(new IllegalStateException("write() while write is pending")) + def write(buf: A): Future[Unit] = { + val (waiter, value, result) = synchronized { + state match { + case State.Failed(exc) => + (null, null, Future.exception(exc)) + + case State.Closed | State.Closing => + (null, null, Future.exception(new IllegalStateException("write() while closed"))) + + case State.Idle => + val p = new Promise[Unit] + state = State.Writing(buf, p) + (null, null, p) + + case State.Reading(n, p) if n < buf.length => + // pending reader doesn't have enough space for this write + val nextp = new Promise[Unit] + state = State.Writing(buf.slice(n, buf.length).asInstanceOf[A], nextp) + (p, Some(buf.slice(0, n).asInstanceOf[A]), nextp) + + case State.Reading(n, p) => + // pending reader has enough space for the full write + state = State.Idle + (p, Some(buf), Future.Done) + + case State.Writing(_, _) => + ( + null, + null, + Future.exception(new IllegalStateException("write() while write is pending")) + ) + } } + + // The waiter and the value are mutually inclusive so just checking against waiter is adequate. + if (waiter != null) waiter.setValue(value) + result } - def fail(cause: Throwable): Unit = synchronized { - state match { - case State.Closed | State.Failed(_) => - // do not update state to failing - case State.Idle | State.Closing => - state = State.Failed(cause) - closep.setException(cause) - case State.Reading(_, p) => - state = State.Failed(cause) - p.setException(cause) - closep.setException(cause) - case State.Writing(_, p) => - state = State.Failed(cause) - p.setException(cause) - closep.setException(cause) + def fail(cause: Throwable): Unit = { + val (closer, reader, writer) = synchronized { + state match { + case State.Closed | State.Failed(_) => + // do not update state to failing + (null, null, null) + case State.Idle | State.Closing => + state = State.Failed(cause) + (closep, null, null) + case State.Reading(_, p) => + state = State.Failed(cause) + (closep, p, null) + case State.Writing(_, p) => + state = State.Failed(cause) + (closep, null, p) + } } - } - def close(deadline: Time): Future[Unit] = synchronized { - state match { - case State.Failed(t) => - Future.exception(t) + if (reader != null) reader.setException(cause) + else if (writer != null) writer.setException(cause) - case State.Closed => - Future.Done + if (closer != null) closer.setException(cause) + } - case State.Closing => - closep + def close(deadline: Time): Future[Unit] = { + val (closer, reader, writer) = synchronized { + state match { + case State.Failed(_) | State.Closed | State.Closing => + (null, null, null) - case State.Idle => - state = State.Closing - closep + case State.Idle => + state = State.Closing + (null, null, null) - case State.Reading(_, p) => - state = State.Closed - p.update(Return.None) - closep.setDone() - Future.Done + case State.Reading(_, p) => + state = State.Closed + (closep, p, null) - case State.Writing(_, p) => - state = State.Closing - p.setException(new IllegalStateException("close() while write is pending")) - closep + case State.Writing(_, p) => + state = State.Closing + (null, null, p) + } } + + if (reader != null) + reader.update(Return.None) + else if (writer != null) + writer.setException(new IllegalStateException("close() while write is pending")) + + if (closer != null) closer.setDone() + + onClose } def onClose: Future[Unit] = closep From 410c4319e311e656310c3a256d5990f60f7e89fb Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 15 Oct 2018 23:17:16 +0000 Subject: [PATCH 038/796] Upgrade scalac 2.12 to 2.12.7 Problem / Solution There is a new scalac 2.12.x version so lets upgrade. JIRA Issues: CSL-7111 Differential Revision: https://phabricator.twitter.biz/D228382 --- .travis.yml | 2 +- build.sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ec949997f0..22e5acd724 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ before_cache: scala: - 2.11.12 - - 2.12.6 + - 2.12.7 jdk: - oraclejdk8 diff --git a/build.sbt b/build.sbt index 12cec026d2..694d8be5ea 100644 --- a/build.sbt +++ b/build.sbt @@ -21,8 +21,8 @@ val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.13.4" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion val defaultProjectSettings = Seq( - scalaVersion := "2.12.6", - crossScalaVersions := Seq("2.11.12", "2.12.6") + scalaVersion := "2.12.7", + crossScalaVersions := Seq("2.11.12", "2.12.7") ) val baseSettings = Seq( From 6c3be47d69bf628a9b87146309e2e02844a2d25b Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 16 Oct 2018 03:48:43 +0000 Subject: [PATCH 039/796] util-core: Move concurrent.AsyncStream.fromReader to io.Reader.toAsyncStream Problem In the interest of having finer grained pants build targets the source code within util-core should not have cycles. Solution Move `c.t.concurrent.AsyncStream.fromReader` to `c.t.io.Reader.toAsyncStream`. Result Removes the cycle between the `c.t.concurrent` and `c.t.io` packages. JIRA Issues: CSL-7094 TBR=true Differential Revision: https://phabricator.twitter.biz/D228277 --- CHANGELOG.rst | 3 +++ .../com/twitter/concurrent/AsyncStream.scala | 10 ---------- .../src/main/scala/com/twitter/io/Reader.scala | 9 +++++++++ .../twitter/concurrent/AsyncStreamTest.scala | 10 ---------- .../test/scala/com/twitter/io/ReaderTest.scala | 17 +++++++++-------- 5 files changed, 21 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8bc0b8ce0e..349adf5159 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,9 @@ Breaking API Changes `c.t.util.StorageUnitConversions`. Use `c.t.conversions.time` and `c.t.conversions.storage` directly. ``PHAB_ID=D227363`` +* util-core: `c.t.concurrent.AsyncStream.fromReader` has been moved to + `c.t.io.Reader.toAsyncStream`. ``PHAB_ID=D228277`` + New Features ~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index df22e803c4..153b3359af 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -1,6 +1,5 @@ package com.twitter.concurrent -import com.twitter.io.{Buf, Reader} import com.twitter.util.{Future, Promise, Return, Throw, Try} import scala.annotation.varargs import scala.collection.mutable @@ -677,15 +676,6 @@ object AsyncStream { case Some(a) => of(a) } - /** - * Transformation (or lift) from [[Reader]] into `AsyncStream`. - */ - def fromReader[A <: Buf](r: Reader[A], chunkSize: Int = Int.MaxValue): AsyncStream[A] = - fromFuture(r.read(chunkSize)).flatMap { - case Some(buf) => buf +:: fromReader(r, chunkSize) - case None => AsyncStream.empty[A] - } - /** * Lift from [[Future]] into `AsyncStream` and then flatten. */ diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index f4064bfea1..4480adeb19 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -263,6 +263,15 @@ object Reader { pipe } + /** + * Transformation (or lift) from [[Reader]] into `AsyncStream`. + */ + def toAsyncStream[A <: Buf](r: Reader[A], chunkSize: Int = Int.MaxValue): AsyncStream[A] = + AsyncStream.fromFuture(r.read(chunkSize)).flatMap { + case Some(buf) => buf +:: Reader.toAsyncStream(r, chunkSize) + case None => AsyncStream.empty[A] + } + /** * Convenient abstraction to read from a stream of Readers as if it were a * single Reader. diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 8cfd285f27..490992a0fd 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -1,7 +1,6 @@ package com.twitter.concurrent import com.twitter.conversions.time._ -import com.twitter.io.{Buf, Reader} import com.twitter.util._ import org.junit.runner.RunWith import org.scalacheck.{Arbitrary, Gen} @@ -622,15 +621,6 @@ class AsyncStreamTest extends FunSuite with GeneratorDrivenPropertyChecks { } } - test("fromReader") { - forAll { l: List[Byte] => - val buf = Buf.ByteArray.Owned(l.toArray) - val as = AsyncStream.fromReader(Reader.fromBuf(buf, 1), chunkSize = 1) - - assert(toSeq(as).map(b => Buf.ByteArray.Owned.extract(b).head) == l) - } - } - test("sum") { forAll { xs: List[Int] => assert(xs.sum == await(AsyncStream.fromSeq(xs).sum)) diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 4df9ec256a..f5caa35293 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -25,14 +25,6 @@ class ReaderTest private def await[T](t: Awaitable[T]): T = Await.result(t, 5.seconds) - private def toSeq(b: Option[Buf]): Seq[Byte] = b match { - case None => fail("Expected full buffer") - case Some(buf) => - val a = new Array[Byte](buf.length) - buf.write(a, 0) - a.toSeq - } - def undefined: AsyncStream[Reader[Buf]] = throw new Exception private def assertReadWhileReading(r: Reader[Buf]): Unit = { @@ -201,6 +193,15 @@ class ReaderTest assert(tailEvaluated.get()) } + test("Reader.toAsyncStream") { + forAll { l: List[Byte] => + val buf = Buf.ByteArray.Owned(l.toArray) + val as = Reader.toAsyncStream(Reader.fromBuf(buf, 1), chunkSize = 1) + + assert(await(as.toSeq()).map(b => Buf.ByteArray.Owned.extract(b).head) == l) + } + } + test("Reader.framed reads framed data") { val getByteArrays: Gen[Seq[Buf]] = Gen.listOf( for { From 6b41ef4711ede2dc1590725f6042da68f33fac5d Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 17 Oct 2018 00:46:28 +0000 Subject: [PATCH 040/796] util-core: More examples for Future APIs Problem/Solution: Adding examples for `handle`, `rescue`, `respond`, and `transform`. Differential Revision: https://phabricator.twitter.biz/D228976 --- .../main/scala/com/twitter/util/Future.scala | 113 ++++++++++++++---- 1 file changed, 92 insertions(+), 21 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 56500b6627..47dd22453b 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1050,7 +1050,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * {{{ * // will return a Future of `Seq(2, 3, 4)` * Future.traverseSequentially(Seq(1, 2, 3)) { i => - * Future(i + 1) + * Future.value(i + 1) * } * }}} * @@ -1436,6 +1436,10 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * println("always printed") * } * }}} + * + * @define awaitresult + * // Await.result blocks the current thread, + * // don't use it except for tests. */ abstract class Future[+A] extends Awaitable[A] { self => @@ -1450,8 +1454,23 @@ abstract class Future[+A] extends Awaitable[A] { self => * libraries). Otherwise, it is a best practice to use one of the * alternatives ([[onSuccess]], [[onFailure]], etc.). * - * @note this should be used for side-effects. + * @example + * {{{ + * import com.twitter.util.{Await, Future, Return, Throw} + * val f1: Future[Int] = Future.value(1) + * val f2: Future[Int] = Future.exception(new Exception("boom!")) + * val printing: Future[Int] => Future[Int] = x => x.respond { + * case Return(_) => println("Here's a Return") + * case Throw(_) => println("Here's an Exception") + * } + * Await.result(printing(f1)) + * // Prints side-effect "Here's a Return" and then returns Future value "1" + * Await.result(printing(f2)) + * // Prints side-effect "Here's an Exception" and then throws java.lang.Exception: boom! + * $awaitresult + * }}} * + * @note this should be used for side-effects. * @param k the side-effect to apply when the computation completes. * The value of the input to `k` will be the result of the * computation to this future. @@ -1472,15 +1491,14 @@ abstract class Future[+A] extends Awaitable[A] { self => * The returned `Future` will be satisfied when this, * the original future, is done. * - * @note this should be used for side-effects. - * - * For example: + * @example * $callbacks * {{{ * val a = Future.value(1) * callbacks(a) // prints "1" and then "always printed" * }}} * + * @note this should be used for side-effects. * @param f the side-effect to apply when the computation completes. * @see [[respond]] if you need the result of the computation for * usage in the side-effect. @@ -1700,6 +1718,20 @@ abstract class Future[+A] extends Awaitable[A] { self => * The returned `Future` will be satisfied when `this`, * the original future, and `f` are done. * + * @example + * {{{ + * import com.twitter.util.{Await, Future, Return, Throw} + * val f1: Future[Int] = Future.value(1) + * val f2: Future[Int] = Future.exception(new Exception("boom!")) + * val transforming: Future[Int] => Future[String] = x => x.transform { + * case Return(i) => Future.value(i.toString) + * case Throw(e) => Future.value(e.getMessage) + * } + * Await.result(transforming(f1)) // String = 1 + * Await.result(transforming(f2)) // String = "boom!" + * $awaitresult + * }}} + * * @see [[respond]] for purely side-effecting callbacks. * @see [[map]] and [[flatMap]] for dealing strictly with successful * computations. @@ -1734,26 +1766,30 @@ abstract class Future[+A] extends Awaitable[A] { self => * If this, the original future, succeeds, run `f` on the result. * * The returned result is a Future that is satisfied when the original future - * and the callback, `f`, are done. For example: + * and the callback, `f`, are done. + * + * @example * {{{ * import com.twitter.util.{Await, Future} * val f: Future[Int] = Future.value(1) * val newf: Future[Int] = f.flatMap { x => - * Future(x + 10) + * Future.value(x + 10) * } * Await.result(newf) // 11 + * $awaitresult * }}} * * If the original future fails, this one will also fail, without executing `f` - * and preserving the failed computation of `this`. For example: + * and preserving the failed computation of `this` + * @example * {{{ * import com.twitter.util.{Await, Future} * val f: Future[Int] = Future.exception(new Exception("boom!")) * val newf: Future[Int] = f.flatMap { x => * println("I'm being executed") // won't print - * Future(x + 10) + * Future.value(x + 10) * } - * Await.result(newf) // java.lang.Exception: boom! + * Await.result(newf) // throws java.lang.Exception: boom! * }}} * * @see [[map]] @@ -1784,6 +1820,19 @@ abstract class Future[+A] extends Awaitable[A] { self => * * This is the equivalent of [[flatMap]] for failed computations. * + * @example + * {{{ + * import com.twitter.util.{Await, Future} + * val f1: Future[Int] = Future.exception(new Exception("boom1!")) + * val f2: Future[Int] = Future.exception(new Exception("boom2!")) + * val newf: Future[Int] => Future[Int] = x => x.rescue { + * case e: Exception if e.getMessage == "boom1!" => Future.value(1) + * } + * Await.result(newf(f1)) // 1 + * Await.result(newf(f2)) // throws java.lang.Exception: boom2! + * $awaitresult + * }}} + * * @see [[handle]] */ def rescue[B >: A]( @@ -1808,7 +1857,9 @@ abstract class Future[+A] extends Awaitable[A] { self => * If this, the original future, succeeds, run `f` on the result. * * The returned result is a Future that is satisfied when the original future - * and the callback, `f`, are done. For example: + * and the callback, `f`, are done. + * + * @example * {{{ * import com.twitter.util.{Await, Future} * val f: Future[Int] = Future.value(1) @@ -1816,10 +1867,13 @@ abstract class Future[+A] extends Awaitable[A] { self => * x + 10 * } * Await.result(newf) // 11 + * $awaitresult * }}} * * If the original future fails, this one will also fail, without executing `f` - * and preserving the failed computation of `this`. For example: + * and preserving the failed computation of `this` + * + * @example * {{{ * import com.twitter.util.{Await, Future} * val f: Future[Int] = Future.exception(new Exception("boom!")) @@ -1827,7 +1881,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * println("I'm being executed") // won't print * x + 10 * } - * Await.result(newf) // java.lang.Exception: boom! + * Await.result(newf) // throws java.lang.Exception: boom! * }}} * * @see [[flatMap]] for computations that return `Future`s. @@ -1847,7 +1901,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * * @note this should be used for side-effects. * - * For example: + * @example * $callbacks * {{{ * val a = Future.value(1) @@ -1876,7 +1930,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * `future.onFailure { case NonFatal(e) => ... }` when the Throwable * is "fatal". * - * For example: + * @example * $callbacks * {{{ * val b = Future.exception(new Exception("boom!")) @@ -1938,6 +1992,19 @@ abstract class Future[+A] extends Awaitable[A] { self => * * This is the equivalent of [[map]] for failed computations. * + * @example + * {{{ + * import com.twitter.util.{Await, Future} + * val f1: Future[Int] = Future.exception(new Exception("boom1!")) + * val f2: Future[Int] = Future.exception(new Exception("boom2!")) + * val newf: Future[Int] => Future[Int] = x => x.handle { + * case e: Exception if e.getMessage == "boom1!" => 1 + * } + * Await.result(newf(f1)) // 1 + * Await.result(newf(f2)) // throws java.lang.Exception: boom2! + * $awaitresult + * }}} + * * @see [[rescue]] */ def handle[B >: A](rescueException: PartialFunction[Throwable, B]): Future[B] = rescue { @@ -2179,30 +2246,34 @@ abstract class Future[+A] extends Awaitable[A] { self => /** * Returns the result of the computation as a `Future[Try[A]]`. - * For example: + * + * @example * {{{ * import com.twitter.util.{Await, Future, Try} - * val fr: Future[Int] = Future(1) + * val fr: Future[Int] = Future.value(1) * val ft: Future[Int] = Future.exception(new Exception("boom!")) * val r: Future[Try[Int]] = fr.liftToTry * val t: Future[Try[Int]] = ft.liftToTry * Await.result(r) // Return(1) * Await.result(t) // Throw(java.lang.Exception: boom!) + * $awaitresult * }}} */ def liftToTry: Future[Try[A]] = transformTry(Future.liftToTry) /** * Lowers a `Future[Try[T]]` into a `Future[T]`. - * For example: + * + * @example * {{{ * import com.twitter.util.{Await, Future, Return, Throw, Try} - * val fr: Future[Try[Int]] = Future(Return(1)) - * val ft: Future[Try[Int]] = Future(Throw(new Exception("boom!"))) + * val fr: Future[Try[Int]] = Future.value(Return(1)) + * val ft: Future[Try[Int]] = Future.value(Throw(new Exception("boom!"))) * val r: Future[Int] = fr.lowerFromTry * val t: Future[Int] = ft.lowerFromTry * Await.result(r) // 1 - * Await.result(t) // java.lang.Exception: boom! + * Await.result(t) // throws java.lang.Exception: boom! + * $awaitresult * }}} */ def lowerFromTry[B](implicit ev: A <:< Try[B]): Future[B] = From 52e9f6dd3a53f2bf9df36e1a8f1df2f4997e204b Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 17 Oct 2018 01:45:17 +0000 Subject: [PATCH 041/796] util-core: Switch to 1:1:1 pants targets Problem / Solution Due to recent commits util-core can now have finer grained pants build targets. JIRA Issues: CSL-7094 Differential Revision: https://phabricator.twitter.biz/D229241 --- util-core/src/main/java/BUILD | 22 +++++------------- .../main/java/com/twitter/concurrent/BUILD | 16 +++++++++++++ util-core/src/main/java/com/twitter/io/BUILD | 16 +++++++++++++ .../src/main/java/com/twitter/service/BUILD | 9 ++++++++ .../src/main/java/com/twitter/util/BUILD | 16 +++++++++++++ .../java/com/twitter/util/javainterop/BUILD | 14 +++++++++++ util-core/src/main/scala/BUILD | 19 +++++++++------ .../main/scala/com/twitter/concurrent/BUILD | 23 +++++++++++++++++++ .../main/scala/com/twitter/conversions/BUILD | 11 +++++++++ util-core/src/main/scala/com/twitter/io/BUILD | 15 ++++++++++++ .../src/main/scala/com/twitter/io/exp/BUILD | 12 ++++++++++ .../src/main/scala/com/twitter/util/BUILD | 23 +++++++++++++++++++ util-core/src/test/java/BUILD | 17 +++++--------- .../test/java/com/twitter/concurrent/BUILD | 10 ++++++++ util-core/src/test/java/com/twitter/io/BUILD | 10 ++++++++ .../src/test/java/com/twitter/service/BUILD | 8 +++++++ .../src/test/java/com/twitter/util/BUILD | 23 +++++++++++++++++++ .../java/com/twitter/util/javainterop/BUILD | 9 ++++++++ util-core/src/test/scala/BUILD | 23 +++++-------------- .../test/scala/com/twitter/concurrent/BUILD | 21 +++++++++++++++++ .../test/scala/com/twitter/conversions/BUILD | 18 +++++++++++++++ util-core/src/test/scala/com/twitter/io/BUILD | 20 ++++++++++++++++ .../src/test/scala/com/twitter/io/exp/BUILD | 19 +++++++++++++++ .../src/test/scala/com/twitter/util/BUILD | 21 +++++++++++++++++ 24 files changed, 344 insertions(+), 51 deletions(-) create mode 100644 util-core/src/main/java/com/twitter/concurrent/BUILD create mode 100644 util-core/src/main/java/com/twitter/io/BUILD create mode 100644 util-core/src/main/java/com/twitter/service/BUILD create mode 100644 util-core/src/main/java/com/twitter/util/BUILD create mode 100644 util-core/src/main/java/com/twitter/util/javainterop/BUILD create mode 100644 util-core/src/main/scala/com/twitter/concurrent/BUILD create mode 100644 util-core/src/main/scala/com/twitter/conversions/BUILD create mode 100644 util-core/src/main/scala/com/twitter/io/BUILD create mode 100644 util-core/src/main/scala/com/twitter/io/exp/BUILD create mode 100644 util-core/src/main/scala/com/twitter/util/BUILD create mode 100644 util-core/src/test/java/com/twitter/concurrent/BUILD create mode 100644 util-core/src/test/java/com/twitter/io/BUILD create mode 100644 util-core/src/test/java/com/twitter/service/BUILD create mode 100644 util-core/src/test/java/com/twitter/util/BUILD create mode 100644 util-core/src/test/java/com/twitter/util/javainterop/BUILD create mode 100644 util-core/src/test/scala/com/twitter/concurrent/BUILD create mode 100644 util-core/src/test/scala/com/twitter/conversions/BUILD create mode 100644 util-core/src/test/scala/com/twitter/io/BUILD create mode 100644 util-core/src/test/scala/com/twitter/io/exp/BUILD create mode 100644 util-core/src/test/scala/com/twitter/util/BUILD diff --git a/util-core/src/main/java/BUILD b/util-core/src/main/java/BUILD index 20ea336458..140c441eee 100644 --- a/util-core/src/main/java/BUILD +++ b/util-core/src/main/java/BUILD @@ -1,19 +1,9 @@ -java_library( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, - provides = artifact( - org = "com.twitter", - name = "util-core-java", - repo = artifactory, - ), +target( dependencies = [ - "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/scala", - "util/util-function/src/main/java", - ], - exports = [ - "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/scala", - "util/util-function/src/main/java", + "util/util-core/src/main/java/com/twitter/concurrent", + "util/util-core/src/main/java/com/twitter/io", + "util/util-core/src/main/java/com/twitter/service", + "util/util-core/src/main/java/com/twitter/util", + "util/util-core/src/main/java/com/twitter/util/javainterop", ], ) diff --git a/util-core/src/main/java/com/twitter/concurrent/BUILD b/util-core/src/main/java/com/twitter/concurrent/BUILD new file mode 100644 index 0000000000..04fa4aba73 --- /dev/null +++ b/util-core/src/main/java/com/twitter/concurrent/BUILD @@ -0,0 +1,16 @@ +java_library( + compiler_option_sets = {"fatal_warnings"}, + provides = artifact( + org = "com.twitter", + name = "util-core-java-concurrent", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/scala", + ], + exports = [ + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/scala", + ], +) diff --git a/util-core/src/main/java/com/twitter/io/BUILD b/util-core/src/main/java/com/twitter/io/BUILD new file mode 100644 index 0000000000..e2b53131c0 --- /dev/null +++ b/util-core/src/main/java/com/twitter/io/BUILD @@ -0,0 +1,16 @@ +java_library( + compiler_option_sets = {"fatal_warnings"}, + provides = artifact( + org = "com.twitter", + name = "util-core-java-io", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/scala", + ], + exports = [ + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/scala", + ], +) diff --git a/util-core/src/main/java/com/twitter/service/BUILD b/util-core/src/main/java/com/twitter/service/BUILD new file mode 100644 index 0000000000..45cc835b0c --- /dev/null +++ b/util-core/src/main/java/com/twitter/service/BUILD @@ -0,0 +1,9 @@ +java_library( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + provides = artifact( + org = "com.twitter", + name = "util-core-java-service", + repo = artifactory, + ), +) diff --git a/util-core/src/main/java/com/twitter/util/BUILD b/util-core/src/main/java/com/twitter/util/BUILD new file mode 100644 index 0000000000..b6bf59859c --- /dev/null +++ b/util-core/src/main/java/com/twitter/util/BUILD @@ -0,0 +1,16 @@ +java_library( + compiler_option_sets = {"fatal_warnings"}, + provides = artifact( + org = "com.twitter", + name = "util-core-java", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/scala", + ], + exports = [ + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/scala", + ], +) diff --git a/util-core/src/main/java/com/twitter/util/javainterop/BUILD b/util-core/src/main/java/com/twitter/util/javainterop/BUILD new file mode 100644 index 0000000000..f610b2f422 --- /dev/null +++ b/util-core/src/main/java/com/twitter/util/javainterop/BUILD @@ -0,0 +1,14 @@ +java_library( + compiler_option_sets = {"fatal_warnings"}, + provides = artifact( + org = "com.twitter", + name = "util-core-java-javainterop", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/scala-lang:scala-library", + ], + exports = [ + "3rdparty/jvm/org/scala-lang:scala-library", + ], +) diff --git a/util-core/src/main/scala/BUILD b/util-core/src/main/scala/BUILD index abb459964f..aa50e43055 100644 --- a/util-core/src/main/scala/BUILD +++ b/util-core/src/main/scala/BUILD @@ -1,18 +1,23 @@ scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, + # This target exists only to act as an aggregator jar. + sources = ["BUILD"], provides = scala_artifact( org = "com.twitter", name = "util-core", repo = artifactory, ), dependencies = [ - "3rdparty/jvm/org/scala-lang:scala-reflect", - "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", - "util/util-function/src/main/java", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-core/src/main/scala/com/twitter/io/exp", + "util/util-core/src/main/scala/com/twitter/util", ], exports = [ - "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", - "util/util-function/src/main/java", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-core/src/main/scala/com/twitter/io/exp", + "util/util-core/src/main/scala/com/twitter/util", ], ) diff --git a/util-core/src/main/scala/com/twitter/concurrent/BUILD b/util-core/src/main/scala/com/twitter/concurrent/BUILD new file mode 100644 index 0000000000..ceb2181ccb --- /dev/null +++ b/util-core/src/main/scala/com/twitter/concurrent/BUILD @@ -0,0 +1,23 @@ +scala_library( + sources = globs( + "*.scala", + exclude = [ + # the com/twitter/util dependency already includes these files + globs( + "NamedPoolThreadFactory.scala", + "Offer.scala", + "Scheduler.scala", + "Tx.scala", + ), + ], + ), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-core-concurrent", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-core/src/main/scala/com/twitter/conversions/BUILD b/util-core/src/main/scala/com/twitter/conversions/BUILD new file mode 100644 index 0000000000..3684ab4eed --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/BUILD @@ -0,0 +1,11 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-core-conversions", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-core/src/main/scala/com/twitter/io/BUILD b/util-core/src/main/scala/com/twitter/io/BUILD new file mode 100644 index 0000000000..5bdb5e3611 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/BUILD @@ -0,0 +1,15 @@ +scala_library( + sources = globs( + "*.scala", + ), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-core-io", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-core/src/main/scala/com/twitter/io/exp/BUILD b/util-core/src/main/scala/com/twitter/io/exp/BUILD new file mode 100644 index 0000000000..d972e0c479 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/exp/BUILD @@ -0,0 +1,12 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-core-io-exp", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/io", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-core/src/main/scala/com/twitter/util/BUILD b/util-core/src/main/scala/com/twitter/util/BUILD new file mode 100644 index 0000000000..8bcc4fd97c --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/BUILD @@ -0,0 +1,23 @@ +scala_library( + sources = globs( + "*.scala", + "../concurrent/NamedPoolThreadFactory.scala", + "../concurrent/Offer.scala", + "../concurrent/Scheduler.scala", + "../concurrent/Tx.scala", + ), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-core-util", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", + "util/util-function/src/main/java", + ], + exports = [ + "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", + "util/util-function/src/main/java", + ], +) diff --git a/util-core/src/test/java/BUILD b/util-core/src/test/java/BUILD index 5bdeb19127..c99528cb20 100644 --- a/util-core/src/test/java/BUILD +++ b/util-core/src/test/java/BUILD @@ -1,14 +1,9 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/com/github/ben-manes/caffeine", - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scala-lang:scala-library", - "3rdparty/jvm/org/scalacheck", - "util/util-core/src/main/java", - "util/util-core/src/main/scala", - "util/util-function/src/main/java", + "util/util-core/src/test/java/com/twitter/concurrent", + "util/util-core/src/test/java/com/twitter/io", + "util/util-core/src/test/java/com/twitter/service", + "util/util-core/src/test/java/com/twitter/util", + "util/util-core/src/test/java/com/twitter/util/javainterop", ], ) diff --git a/util-core/src/test/java/com/twitter/concurrent/BUILD b/util-core/src/test/java/com/twitter/concurrent/BUILD new file mode 100644 index 0000000000..809d672c40 --- /dev/null +++ b/util-core/src/test/java/com/twitter/concurrent/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/java/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/concurrent", + ], +) diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD new file mode 100644 index 0000000000..bbb0aceff9 --- /dev/null +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/java/com/twitter/io", + "util/util-core/src/main/scala/com/twitter/io", + ], +) diff --git a/util-core/src/test/java/com/twitter/service/BUILD b/util-core/src/test/java/com/twitter/service/BUILD new file mode 100644 index 0000000000..bb47e03bf5 --- /dev/null +++ b/util-core/src/test/java/com/twitter/service/BUILD @@ -0,0 +1,8 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "util/util-core/src/main/java/com/twitter/service", + ], +) diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD new file mode 100644 index 0000000000..bee651849b --- /dev/null +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -0,0 +1,23 @@ +java_library( + name = "object_size_calculator", + sources = globs("ObjectSizeCalculator.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/com/github/ben-manes/caffeine", + ], +) + +junit_tests( + sources = globs( + "*.java", + exclude = [globs("ObjectSizeCalculator.java")], + ), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + ":object_size_calculator", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/java/com/twitter/util", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-core/src/test/java/com/twitter/util/javainterop/BUILD b/util-core/src/test/java/com/twitter/util/javainterop/BUILD new file mode 100644 index 0000000000..c66493ec25 --- /dev/null +++ b/util-core/src/test/java/com/twitter/util/javainterop/BUILD @@ -0,0 +1,9 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core/src/main/java/com/twitter/util/javainterop", + ], +) diff --git a/util-core/src/test/scala/BUILD b/util-core/src/test/scala/BUILD index 3f0d575739..ca90513fde 100644 --- a/util-core/src/test/scala/BUILD +++ b/util-core/src/test/scala/BUILD @@ -1,20 +1,9 @@ -junit_tests( - sources = rglobs("*.scala"), - # fatal_warnings runs into an issue with scalatest/Future.isDone: - # - # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` - # [warn] assert(future.close().isDone) - # [warn] ^ - # - # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` - # for this target, we could try enabling this again. - compiler_option_sets = {}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalacheck", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala", - "util/util-core/src/test/java", + "util/util-core/src/test/scala/com/twitter/concurrent", + "util/util-core/src/test/scala/com/twitter/conversions", + "util/util-core/src/test/scala/com/twitter/io", + "util/util-core/src/test/scala/com/twitter/io/exp", + "util/util-core/src/test/scala/com/twitter/util", ], ) diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD new file mode 100644 index 0000000000..6808830495 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -0,0 +1,21 @@ +junit_tests( + sources = globs("*.scala"), + # fatal_warnings runs into an issue with scalatest/Future.isDone: + # + # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` + # [warn] assert(future.close().isDone) + # [warn] ^ + # + # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` + # for this target, we could try enabling this again. + compiler_option_sets = {}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/test/java/com/twitter/util:object_size_calculator", + ], +) diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD new file mode 100644 index 0000000000..6ba685bd61 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -0,0 +1,18 @@ +junit_tests( + sources = globs("*.scala"), + # fatal_warnings runs into an issue with scalatest/Future.isDone: + # + # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` + # [warn] assert(future.close().isDone) + # [warn] ^ + # + # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` + # for this target, we could try enabling this again. + compiler_option_sets = {}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/conversions", + ], +) diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD new file mode 100644 index 0000000000..88d823c101 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -0,0 +1,20 @@ +junit_tests( + sources = globs("*.scala"), + # fatal_warnings runs into an issue with scalatest/Future.isDone: + # + # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` + # [warn] assert(future.close().isDone) + # [warn] ^ + # + # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` + # for this target, we could try enabling this again. + compiler_option_sets = {}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io", + ], +) diff --git a/util-core/src/test/scala/com/twitter/io/exp/BUILD b/util-core/src/test/scala/com/twitter/io/exp/BUILD new file mode 100644 index 0000000000..2e5443914c --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/exp/BUILD @@ -0,0 +1,19 @@ +junit_tests( + sources = globs("*.scala"), + # fatal_warnings runs into an issue with scalatest/Future.isDone: + # + # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` + # [warn] assert(future.close().isDone) + # [warn] ^ + # + # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` + # for this target, we could try enabling this again. + compiler_option_sets = {}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io/exp", + ], +) diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD new file mode 100644 index 0000000000..3d2ebe9e00 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -0,0 +1,21 @@ +junit_tests( + sources = globs("*.scala"), + # fatal_warnings runs into an issue with scalatest/Future.isDone: + # + # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` + # [warn] assert(future.close().isDone) + # [warn] ^ + # + # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` + # for this target, we could try enabling this again. + compiler_option_sets = {}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-core/src/test/java/com/twitter/util:object_size_calculator", + ], +) From f1c86b43267c81ba47afe013a88592079c3f553e Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Thu, 18 Oct 2018 01:30:50 +0000 Subject: [PATCH 042/796] util: Migrate more projects to pants 1:1:1 Problem We'd like our modules to follow pants 1:1:1 style. Solution Update more modules to use that. Result Migrated: - util-app - util-collection - util-function - util-registry - util-tunable JIRA Issues: CSL-7094 Differential Revision: https://phabricator.twitter.biz/D229840 --- util-app/src/main/java/BUILD | 15 ++++-------- util-app/src/main/java/com/twitter/app/BUILD | 10 ++++++++ util-app/src/main/scala/BUILD | 18 +++------------ util-app/src/main/scala/com/twitter/app/BUILD | 23 +++++++++++++++++++ util-app/src/test/java/BUILD | 11 ++------- util-app/src/test/java/com/twitter/app/BUILD | 12 ++++++++++ util-app/src/test/scala/BUILD | 12 ++-------- util-app/src/test/scala/com/twitter/app/BUILD | 13 +++++++++++ util-collection/src/main/scala/BUILD | 10 ++++---- .../main/scala/com/twitter/collection/BUILD | 14 +++++++++++ .../src/main/scala/com/twitter/util/BUILD | 8 +++++++ util-collection/src/test/scala/BUILD | 12 +++------- .../test/scala/com/twitter/collection/BUILD | 11 +++++++++ .../src/test/scala/com/twitter/util/BUILD | 10 ++++++++ util-function/src/main/java/BUILD | 12 ++++------ .../src/main/java/com/twitter/function/BUILD | 8 +++++++ util-registry/src/main/scala/BUILD | 11 ++------- .../scala/com/twitter/util/registry/BUILD | 11 +++++++++ util-registry/src/test/java/BUILD | 8 ++----- .../test/java/com/twitter/util/registry/BUILD | 9 ++++++++ util-registry/src/test/scala/BUILD | 9 ++------ .../scala/com/twitter/util/registry/BUILD | 10 ++++++++ .../src/main/scala/com/twitter/logging/BUILD | 16 +++++++++++++ util-tunable/src/main/scala/BUILD | 22 ++---------------- .../main/scala/com/twitter/util/tunable/BUILD | 19 +++++++++++++++ .../com/twitter/util/tunable/linter/BUILD | 7 +++--- .../test/java/com/twitter/util/tunable/BUILD | 9 ++++++++ util-tunable/src/test/scala/BUILD | 14 ++--------- .../test/scala/com/twitter/util/tunable/BUILD | 15 ++++++++++++ 29 files changed, 236 insertions(+), 123 deletions(-) create mode 100644 util-app/src/main/java/com/twitter/app/BUILD create mode 100644 util-app/src/main/scala/com/twitter/app/BUILD create mode 100644 util-app/src/test/java/com/twitter/app/BUILD create mode 100644 util-app/src/test/scala/com/twitter/app/BUILD create mode 100644 util-collection/src/main/scala/com/twitter/collection/BUILD create mode 100644 util-collection/src/main/scala/com/twitter/util/BUILD create mode 100644 util-collection/src/test/scala/com/twitter/collection/BUILD create mode 100644 util-collection/src/test/scala/com/twitter/util/BUILD create mode 100644 util-function/src/main/java/com/twitter/function/BUILD create mode 100644 util-registry/src/main/scala/com/twitter/util/registry/BUILD create mode 100644 util-registry/src/test/java/com/twitter/util/registry/BUILD create mode 100644 util-registry/src/test/scala/com/twitter/util/registry/BUILD create mode 100644 util-test/src/main/scala/com/twitter/logging/BUILD create mode 100644 util-tunable/src/main/scala/com/twitter/util/tunable/BUILD create mode 100644 util-tunable/src/test/java/com/twitter/util/tunable/BUILD create mode 100644 util-tunable/src/test/scala/com/twitter/util/tunable/BUILD diff --git a/util-app/src/main/java/BUILD b/util-app/src/main/java/BUILD index 7dd64b3ad9..f7e60703a5 100644 --- a/util-app/src/main/java/BUILD +++ b/util-app/src/main/java/BUILD @@ -1,11 +1,6 @@ -java_library( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, - provides = artifact( - org = "com.twitter", - name = "util-app-java", - repo = artifactory, - ), - # Not detected as required by zinc. - scope = "forced", +target( + dependencies = [ + "util/util-app/src/main/java/com/twitter/app", + "util/util-app/src/main/scala/com/twitter/app", + ], ) diff --git a/util-app/src/main/java/com/twitter/app/BUILD b/util-app/src/main/java/com/twitter/app/BUILD new file mode 100644 index 0000000000..edb0841aab --- /dev/null +++ b/util-app/src/main/java/com/twitter/app/BUILD @@ -0,0 +1,10 @@ +java_library( + compiler_option_sets = {"fatal_warnings"}, + provides = artifact( + org = "com.twitter", + name = "util-app-java", + repo = artifactory, + ), + # Not detected as required by zinc. + scope = "forced", +) diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index 451b5d05fa..f7e60703a5 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -1,18 +1,6 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-app", - repo = artifactory, - ), +target( dependencies = [ - "util/util-app/src/main/java", - "util/util-core/src/main/scala", - "util/util-registry/src/main/scala", - ], - exports = [ - "util/util-app/src/main/java", - "util/util-core/src/main/scala", + "util/util-app/src/main/java/com/twitter/app", + "util/util-app/src/main/scala/com/twitter/app", ], ) diff --git a/util-app/src/main/scala/com/twitter/app/BUILD b/util-app/src/main/scala/com/twitter/app/BUILD new file mode 100644 index 0000000000..8f4a5ab625 --- /dev/null +++ b/util-app/src/main/scala/com/twitter/app/BUILD @@ -0,0 +1,23 @@ +scala_library( + sources = globs( + "*.scala", + "../finagle/util/flags.scala", + ), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-app", + repo = artifactory, + ), + dependencies = [ + "util/util-app/src/main/java/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-registry/src/main/scala/com/twitter/util/registry", + ], + exports = [ + "util/util-app/src/main/java/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-app/src/test/java/BUILD b/util-app/src/test/java/BUILD index 84ed3d60db..6edff9df68 100644 --- a/util-app/src/test/java/BUILD +++ b/util-app/src/test/java/BUILD @@ -1,12 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-app/src/main/java", - "util/util-app/src/main/scala", - "util/util-core/src/main/scala", - "util/util-function/src/main/java", + "util/util-app/src/test/java/com/twitter/app", ], ) diff --git a/util-app/src/test/java/com/twitter/app/BUILD b/util-app/src/test/java/com/twitter/app/BUILD new file mode 100644 index 0000000000..6fbe02072d --- /dev/null +++ b/util-app/src/test/java/com/twitter/app/BUILD @@ -0,0 +1,12 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-app/src/main/java/com/twitter/app", + "util/util-app/src/main/scala/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-function/src/main/java/com/twitter/function", + ], +) diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index 9cbb7f85a3..6e3fe3ea41 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -1,13 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-app/src/main/scala", - "util/util-app/src/test/resources", - "util/util-core/src/main/scala", - "util/util-registry/src/main/scala", + "util/util-app/src/test/scala/com/twitter/app", ], ) diff --git a/util-app/src/test/scala/com/twitter/app/BUILD b/util-app/src/test/scala/com/twitter/app/BUILD new file mode 100644 index 0000000000..1f4e7e0faa --- /dev/null +++ b/util-app/src/test/scala/com/twitter/app/BUILD @@ -0,0 +1,13 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/test/resources", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-registry/src/main/scala/com/twitter/util/registry", + ], +) diff --git a/util-collection/src/main/scala/BUILD b/util-collection/src/main/scala/BUILD index cfe7877044..7bf3e97600 100644 --- a/util-collection/src/main/scala/BUILD +++ b/util-collection/src/main/scala/BUILD @@ -1,15 +1,17 @@ scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, + # This target exists only to act as an aggregator jar. + sources = ["BUILD"], provides = scala_artifact( org = "com.twitter", name = "util-collection", repo = artifactory, ), dependencies = [ - "util/util-core/src/main/scala", + "util/util-collection/src/main/scala/com/twitter/collection", + "util/util-collection/src/main/scala/com/twitter/util", ], exports = [ - "util/util-core/src/main/scala", + "util/util-collection/src/main/scala/com/twitter/collection", + "util/util-collection/src/main/scala/com/twitter/util", ], ) diff --git a/util-collection/src/main/scala/com/twitter/collection/BUILD b/util-collection/src/main/scala/com/twitter/collection/BUILD new file mode 100644 index 0000000000..b615a1425b --- /dev/null +++ b/util-collection/src/main/scala/com/twitter/collection/BUILD @@ -0,0 +1,14 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-collection-collection", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/util", + ], + exports = [ + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-collection/src/main/scala/com/twitter/util/BUILD b/util-collection/src/main/scala/com/twitter/util/BUILD new file mode 100644 index 0000000000..e44adb8f25 --- /dev/null +++ b/util-collection/src/main/scala/com/twitter/util/BUILD @@ -0,0 +1,8 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-collection-util", + repo = artifactory, + ), +) diff --git a/util-collection/src/test/scala/BUILD b/util-collection/src/test/scala/BUILD index a6e49c881d..aa0cffd6bd 100644 --- a/util-collection/src/test/scala/BUILD +++ b/util-collection/src/test/scala/BUILD @@ -1,12 +1,6 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalacheck", - "3rdparty/jvm/org/scalatest", - "util/util-collection/src/main/scala", - "util/util-core/src/main/scala", + "util/util-collection/src/test/scala/com/twitter/collection", + "util/util-collection/src/test/scala/com/twitter/util", ], ) diff --git a/util-collection/src/test/scala/com/twitter/collection/BUILD b/util-collection/src/test/scala/com/twitter/collection/BUILD new file mode 100644 index 0000000000..6c4fc61ea8 --- /dev/null +++ b/util-collection/src/test/scala/com/twitter/collection/BUILD @@ -0,0 +1,11 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "util/util-collection/src/main/scala/com/twitter/collection", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-collection/src/test/scala/com/twitter/util/BUILD b/util-collection/src/test/scala/com/twitter/util/BUILD new file mode 100644 index 0000000000..2ce910c542 --- /dev/null +++ b/util-collection/src/test/scala/com/twitter/util/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "util/util-collection/src/main/scala/com/twitter/util", + ], +) diff --git a/util-function/src/main/java/BUILD b/util-function/src/main/java/BUILD index 02d4f368a2..96528c9177 100644 --- a/util-function/src/main/java/BUILD +++ b/util-function/src/main/java/BUILD @@ -1,9 +1,5 @@ -java_library( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, - provides = artifact( - org = "com.twitter", - name = "util-function-java", - repo = artifactory, - ), +target( + dependencies = [ + "util/util-function/src/main/java/com/twitter/function", + ], ) diff --git a/util-function/src/main/java/com/twitter/function/BUILD b/util-function/src/main/java/com/twitter/function/BUILD new file mode 100644 index 0000000000..70397fbe4d --- /dev/null +++ b/util-function/src/main/java/com/twitter/function/BUILD @@ -0,0 +1,8 @@ +java_library( + compiler_option_sets = {"fatal_warnings"}, + provides = artifact( + org = "com.twitter", + name = "util-function-java", + repo = artifactory, + ), +) diff --git a/util-registry/src/main/scala/BUILD b/util-registry/src/main/scala/BUILD index 52865c8e2f..6c76327ab4 100644 --- a/util-registry/src/main/scala/BUILD +++ b/util-registry/src/main/scala/BUILD @@ -1,12 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-registry", - repo = artifactory, - ), +target( dependencies = [ - "util/util-core/src/main/scala", + "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-registry/src/main/scala/com/twitter/util/registry/BUILD b/util-registry/src/main/scala/com/twitter/util/registry/BUILD new file mode 100644 index 0000000000..e2d19187f2 --- /dev/null +++ b/util-registry/src/main/scala/com/twitter/util/registry/BUILD @@ -0,0 +1,11 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-registry", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-registry/src/test/java/BUILD b/util-registry/src/test/java/BUILD index c958405e33..92f3a26ae8 100644 --- a/util-registry/src/test/java/BUILD +++ b/util-registry/src/test/java/BUILD @@ -1,9 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-registry/src/main/scala", + "util/util-registry/src/test/java/com/twitter/util/registry", ], ) diff --git a/util-registry/src/test/java/com/twitter/util/registry/BUILD b/util-registry/src/test/java/com/twitter/util/registry/BUILD new file mode 100644 index 0000000000..20a7de6cba --- /dev/null +++ b/util-registry/src/test/java/com/twitter/util/registry/BUILD @@ -0,0 +1,9 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-registry/src/main/scala/com/twitter/util/registry", + ], +) diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index 9f47327643..03e8040f41 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -1,10 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-registry/src/main/scala", + "util/util-registry/src/test/scala/com/twitter/util/registry", ], ) diff --git a/util-registry/src/test/scala/com/twitter/util/registry/BUILD b/util-registry/src/test/scala/com/twitter/util/registry/BUILD new file mode 100644 index 0000000000..3867a21cf1 --- /dev/null +++ b/util-registry/src/test/scala/com/twitter/util/registry/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-registry/src/main/scala/com/twitter/util/registry", + ], +) diff --git a/util-test/src/main/scala/com/twitter/logging/BUILD b/util-test/src/main/scala/com/twitter/logging/BUILD new file mode 100644 index 0000000000..ccec239edc --- /dev/null +++ b/util-test/src/main/scala/com/twitter/logging/BUILD @@ -0,0 +1,16 @@ +scala_library( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-test-logging", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/scalatest", + "util/util-logging/src/main/scala", + ], + exports = [ + "3rdparty/jvm/org/scalatest", + ], +) diff --git a/util-tunable/src/main/scala/BUILD b/util-tunable/src/main/scala/BUILD index e5cd7f0aca..32fc36f7d1 100644 --- a/util-tunable/src/main/scala/BUILD +++ b/util-tunable/src/main/scala/BUILD @@ -1,23 +1,5 @@ -scala_library( - name = "scala", - sources = rglobs( - "*.scala", - exclude = [rglobs("util/tunable/linter/*")], - ), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-tunable", - repo = artifactory, - ), +target( dependencies = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", - "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", - "util/util-app/src/main/scala", - "util/util-core/src/main/scala", - ], - exports = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "util/util-tunable/src/main/scala/com/twitter/util/tunable", ], ) diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD new file mode 100644 index 0000000000..03c25bf40f --- /dev/null +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD @@ -0,0 +1,19 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-tunable", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + "util/util-app/src/main/scala/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/util", + ], + exports = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + ], +) diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD index 6a83f5e3de..a26d6ad8ed 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD @@ -1,13 +1,12 @@ scala_library( name = "scala", - sources = rglobs("*.scala"), dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", - "util/util-app/src/main/scala", - "util/util-core/src/main/scala", - "util/util-tunable/src/main/scala", + "util/util-app/src/main/scala/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-tunable/src/main/scala/com/twitter/util/tunable", ], ) diff --git a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD new file mode 100644 index 0000000000..b34dd76ac5 --- /dev/null +++ b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD @@ -0,0 +1,9 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-tunable/src/main/scala/com/twitter/util/tunable", + ], +) diff --git a/util-tunable/src/test/scala/BUILD b/util-tunable/src/test/scala/BUILD index 049d36ab76..5a6a16dd85 100644 --- a/util-tunable/src/test/scala/BUILD +++ b/util-tunable/src/test/scala/BUILD @@ -1,15 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", - "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalacheck", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala", - "util/util-tunable/src/main/scala", - "util/util-tunable/src/test/resources", + "util/util-tunable/src/test/scala/com/twitter/util/tunable", ], ) diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD new file mode 100644 index 0000000000..b6f84bf142 --- /dev/null +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -0,0 +1,15 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-tunable/src/main/scala/com/twitter/util/tunable", + "util/util-tunable/src/test/resources", + ], +) From 9f6d47c9a37d7448c02493967f3151c7ba465f31 Mon Sep 17 00:00:00 2001 From: Jordan Parker Date: Thu, 18 Oct 2018 23:28:58 +0000 Subject: [PATCH 043/796] Twitter-oss: Prepare OSS libraries for release 18.10.0 Problem We want to release the next versions of our Twitter OSS libraries 18.10.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-7105 Differential Revision: https://phabricator.twitter.biz/D230664 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 349adf5159..341a47f0eb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.10.0 +------- + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index bebe09a8fd..b331c1748c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "18.9.1" +val collUtils = "com.twitter" %% "util-collection" % "18.10.0" ``` # Units diff --git a/build.sbt b/build.sbt index 694d8be5ea..f04098d126 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.10.0-SNAPSHOT" +val releaseVersion = "18.10.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 1a8131a6252985c1805edb71495d443d84cf7c7a Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Fri, 19 Oct 2018 03:34:04 +0000 Subject: [PATCH 044/796] util: Migrate more projects to pants 1:1:1 Problem We'd like our modules to follow pants 1:1:1 style. Solution Update more modules to use that. Result Migrated: - util-cache - util-cache-guava - util-codec - util-hashing - util-intellij - util-jvm - util-lint - util-stats JIRA Issues: CSL-7094 TBR=true Differential Revision: https://phabricator.twitter.biz/D230507 --- util-cache-guava/src/main/scala/BUILD | 13 +-- .../main/scala/com/twitter/cache/guava/BUILD | 13 +++ util-cache-guava/src/test/java/BUILD | 12 +-- .../test/java/com/twitter/cache/guava/BUILD | 12 +++ util-cache-guava/src/test/scala/BUILD | 14 +-- .../test/scala/com/twitter/cache/guava/BUILD | 14 +++ util-cache/src/main/scala/BUILD | 13 ++- .../src/main/scala/com/twitter/cache/BUILD | 11 +++ .../scala/com/twitter/cache/caffeine/BUILD | 13 +++ util-cache/src/test/java/BUILD | 11 +-- .../src/test/java/com/twitter/cache/BUILD | 11 +++ util-cache/src/test/scala/BUILD | 28 +----- .../src/test/scala/com/twitter/cache/BUILD | 38 ++++++++ .../scala/com/twitter/cache/caffeine/BUILD | 14 +++ util-codec/src/main/scala/BUILD | 11 +-- .../src/main/scala/com/twitter/util/BUILD | 11 +++ util-codec/src/test/scala/BUILD | 9 +- .../src/test/scala/com/twitter/util/BUILD | 9 ++ util-hashing/src/main/scala/BUILD | 12 +-- .../src/main/scala/com/twitter/hashing/BUILD | 8 ++ util-hashing/src/test/scala/BUILD | 12 +-- .../src/test/scala/com/twitter/hashing/BUILD | 12 +++ .../com/twitter/util/capturepoints/BUILD | 2 +- util-jvm/src/main/scala/BUILD | 16 +--- util-jvm/src/main/scala/com/twitter/jvm/BUILD | 17 ++++ util-jvm/src/test/java/BUILD | 7 +- util-jvm/src/test/java/com/twitter/jvm/BUILD | 8 ++ util-jvm/src/test/scala/BUILD | 21 +---- util-jvm/src/test/scala/com/twitter/jvm/BUILD | 36 +++++++ .../scala/com/twitter/jvm/EstimatorApp.scala | 92 ++++++++++++++++++ .../scala/com/twitter/jvm/EstimatorTest.scala | 94 ------------------- util-lint/src/main/scala/BUILD | 11 +-- .../main/scala/com/twitter/util/lint/BUILD | 11 +++ util-lint/src/test/scala/BUILD | 9 +- .../test/scala/com/twitter/util/lint/BUILD | 10 ++ util-stats/src/main/scala/BUILD | 17 +--- .../scala/com/twitter/finagle/stats/BUILD | 17 ++++ util-stats/src/test/java/BUILD | 8 +- .../test/java/com/twitter/finagle/stats/BUILD | 9 ++ util-stats/src/test/scala/BUILD | 11 +-- .../scala/com/twitter/finagle/stats/BUILD | 13 +++ 41 files changed, 425 insertions(+), 285 deletions(-) create mode 100644 util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD create mode 100644 util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD create mode 100644 util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD create mode 100644 util-cache/src/main/scala/com/twitter/cache/BUILD create mode 100644 util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD create mode 100644 util-cache/src/test/java/com/twitter/cache/BUILD create mode 100644 util-cache/src/test/scala/com/twitter/cache/BUILD create mode 100644 util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD create mode 100644 util-codec/src/main/scala/com/twitter/util/BUILD create mode 100644 util-codec/src/test/scala/com/twitter/util/BUILD create mode 100644 util-hashing/src/main/scala/com/twitter/hashing/BUILD create mode 100644 util-hashing/src/test/scala/com/twitter/hashing/BUILD create mode 100644 util-jvm/src/main/scala/com/twitter/jvm/BUILD create mode 100644 util-jvm/src/test/java/com/twitter/jvm/BUILD create mode 100644 util-jvm/src/test/scala/com/twitter/jvm/BUILD create mode 100644 util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala create mode 100644 util-lint/src/main/scala/com/twitter/util/lint/BUILD create mode 100644 util-lint/src/test/scala/com/twitter/util/lint/BUILD create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/BUILD create mode 100644 util-stats/src/test/java/com/twitter/finagle/stats/BUILD create mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/BUILD diff --git a/util-cache-guava/src/main/scala/BUILD b/util-cache-guava/src/main/scala/BUILD index 47bcf67204..9be4964b4a 100644 --- a/util-cache-guava/src/main/scala/BUILD +++ b/util-cache-guava/src/main/scala/BUILD @@ -1,14 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-cache-guava", - repo = artifactory, - ), +target( dependencies = [ - "3rdparty/jvm/com/google/guava", - "util/util-cache/src/main/scala", - "util/util-core/src/main/scala", + "util/util-cache-guava/src/main/scala/com/twitter/cache/guava", ], ) diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD new file mode 100644 index 0000000000..a2be9cee82 --- /dev/null +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD @@ -0,0 +1,13 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-cache-guava", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/com/google/guava", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-cache-guava/src/test/java/BUILD b/util-cache-guava/src/test/java/BUILD index 12372efcc7..76b5264c8b 100644 --- a/util-cache-guava/src/test/java/BUILD +++ b/util-cache-guava/src/test/java/BUILD @@ -1,13 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/com/google/guava", - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-cache", - "util/util-cache-guava/src/main/scala", - "util/util-core/src/main/scala", + "util/util-cache-guava/src/test/java/com/twitter/cache/guava", ], ) diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD new file mode 100644 index 0000000000..73382d8c71 --- /dev/null +++ b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD @@ -0,0 +1,12 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/com/google/guava", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-cache-guava/src/main/scala/com/twitter/cache/guava", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-cache-guava/src/test/scala/BUILD b/util-cache-guava/src/test/scala/BUILD index 3ca6340ec2..2f54421711 100644 --- a/util-cache-guava/src/test/scala/BUILD +++ b/util-cache-guava/src/test/scala/BUILD @@ -1,15 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/com/google/code/findbugs:jsr305", - "3rdparty/jvm/com/google/guava", - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-cache-guava/src/main/scala", - "util/util-cache/src/main/scala", - "util/util-cache/src/test/scala:abstract_tests", - "util/util-core/src/main/scala", + "util/util-cache-guava/src/test/scala/com/twitter/cache/guava", ], ) diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD new file mode 100644 index 0000000000..6c589a5dc6 --- /dev/null +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD @@ -0,0 +1,14 @@ +junit_tests( + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/com/google/code/findbugs:jsr305", + "3rdparty/jvm/com/google/guava", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "util/util-cache-guava/src/main/scala/com/twitter/cache/guava", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-cache/src/main/scala/BUILD b/util-cache/src/main/scala/BUILD index 12b85099f2..77874242b5 100644 --- a/util-cache/src/main/scala/BUILD +++ b/util-cache/src/main/scala/BUILD @@ -1,16 +1,19 @@ scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, + # This target exists only to act as an aggregator jar. + sources = ["BUILD"], provides = scala_artifact( org = "com.twitter", name = "util-cache", repo = artifactory, ), dependencies = [ - "3rdparty/jvm/com/github/ben-manes/caffeine", - "util/util-core/src/main/scala", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-cache/src/main/scala/com/twitter/cache/caffeine", + "util/util-core/src/main/scala/com/twitter/util", ], exports = [ - "util/util-core/src/main/scala", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-cache/src/main/scala/com/twitter/cache/caffeine", + "util/util-core/src/main/scala/com/twitter/util", ], ) diff --git a/util-cache/src/main/scala/com/twitter/cache/BUILD b/util-cache/src/main/scala/com/twitter/cache/BUILD new file mode 100644 index 0000000000..bc4e13b68f --- /dev/null +++ b/util-cache/src/main/scala/com/twitter/cache/BUILD @@ -0,0 +1,11 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-cache-cache", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD new file mode 100644 index 0000000000..81597a3ee8 --- /dev/null +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD @@ -0,0 +1,13 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-cache-caffeine", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/com/github/ben-manes/caffeine", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-cache/src/test/java/BUILD b/util-cache/src/test/java/BUILD index a1ef1beb6b..97107dac55 100644 --- a/util-cache/src/test/java/BUILD +++ b/util-cache/src/test/java/BUILD @@ -1,12 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/com/google/guava", - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-cache", - "util/util-core/src/main/scala", + "util/util-cache/src/test/java/com/twitter/cache", ], ) diff --git a/util-cache/src/test/java/com/twitter/cache/BUILD b/util-cache/src/test/java/com/twitter/cache/BUILD new file mode 100644 index 0000000000..a67715fc2e --- /dev/null +++ b/util-cache/src/test/java/com/twitter/cache/BUILD @@ -0,0 +1,11 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-cache/src/test/scala/BUILD b/util-cache/src/test/scala/BUILD index 08ba112d36..dee129b504 100644 --- a/util-cache/src/test/scala/BUILD +++ b/util-cache/src/test/scala/BUILD @@ -1,28 +1,6 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/com/github/ben-manes/caffeine", - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-cache", - "util/util-core/src/main/scala", - ], -) - -scala_library( - name = "abstract_tests", - sources = [ - "com/twitter/cache/AbstractFutureCacheTest.scala", - "com/twitter/cache/AbstractLoadingFutureCacheTest.scala", - ], - compiler_option_sets = {"fatal_warnings"}, - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-cache", - "util/util-core/src/main/scala", + "util/util-cache/src/test/scala/com/twitter/cache", + "util/util-cache/src/test/scala/com/twitter/cache/caffeine", ], ) diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD new file mode 100644 index 0000000000..f6ac12a6b4 --- /dev/null +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -0,0 +1,38 @@ +junit_tests( + sources = globs( + "*.scala", + exclude = [ + globs( + "AbstractFutureCacheTest.scala", + "AbstractLoadingFutureCacheTest.scala", + ), + ], + ), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + ":abstract_tests", + "3rdparty/jvm/com/github/ben-manes/caffeine", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-cache/src/main/scala/com/twitter/cache/caffeine", + "util/util-core/src/main/scala/com/twitter/util", + ], +) + +scala_library( + name = "abstract_tests", + sources = [ + "AbstractFutureCacheTest.scala", + "AbstractLoadingFutureCacheTest.scala", + ], + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD new file mode 100644 index 0000000000..c56d0d4896 --- /dev/null +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -0,0 +1,14 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/com/github/ben-manes/caffeine", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-cache/src/main/scala/com/twitter/cache/caffeine", + "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-codec/src/main/scala/BUILD b/util-codec/src/main/scala/BUILD index 8655e57304..6d8306a0d3 100644 --- a/util-codec/src/main/scala/BUILD +++ b/util-codec/src/main/scala/BUILD @@ -1,12 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-codec", - repo = artifactory, - ), +target( dependencies = [ - "util/util-core/src/main/scala", + "util/util-codec/src/main/scala/com/twitter/util", ], ) diff --git a/util-codec/src/main/scala/com/twitter/util/BUILD b/util-codec/src/main/scala/com/twitter/util/BUILD new file mode 100644 index 0000000000..74e11682df --- /dev/null +++ b/util-codec/src/main/scala/com/twitter/util/BUILD @@ -0,0 +1,11 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-codec", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/io", + ], +) diff --git a/util-codec/src/test/scala/BUILD b/util-codec/src/test/scala/BUILD index 077f627e3c..b4c871f06b 100644 --- a/util-codec/src/test/scala/BUILD +++ b/util-codec/src/test/scala/BUILD @@ -1,10 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-codec/src/main/scala", + "util/util-codec/src/test/scala/com/twitter/util", ], ) diff --git a/util-codec/src/test/scala/com/twitter/util/BUILD b/util-codec/src/test/scala/com/twitter/util/BUILD new file mode 100644 index 0000000000..cf725c0052 --- /dev/null +++ b/util-codec/src/test/scala/com/twitter/util/BUILD @@ -0,0 +1,9 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "util/util-codec/src/main/scala/com/twitter/util", + ], +) diff --git a/util-hashing/src/main/scala/BUILD b/util-hashing/src/main/scala/BUILD index 3c18e8017c..47781f687f 100644 --- a/util-hashing/src/main/scala/BUILD +++ b/util-hashing/src/main/scala/BUILD @@ -1,9 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-hashing", - repo = artifactory, - ), +target( + dependencies = [ + "util/util-hashing/src/main/scala/com/twitter/hashing", + ], ) diff --git a/util-hashing/src/main/scala/com/twitter/hashing/BUILD b/util-hashing/src/main/scala/com/twitter/hashing/BUILD new file mode 100644 index 0000000000..d5368648b1 --- /dev/null +++ b/util-hashing/src/main/scala/com/twitter/hashing/BUILD @@ -0,0 +1,8 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-hashing", + repo = artifactory, + ), +) diff --git a/util-hashing/src/test/scala/BUILD b/util-hashing/src/test/scala/BUILD index 95f83bfa5f..50c8534f02 100644 --- a/util-hashing/src/test/scala/BUILD +++ b/util-hashing/src/test/scala/BUILD @@ -1,13 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalacheck", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala", - "util/util-hashing/src/main/scala", - "util/util-hashing/src/test/resources", + "util/util-hashing/src/test/scala/com/twitter/hashing", ], ) diff --git a/util-hashing/src/test/scala/com/twitter/hashing/BUILD b/util-hashing/src/test/scala/com/twitter/hashing/BUILD new file mode 100644 index 0000000000..9fd036514e --- /dev/null +++ b/util-hashing/src/test/scala/com/twitter/hashing/BUILD @@ -0,0 +1,12 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-hashing/src/main/scala/com/twitter/hashing", + "util/util-hashing/src/test/resources", + ], +) diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD index 3edc8e9b63..fbc6215bc0 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD @@ -4,7 +4,7 @@ junit_tests( strict_deps = True, dependencies = [ "3rdparty/jvm/org/scalatest", - "util/util-core", + "util/util-core/src/main/scala/com/twitter/util", "util/util-intellij/src/main/resources", ], ) diff --git a/util-jvm/src/main/scala/BUILD b/util-jvm/src/main/scala/BUILD index d7aba085b4..1140b99e90 100644 --- a/util-jvm/src/main/scala/BUILD +++ b/util-jvm/src/main/scala/BUILD @@ -1,17 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-jvm", - repo = artifactory, - ), +target( dependencies = [ - "util/util-app/src/main/scala", - "util/util-core/src/main/scala", - "util/util-stats/src/main/scala", - ], - exports = [ - "util/util-core/src/main/scala", + "util/util-jvm/src/main/scala/com/twitter/jvm", ], ) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/BUILD b/util-jvm/src/main/scala/com/twitter/jvm/BUILD new file mode 100644 index 0000000000..67957f1aac --- /dev/null +++ b/util-jvm/src/main/scala/com/twitter/jvm/BUILD @@ -0,0 +1,17 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-jvm", + repo = artifactory, + ), + dependencies = [ + "util/util-app/src/main/scala/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", + ], + exports = [ + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-jvm/src/test/java/BUILD b/util-jvm/src/test/java/BUILD index f19e7bd14c..33ad7812e9 100644 --- a/util-jvm/src/test/java/BUILD +++ b/util-jvm/src/test/java/BUILD @@ -1,8 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "util/util-jvm/src/main/scala", + "util/util-jvm/src/test/java/com/twitter/jvm", ], ) diff --git a/util-jvm/src/test/java/com/twitter/jvm/BUILD b/util-jvm/src/test/java/com/twitter/jvm/BUILD new file mode 100644 index 0000000000..767a31f66c --- /dev/null +++ b/util-jvm/src/test/java/com/twitter/jvm/BUILD @@ -0,0 +1,8 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "util/util-jvm/src/main/scala/com/twitter/jvm", + ], +) diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index 181cf7af01..6f1f6ef0ad 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -1,22 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-app/src/main/scala", - "util/util-core/src/main/scala", - "util/util-jvm/src/main/scala", - "util/util-logging/src/main/scala", - "util/util-test/src/main/scala", - ], -) - -jvm_binary( - name = "bin", - main = "com.twitter.jvm.EstimatorApp", - dependencies = [ - ":scala", + "util/util-jvm/src/test/scala/com/twitter/jvm", ], ) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/BUILD b/util-jvm/src/test/scala/com/twitter/jvm/BUILD new file mode 100644 index 0000000000..71e4f440ed --- /dev/null +++ b/util-jvm/src/test/scala/com/twitter/jvm/BUILD @@ -0,0 +1,36 @@ +junit_tests( + sources = globs( + "*.scala", + exclude = [ + globs("EstimatorApp.scala"), + ], + ), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-app/src/main/scala/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-jvm/src/main/scala/com/twitter/jvm", + "util/util-logging/src/main/scala", + "util/util-test/src/main/scala", + ], +) + +scala_library( + name = "estimator_app", + sources = globs("EstimatorApp.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "util/util-jvm/src/main/scala/com/twitter/jvm", + ], +) + +jvm_binary( + name = "bin", + main = "com.twitter.jvm.EstimatorApp", + dependencies = [ + ":estimator_app", + ], +) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala new file mode 100644 index 0000000000..827b8264e2 --- /dev/null +++ b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala @@ -0,0 +1,92 @@ +package com.twitter.jvm + +/** + * Take a GC log produced by: + * + * {{{ + * $ jstat -gc \$PID 250 ... + * }}} + * + * And report on GC prediction accuracy. Time is + * indexed by the jstat output, and the columns are, + * in order: current time, next gc, estimated next GC. + */ +object EstimatorApp extends App { + import com.twitter.conversions.storage._ + + val estimator = args match { + case Array("kalman", n, error) => + new KalmanGaussianError(n.toInt, error.toDouble) + case Array("windowed", n, windows) => + new WindowedMeans( + n.toInt, + windows.split(",") map { w => + w.split(":") match { + case Array(w, i) => (w.toInt, i.toInt) + case _ => throw new IllegalArgumentException("bad weight, count pair " + w) + } + } + ) + case Array("load", interval) => + new LoadAverage(interval.toDouble) + case _ => throw new IllegalArgumentException("bad args ") + } + + val lines = scala.io.Source.stdin.getLines().drop(1) + val states = lines.toArray map (_.split(" ") filter (_ != "") map (_.toDouble)) collect { + case Array(s0c, s1c, s0u, s1u, ec, eu, oc, ou, pc, pu, ygc, ygct, fgc, fgct, gct) => + PoolState(ygc.toLong, ec.toLong.bytes, eu.toLong.bytes) + } + + var elapsed = 1 + for (List(begin, end) <- states.toList.sliding(2)) { + val allocated = (end - begin).used + estimator.measure(allocated.inBytes) + val r = end.capacity - end.used + val i = (r.inBytes / estimator.estimate.toLong) + elapsed + val j = states.indexWhere(_.numCollections > end.numCollections) + + if (j > 0) + println("%d %d %d".format(elapsed, j, i)) + + elapsed += 1 + } +} + +/* + +The following script is useful for plotting +results from EstimatorApp: + + % scala ... com.twitter.jvm.EstimatorApp [ARGS] > /tmp/out + +#!/usr/bin/env gnuplot + +set terminal png size 800,600 +set title "GC predictor" + +set macros + +set grid +set timestamp "Generated on %Y-%m-%d by `whoami`" font "Helvetica-Oblique, 8pt" +set noclip +set xrange [0:1100] + +set key reverse left Left # box + +set ylabel "Time of GC" textcolor lt 3 +set yrange [0:1100] +set mytics 5 +set y2tics + +set xlabel "Time" textcolor lt 4 +set mxtics 5 + +set boxwidth 0.5 +set style fill transparent pattern 4 bo + +plot "< awk '{print $1 \" \" $2}' /tmp/out" title "Actual", \ + "< awk '{print $1 \" \" $3}' /tmp/out" title "Predicted", \ + "< awk '{print $1 \" \" $1}' /tmp/out" title "time" with lines + + */ diff --git a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala index 96b849e6b4..fed1867e10 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala @@ -1,10 +1,7 @@ package com.twitter.jvm -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class EstimatorTest extends FunSuite { test("LoadAverage") { // This makes LoadAverage.a = 1/2 for easy testing. @@ -22,94 +19,3 @@ class EstimatorTest extends FunSuite { assert(e.estimate == 0) } } - -/** - * Take a GC log produced by: - * - * {{{ - * $ jstat -gc \$PID 250 ... - * }}} - * - * And report on GC prediction accuracy. Time is - * indexed by the jstat output, and the columns are, - * in order: current time, next gc, estimated next GC. - */ -object EstimatorApp extends App { - import com.twitter.conversions.storage._ - - val estimator = args match { - case Array("kalman", n, error) => - new KalmanGaussianError(n.toInt, error.toDouble) - case Array("windowed", n, windows) => - new WindowedMeans( - n.toInt, - windows.split(",") map { w => - w.split(":") match { - case Array(w, i) => (w.toInt, i.toInt) - case _ => throw new IllegalArgumentException("bad weight, count pair " + w) - } - } - ) - case Array("load", interval) => - new LoadAverage(interval.toDouble) - case _ => throw new IllegalArgumentException("bad args ") - } - - val lines = scala.io.Source.stdin.getLines().drop(1) - val states = lines.toArray map (_.split(" ") filter (_ != "") map (_.toDouble)) collect { - case Array(s0c, s1c, s0u, s1u, ec, eu, oc, ou, pc, pu, ygc, ygct, fgc, fgct, gct) => - PoolState(ygc.toLong, ec.toLong.bytes, eu.toLong.bytes) - } - - var elapsed = 1 - for (List(begin, end) <- states.toList.sliding(2)) { - val allocated = (end - begin).used - estimator.measure(allocated.inBytes) - val r = end.capacity - end.used - val i = (r.inBytes / estimator.estimate.toLong) + elapsed - val j = states.indexWhere(_.numCollections > end.numCollections) - - if (j > 0) - println("%d %d %d".format(elapsed, j, i)) - - elapsed += 1 - } -} - -/* - -The following script is useful for plotting -results from EstimatorApp: - - % scala ... com.twitter.jvm.EstimatorApp [ARGS] > /tmp/out - -#!/usr/bin/env gnuplot - -set terminal png size 800,600 -set title "GC predictor" - -set macros - -set grid -set timestamp "Generated on %Y-%m-%d by `whoami`" font "Helvetica-Oblique, 8pt" -set noclip -set xrange [0:1100] - -set key reverse left Left # box - -set ylabel "Time of GC" textcolor lt 3 -set yrange [0:1100] -set mytics 5 -set y2tics - -set xlabel "Time" textcolor lt 4 -set mxtics 5 - -set boxwidth 0.5 -set style fill transparent pattern 4 bo - -plot "< awk '{print $1 \" \" $2}' /tmp/out" title "Actual", \ - "< awk '{print $1 \" \" $3}' /tmp/out" title "Predicted", \ - "< awk '{print $1 \" \" $1}' /tmp/out" title "time" with lines - - */ diff --git a/util-lint/src/main/scala/BUILD b/util-lint/src/main/scala/BUILD index 98291863b0..01f00626ec 100644 --- a/util-lint/src/main/scala/BUILD +++ b/util-lint/src/main/scala/BUILD @@ -1,12 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-lint", - repo = artifactory, - ), +target( dependencies = [ - "util/util-core/src/main/scala", + "util/util-lint/src/main/scala/com/twitter/util/lint", ], ) diff --git a/util-lint/src/main/scala/com/twitter/util/lint/BUILD b/util-lint/src/main/scala/com/twitter/util/lint/BUILD new file mode 100644 index 0000000000..58c0943416 --- /dev/null +++ b/util-lint/src/main/scala/com/twitter/util/lint/BUILD @@ -0,0 +1,11 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-lint", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-lint/src/test/scala/BUILD b/util-lint/src/test/scala/BUILD index 5487109e51..22b1a0076a 100644 --- a/util-lint/src/test/scala/BUILD +++ b/util-lint/src/test/scala/BUILD @@ -1,10 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-lint/src/main/scala", + "util/util-lint/src/test/scala/com/twitter/util/lint", ], ) diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD new file mode 100644 index 0000000000..76d6e0f76c --- /dev/null +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-lint/src/main/scala/com/twitter/util/lint", + ], +) diff --git a/util-stats/src/main/scala/BUILD b/util-stats/src/main/scala/BUILD index ac704c3c25..23c6fb4ad5 100644 --- a/util-stats/src/main/scala/BUILD +++ b/util-stats/src/main/scala/BUILD @@ -1,18 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-stats", - repo = artifactory, - ), +target( dependencies = [ - "3rdparty/jvm/com/github/ben-manes/caffeine", - "3rdparty/jvm/com/google/code/findbugs:jsr305", - "util/util-core/src/main/scala", - "util/util-lint/src/main/scala", - ], - exports = [ - "3rdparty/jvm/com/github/ben-manes/caffeine", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD new file mode 100644 index 0000000000..5a230c555e --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD @@ -0,0 +1,17 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-stats", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/com/github/ben-manes/caffeine", + "3rdparty/jvm/com/google/code/findbugs:jsr305", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-lint/src/main/scala/com/twitter/util/lint", + ], + exports = [ + "3rdparty/jvm/com/github/ben-manes/caffeine", + ], +) diff --git a/util-stats/src/test/java/BUILD b/util-stats/src/test/java/BUILD index 05b139b929..ee3927bab5 100644 --- a/util-stats/src/test/java/BUILD +++ b/util-stats/src/test/java/BUILD @@ -1,9 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "util/util-core/src/main/scala", - "util/util-stats/src/main/scala", + "util/util-stats/src/test/java/com/twitter/finagle/stats", ], ) diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/BUILD b/util-stats/src/test/java/com/twitter/finagle/stats/BUILD new file mode 100644 index 0000000000..5a636b8ad8 --- /dev/null +++ b/util-stats/src/test/java/com/twitter/finagle/stats/BUILD @@ -0,0 +1,9 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", + ], +) diff --git a/util-stats/src/test/scala/BUILD b/util-stats/src/test/scala/BUILD index 82321d2d04..9071fa8e03 100644 --- a/util-stats/src/test/scala/BUILD +++ b/util-stats/src/test/scala/BUILD @@ -1,12 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalacheck", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala", - "util/util-stats/src/main/scala", + "util/util-stats/src/test/scala/com/twitter/finagle/stats", ], ) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD new file mode 100644 index 0000000000..7d9e9c982c --- /dev/null +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD @@ -0,0 +1,13 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", + ], +) From cdf2b459042b8c457dce438e036957bd224e8974 Mon Sep 17 00:00:00 2001 From: Jordan Parker Date: Fri, 19 Oct 2018 20:40:48 +0000 Subject: [PATCH 045/796] Twitter-oss: update OSS libraries post-release to 18.11.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. TBR=true Differential Revision: https://phabricator.twitter.biz/D231095 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b331c1748c..74e5e6c316 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index f04098d126..fa7cf22819 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.10.0" +val releaseVersion = "18.11.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 9bde57caffa7e127967fb2452beab3d09fd9888e Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 22 Oct 2018 18:37:32 +0000 Subject: [PATCH 046/796] util-core: Make Future.raiseWithin take a thunk Throwable Problem Future.raiseWithin takes an Exception value as an argument which is inconsistent with the other Future timer based API's and can be costly since creating an exception often necessitates creating a stacktrace. Solution Make the exception call-by-name. JIRA Issues: CSL-7133 TBR=true Differential Revision: https://phabricator.twitter.biz/D229559 --- CHANGELOG.rst | 3 +++ util-core/src/main/scala/com/twitter/util/Future.scala | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 341a47f0eb..53db6cb3e9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* util-core: `c.t.u.Future.raiseWithin` methods now take the timeout exception as a call-by-name + parameter instead of a strict exception. ``PHAB_ID=D229559`` + 18.10.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 47dd22453b..b92eb75dfd 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1595,7 +1595,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * * ''Note'': On timeout, the underlying future is interrupted. */ - def raiseWithin(timeout: Duration, exc: Throwable)(implicit timer: Timer): Future[A] = + def raiseWithin(timeout: Duration, exc: => Throwable)(implicit timer: Timer): Future[A] = raiseWithin(timer, timeout, exc) /** @@ -1603,7 +1603,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * * ''Note'': On timeout, the underlying future is interrupted. */ - def raiseWithin(timer: Timer, timeout: Duration, exc: Throwable): Future[A] = { + def raiseWithin(timer: Timer, timeout: Duration, exc: => Throwable): Future[A] = { if (timeout == Duration.Top || isDefined) return this From e48fd5f4dfef01c58555cabbe76fbe68014446ca Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 22 Oct 2018 20:19:31 +0000 Subject: [PATCH 047/796] util: Better CHANGELOG entry Problem / Solution We didn't provide guidance for Java users to use the new call-by-name API of `Future.raiseWithin`, so lets add some. Differential Revision: https://phabricator.twitter.biz/D231660 --- CHANGELOG.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 53db6cb3e9..9f45f1fbb1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,7 +8,9 @@ Unreleased ---------- * util-core: `c.t.u.Future.raiseWithin` methods now take the timeout exception as a call-by-name - parameter instead of a strict exception. ``PHAB_ID=D229559`` + parameter instead of a strict exception. While Scala programs should compile as usual, Java + users will need to use a `scala.Function0` as the second parameter. The helper + `c.t.u.Function.func0` can be helpful. ``PHAB_ID=D229559`` 18.10.0 ------- From 3844d09ffa79ce8ca1b334944f252b00574240bb Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Mon, 22 Oct 2018 20:49:13 +0000 Subject: [PATCH 048/796] util: Migrate rest of projects to pants 1:1:1 Problem We'd like our modules to follow pants 1:1:1 style. Solution Update the rest of util's modules to use that. Result Migrated: - util-logging - util-reflect - util-security - util-slf4j-api - util-slf4j-jul-bridge - util-test - util-thrift - util-zk JIRA Issues: CSL-7094 Differential Revision: https://phabricator.twitter.biz/D231514 --- util-logging/src/main/scala/BUILD | 16 ++------------ .../src/main/scala/com/twitter/logging/BUILD | 19 +++++++++++++++++ util-logging/src/test/java/BUILD | 7 ++----- .../src/test/java/com/twitter/logging/BUILD | 8 +++++++ util-logging/src/test/scala/BUILD | 12 ++--------- .../src/test/scala/com/twitter/logging/BUILD | 14 +++++++++++++ util-reflect/src/main/scala/BUILD | 15 ++----------- .../main/scala/com/twitter/util/reflect/BUILD | 15 +++++++++++++ util-reflect/src/test/scala/BUILD | 9 ++------ .../test/scala/com/twitter/util/reflect/BUILD | 10 +++++++++ util-security/src/main/scala/BUILD | 12 ++--------- .../scala/com/twitter/util/security/BUILD | 12 +++++++++++ util-security/src/test/java/BUILD | 8 ++----- .../test/java/com/twitter/util/security/BUILD | 9 ++++++++ util-security/src/test/scala/BUILD | 11 ++-------- .../scala/com/twitter/util/security/BUILD | 13 ++++++++++++ util-slf4j-api/src/main/scala/BUILD | 14 ++----------- .../main/scala/com/twitter/util/logging/BUILD | 14 +++++++++++++ util-slf4j-api/src/test/java/BUILD | 10 ++------- .../test/java/com/twitter/util/logging/BUILD | 10 +++++++++ util-slf4j-api/src/test/scala/BUILD | 12 ++--------- .../test/scala/com/twitter/util/logging/BUILD | 13 ++++++++++++ util-slf4j-jul-bridge/src/main/scala/BUILD | 14 ++----------- .../main/scala/com/twitter/util/logging/BUILD | 14 +++++++++++++ util-test/src/main/scala/BUILD | 11 +++++----- .../src/main/scala/com/twitter/logging/BUILD | 3 +-- .../main/scala/com/twitter/util/testing/BUILD | 15 +++++++++++++ util-test/src/test/scala/BUILD | 9 ++------ .../test/scala/com/twitter/util/testing/BUILD | 10 +++++++++ util-thrift/src/main/scala/BUILD | 18 ++-------------- .../src/main/scala/com/twitter/util/BUILD | 17 +++++++++++++++ util-thrift/src/test/java/BUILD | 6 ++---- .../src/test/java/com/twitter/util/BUILD | 6 ++++++ util-thrift/src/test/scala/BUILD | 10 ++------- .../src/test/scala/com/twitter/util/BUILD | 10 +++++++++ util-zk-test/src/main/scala/BUILD | 14 ++----------- .../src/main/scala/com/twitter/zk/BUILD | 14 +++++++++++++ util-zk-test/src/test/java/BUILD | 7 ++----- .../src/test/java/com/twitter/zk/BUILD | 8 +++++++ util-zk-test/src/test/scala/BUILD | 11 ++-------- .../src/test/scala/com/twitter/zk/BUILD | 11 ++++++++++ util-zk/src/main/scala/BUILD | 13 ++++++------ util-zk/src/main/scala/com/twitter/zk/BUILD | 21 +++++++++++++++++++ .../scala/com/twitter/zk/coordination/BUILD | 20 ++++++++++++++++++ util-zk/src/test/scala/BUILD | 14 +++---------- util-zk/src/test/scala/com/twitter/zk/BUILD | 14 +++++++++++++ .../scala/com/twitter/zk/coordination/BUILD | 14 +++++++++++++ 47 files changed, 365 insertions(+), 202 deletions(-) create mode 100644 util-logging/src/main/scala/com/twitter/logging/BUILD create mode 100644 util-logging/src/test/java/com/twitter/logging/BUILD create mode 100644 util-logging/src/test/scala/com/twitter/logging/BUILD create mode 100644 util-reflect/src/main/scala/com/twitter/util/reflect/BUILD create mode 100644 util-reflect/src/test/scala/com/twitter/util/reflect/BUILD create mode 100644 util-security/src/main/scala/com/twitter/util/security/BUILD create mode 100644 util-security/src/test/java/com/twitter/util/security/BUILD create mode 100644 util-security/src/test/scala/com/twitter/util/security/BUILD create mode 100644 util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD create mode 100644 util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD create mode 100644 util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD create mode 100644 util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD create mode 100644 util-test/src/main/scala/com/twitter/util/testing/BUILD create mode 100644 util-test/src/test/scala/com/twitter/util/testing/BUILD create mode 100644 util-thrift/src/main/scala/com/twitter/util/BUILD create mode 100644 util-thrift/src/test/java/com/twitter/util/BUILD create mode 100644 util-thrift/src/test/scala/com/twitter/util/BUILD create mode 100644 util-zk-test/src/main/scala/com/twitter/zk/BUILD create mode 100644 util-zk-test/src/test/java/com/twitter/zk/BUILD create mode 100644 util-zk-test/src/test/scala/com/twitter/zk/BUILD create mode 100644 util-zk/src/main/scala/com/twitter/zk/BUILD create mode 100644 util-zk/src/main/scala/com/twitter/zk/coordination/BUILD create mode 100644 util-zk/src/test/scala/com/twitter/zk/BUILD create mode 100644 util-zk/src/test/scala/com/twitter/zk/coordination/BUILD diff --git a/util-logging/src/main/scala/BUILD b/util-logging/src/main/scala/BUILD index 2d4741117d..64906b48ca 100644 --- a/util-logging/src/main/scala/BUILD +++ b/util-logging/src/main/scala/BUILD @@ -1,17 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-logging", - repo = artifactory, - ), +target( dependencies = [ - "util/util-app/src/main/scala", - "util/util-core/src/main/scala", - "util/util-stats/src/main/scala", - ], - exports = [ - "util/util-app/src/main/scala", + "util/util-logging/src/main/scala/com/twitter/logging", ], ) diff --git a/util-logging/src/main/scala/com/twitter/logging/BUILD b/util-logging/src/main/scala/com/twitter/logging/BUILD new file mode 100644 index 0000000000..67f2e55270 --- /dev/null +++ b/util-logging/src/main/scala/com/twitter/logging/BUILD @@ -0,0 +1,19 @@ +scala_library( + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-logging", + repo = artifactory, + ), + dependencies = [ + "util/util-app/src/main/scala/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", + ], + exports = [ + "util/util-app/src/main/scala/com/twitter/app", + ], +) diff --git a/util-logging/src/test/java/BUILD b/util-logging/src/test/java/BUILD index f1f271ce4d..076e509f5c 100644 --- a/util-logging/src/test/java/BUILD +++ b/util-logging/src/test/java/BUILD @@ -1,8 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "util/util-logging/src/main/scala", + "util/util-logging/src/test/java/com/twitter/logging", ], ) diff --git a/util-logging/src/test/java/com/twitter/logging/BUILD b/util-logging/src/test/java/com/twitter/logging/BUILD new file mode 100644 index 0000000000..aa1d2539b9 --- /dev/null +++ b/util-logging/src/test/java/com/twitter/logging/BUILD @@ -0,0 +1,8 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "util/util-logging/src/main/scala/com/twitter/logging", + ], +) diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 61cdfb74f1..822a12ed7f 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -1,13 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-app/src/main/scala", - "util/util-core/src/main/scala", - "util/util-logging/src/main/scala", - "util/util-stats/src/main/scala", + "util/util-logging/src/test/scala/com/twitter/logging", ], ) diff --git a/util-logging/src/test/scala/com/twitter/logging/BUILD b/util-logging/src/test/scala/com/twitter/logging/BUILD new file mode 100644 index 0000000000..f0920cd015 --- /dev/null +++ b/util-logging/src/test/scala/com/twitter/logging/BUILD @@ -0,0 +1,14 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-app/src/main/scala/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-logging/src/main/scala/com/twitter/logging", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", + ], +) diff --git a/util-reflect/src/main/scala/BUILD b/util-reflect/src/main/scala/BUILD index 86d1d68f26..b3d5909852 100644 --- a/util-reflect/src/main/scala/BUILD +++ b/util-reflect/src/main/scala/BUILD @@ -1,16 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-reflect", - repo = artifactory, - ), +target( dependencies = [ - "3rdparty/jvm/cglib", - "util/util-core/src/main/scala", - ], - exports = [ - "3rdparty/jvm/cglib", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", ], ) diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD new file mode 100644 index 0000000000..0277d95b02 --- /dev/null +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD @@ -0,0 +1,15 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-reflect", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/cglib", + "util/util-core/src/main/scala/com/twitter/util", + ], + exports = [ + "3rdparty/jvm/cglib", + ], +) diff --git a/util-reflect/src/test/scala/BUILD b/util-reflect/src/test/scala/BUILD index df9bea67f6..e5ef76f364 100644 --- a/util-reflect/src/test/scala/BUILD +++ b/util-reflect/src/test/scala/BUILD @@ -1,10 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala", - "util/util-reflect/src/main/scala", + "util/util-reflect/src/test/scala/com/twitter/util/reflect", ], ) diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD new file mode 100644 index 0000000000..e8ffe7c0a1 --- /dev/null +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + ], +) diff --git a/util-security/src/main/scala/BUILD b/util-security/src/main/scala/BUILD index 8ea39d4af3..e337f9b286 100644 --- a/util-security/src/main/scala/BUILD +++ b/util-security/src/main/scala/BUILD @@ -1,13 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-security", - repo = artifactory, - ), +target( dependencies = [ - "util/util-core/src/main/scala", - "util/util-logging", + "util/util-security/src/main/scala/com/twitter/util/security", ], ) diff --git a/util-security/src/main/scala/com/twitter/util/security/BUILD b/util-security/src/main/scala/com/twitter/util/security/BUILD new file mode 100644 index 0000000000..7e8bcf3479 --- /dev/null +++ b/util-security/src/main/scala/com/twitter/util/security/BUILD @@ -0,0 +1,12 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-security", + repo = artifactory, + ), + dependencies = [ + "util/util-core/src/main/scala/com/twitter/util", + "util/util-logging/src/main/scala/com/twitter/logging", + ], +) diff --git a/util-security/src/test/java/BUILD b/util-security/src/test/java/BUILD index e0fc120cf9..f2370e4dd8 100644 --- a/util-security/src/test/java/BUILD +++ b/util-security/src/test/java/BUILD @@ -1,9 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "util/util-core/src/main/scala", - "util/util-security/src/main/scala", + "util/util-security/src/test/java/com/twitter/util/security", ], ) diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD new file mode 100644 index 0000000000..e46848252f --- /dev/null +++ b/util-security/src/test/java/com/twitter/util/security/BUILD @@ -0,0 +1,9 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-security/src/main/scala/com/twitter/util/security", + ], +) diff --git a/util-security/src/test/scala/BUILD b/util-security/src/test/scala/BUILD index 320dcc0964..224798f5bf 100644 --- a/util-security/src/test/scala/BUILD +++ b/util-security/src/test/scala/BUILD @@ -1,12 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala", - "util/util-logging", - "util/util-security/src/main/scala", - "util/util-security/src/test/resources", + "util/util-security/src/test/scala/com/twitter/util/security", ], ) diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD new file mode 100644 index 0000000000..a8ae77fa43 --- /dev/null +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -0,0 +1,13 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-logging/src/main/scala/com/twitter/logging", + "util/util-security/src/main/scala/com/twitter/util/security", + "util/util-security/src/test/resources", + ], +) diff --git a/util-slf4j-api/src/main/scala/BUILD b/util-slf4j-api/src/main/scala/BUILD index 056dfa6aaf..dc62b38ffd 100644 --- a/util-slf4j-api/src/main/scala/BUILD +++ b/util-slf4j-api/src/main/scala/BUILD @@ -1,15 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-slf4j", - repo = artifactory, - ), +target( dependencies = [ - "3rdparty/jvm/org/slf4j:slf4j-api", - ], - exports = [ - "3rdparty/jvm/org/slf4j:slf4j-api", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], ) diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD new file mode 100644 index 0000000000..e6d6b8578d --- /dev/null +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD @@ -0,0 +1,14 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-slf4j", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/slf4j:slf4j-api", + ], + exports = [ + "3rdparty/jvm/org/slf4j:slf4j-api", + ], +) diff --git a/util-slf4j-api/src/test/java/BUILD b/util-slf4j-api/src/test/java/BUILD index e415f5b644..fcd4c2c567 100644 --- a/util-slf4j-api/src/test/java/BUILD +++ b/util-slf4j-api/src/test/java/BUILD @@ -1,11 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-simple", - "util/util-slf4j-api/src/main/scala", - "util/util-slf4j-api/src/test/scala", + "util/util-slf4j-api/src/test/java/com/twitter/util/logging", ], ) diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD new file mode 100644 index 0000000000..a261c4cc21 --- /dev/null +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-simple", + "util/util-slf4j-api/src/main/scala", + ], +) diff --git a/util-slf4j-api/src/test/scala/BUILD b/util-slf4j-api/src/test/scala/BUILD index b46087de5f..34bc6295e6 100644 --- a/util-slf4j-api/src/test/scala/BUILD +++ b/util-slf4j-api/src/test/scala/BUILD @@ -1,13 +1,5 @@ -junit_tests( - sources = rglobs("*.scala", "*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-simple", - "util/util-core/src/main/scala", - "util/util-slf4j-api/src/main/scala", - "util/util-slf4j-api/src/test/resources", + "util/util-slf4j-api/src/test/scala/com/twitter/util/logging", ], ) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD new file mode 100644 index 0000000000..15efdb04be --- /dev/null +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -0,0 +1,13 @@ +junit_tests( + sources = globs("*.scala", "*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/slf4j:slf4j-api", + "3rdparty/jvm/org/slf4j:slf4j-simple", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + "util/util-slf4j-api/src/test/resources", + ], +) diff --git a/util-slf4j-jul-bridge/src/main/scala/BUILD b/util-slf4j-jul-bridge/src/main/scala/BUILD index 34571a74c0..aed7bcd15b 100644 --- a/util-slf4j-jul-bridge/src/main/scala/BUILD +++ b/util-slf4j-jul-bridge/src/main/scala/BUILD @@ -1,15 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-slf4j-jul-bridge", - repo = artifactory, - ), +target( dependencies = [ - "3rdparty/jvm/org/slf4j:jul-to-slf4j", - "3rdparty/jvm/org/slf4j:slf4j-api", - "util/util-core/src/main/scala", - "util/util-slf4j-api/src/main/scala", + "util/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging", ], ) diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD new file mode 100644 index 0000000000..94adec6123 --- /dev/null +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD @@ -0,0 +1,14 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-slf4j-jul-bridge", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/slf4j:jul-to-slf4j", + "3rdparty/jvm/org/slf4j:slf4j-api", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + ], +) diff --git a/util-test/src/main/scala/BUILD b/util-test/src/main/scala/BUILD index 9f4bcaf81d..e899d2a652 100644 --- a/util-test/src/main/scala/BUILD +++ b/util-test/src/main/scala/BUILD @@ -1,17 +1,18 @@ scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, + # This target exists only to act as an aggregator jar. + sources = ["BUILD"], provides = scala_artifact( org = "com.twitter", name = "util-test", repo = artifactory, ), dependencies = [ - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-logging/src/main/scala", + "util/util-test/src/main/scala/com/twitter/logging", + "util/util-test/src/main/scala/com/twitter/util/testing", ], exports = [ "3rdparty/jvm/org/scalatest", + "util/util-test/src/main/scala/com/twitter/logging", + "util/util-test/src/main/scala/com/twitter/util/testing", ], ) diff --git a/util-test/src/main/scala/com/twitter/logging/BUILD b/util-test/src/main/scala/com/twitter/logging/BUILD index ccec239edc..723ed1f465 100644 --- a/util-test/src/main/scala/com/twitter/logging/BUILD +++ b/util-test/src/main/scala/com/twitter/logging/BUILD @@ -1,5 +1,4 @@ scala_library( - sources = globs("*.scala"), compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", @@ -8,7 +7,7 @@ scala_library( ), dependencies = [ "3rdparty/jvm/org/scalatest", - "util/util-logging/src/main/scala", + "util/util-logging/src/main/scala/com/twitter/logging", ], exports = [ "3rdparty/jvm/org/scalatest", diff --git a/util-test/src/main/scala/com/twitter/util/testing/BUILD b/util-test/src/main/scala/com/twitter/util/testing/BUILD new file mode 100644 index 0000000000..43f24f67a7 --- /dev/null +++ b/util-test/src/main/scala/com/twitter/util/testing/BUILD @@ -0,0 +1,15 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-test-testing", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + ], + exports = [ + "3rdparty/jvm/org/scalatest", + ], +) diff --git a/util-test/src/test/scala/BUILD b/util-test/src/test/scala/BUILD index 70eb5fdb8e..ede2bb7120 100644 --- a/util-test/src/test/scala/BUILD +++ b/util-test/src/test/scala/BUILD @@ -1,10 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-test/src/main/scala", + "util/util-test/src/test/scala/com/twitter/util/testing", ], ) diff --git a/util-test/src/test/scala/com/twitter/util/testing/BUILD b/util-test/src/test/scala/com/twitter/util/testing/BUILD new file mode 100644 index 0000000000..14f42b3025 --- /dev/null +++ b/util-test/src/test/scala/com/twitter/util/testing/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-test/src/main/scala/com/twitter/util/testing", + ], +) diff --git a/util-thrift/src/main/scala/BUILD b/util-thrift/src/main/scala/BUILD index 1e9084b28b..129ea9ebbb 100644 --- a/util-thrift/src/main/scala/BUILD +++ b/util-thrift/src/main/scala/BUILD @@ -1,19 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-thrift", - repo = artifactory, - ), +target( dependencies = [ - "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", - "3rdparty/jvm/org/apache/thrift:libthrift", - "util/util-codec", - "util/util-core/src/main/scala", - ], - exports = [ - "3rdparty/jvm/org/apache/thrift:libthrift", - "util/util-codec", + "util/util-thrift/src/main/scala/com/twitter/util", ], ) diff --git a/util-thrift/src/main/scala/com/twitter/util/BUILD b/util-thrift/src/main/scala/com/twitter/util/BUILD new file mode 100644 index 0000000000..84d006fc2b --- /dev/null +++ b/util-thrift/src/main/scala/com/twitter/util/BUILD @@ -0,0 +1,17 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-thrift", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/org/apache/thrift:libthrift", + "util/util-codec/src/main/scala/com/twitter/util", + ], + exports = [ + "3rdparty/jvm/org/apache/thrift:libthrift", + "util/util-codec/src/main/scala/com/twitter/util", + ], +) diff --git a/util-thrift/src/test/java/BUILD b/util-thrift/src/test/java/BUILD index 967b1e8f00..60c095c87f 100644 --- a/util-thrift/src/test/java/BUILD +++ b/util-thrift/src/test/java/BUILD @@ -1,7 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/org/apache/thrift:libthrift", + "util/util-thrift/src/test/java/com/twitter/util", ], ) diff --git a/util-thrift/src/test/java/com/twitter/util/BUILD b/util-thrift/src/test/java/com/twitter/util/BUILD new file mode 100644 index 0000000000..58e22a2a2a --- /dev/null +++ b/util-thrift/src/test/java/com/twitter/util/BUILD @@ -0,0 +1,6 @@ +java_library( + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/org/apache/thrift:libthrift", + ], +) diff --git a/util-thrift/src/test/scala/BUILD b/util-thrift/src/test/scala/BUILD index 625d2299fc..92308690a9 100644 --- a/util-thrift/src/test/scala/BUILD +++ b/util-thrift/src/test/scala/BUILD @@ -1,11 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala", - "util/util-thrift/src/main/scala", - "util/util-thrift/src/test/java", + "util/util-thrift/src/test/scala/com/twitter/util", ], ) diff --git a/util-thrift/src/test/scala/com/twitter/util/BUILD b/util-thrift/src/test/scala/com/twitter/util/BUILD new file mode 100644 index 0000000000..255b83cdb0 --- /dev/null +++ b/util-thrift/src/test/scala/com/twitter/util/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "util/util-thrift/src/main/scala/com/twitter/util", + "util/util-thrift/src/test/java/com/twitter/util", + ], +) diff --git a/util-zk-test/src/main/scala/BUILD b/util-zk-test/src/main/scala/BUILD index 5b001bc546..2743ea97b0 100644 --- a/util-zk-test/src/main/scala/BUILD +++ b/util-zk-test/src/main/scala/BUILD @@ -1,15 +1,5 @@ -scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-zk-test", - repo = artifactory, - ), +target( dependencies = [ - "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", - ], - exports = [ - "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", + "util/util-zk-test/src/main/scala/com/twitter/zk", ], ) diff --git a/util-zk-test/src/main/scala/com/twitter/zk/BUILD b/util-zk-test/src/main/scala/com/twitter/zk/BUILD new file mode 100644 index 0000000000..3353aa20fd --- /dev/null +++ b/util-zk-test/src/main/scala/com/twitter/zk/BUILD @@ -0,0 +1,14 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-zk-test", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", + ], + exports = [ + "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", + ], +) diff --git a/util-zk-test/src/test/java/BUILD b/util-zk-test/src/test/java/BUILD index ad490fcffe..ceef82a3ff 100644 --- a/util-zk-test/src/test/java/BUILD +++ b/util-zk-test/src/test/java/BUILD @@ -1,8 +1,5 @@ -junit_tests( - sources = rglobs("*.java"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "util/util-zk-test", + "util/util-zk-test/src/test/java/com/twitter/zk", ], ) diff --git a/util-zk-test/src/test/java/com/twitter/zk/BUILD b/util-zk-test/src/test/java/com/twitter/zk/BUILD new file mode 100644 index 0000000000..0fe79b870e --- /dev/null +++ b/util-zk-test/src/test/java/com/twitter/zk/BUILD @@ -0,0 +1,8 @@ +junit_tests( + sources = globs("*.java"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "util/util-zk-test/src/main/scala/com/twitter/zk", + ], +) diff --git a/util-zk-test/src/test/scala/BUILD b/util-zk-test/src/test/scala/BUILD index 17cba974a6..77c3ee4906 100644 --- a/util-zk-test/src/test/scala/BUILD +++ b/util-zk-test/src/test/scala/BUILD @@ -1,12 +1,5 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-core", - "util/util-zk-test", + "util/util-zk-test/src/test/scala/com/twitter/zk", ], ) diff --git a/util-zk-test/src/test/scala/com/twitter/zk/BUILD b/util-zk-test/src/test/scala/com/twitter/zk/BUILD new file mode 100644 index 0000000000..be60034a67 --- /dev/null +++ b/util-zk-test/src/test/scala/com/twitter/zk/BUILD @@ -0,0 +1,11 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-zk-test/src/main/scala/com/twitter/zk", + ], +) diff --git a/util-zk/src/main/scala/BUILD b/util-zk/src/main/scala/BUILD index 655324cde4..932bf03ff3 100644 --- a/util-zk/src/main/scala/BUILD +++ b/util-zk/src/main/scala/BUILD @@ -1,18 +1,17 @@ scala_library( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, + # This target exists only to act as an aggregator jar. + sources = ["BUILD"], provides = scala_artifact( org = "com.twitter", name = "util-zk", repo = artifactory, ), dependencies = [ - "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", - "util/util-core/src/main/scala", - "util/util-logging/src/main/scala", + "util/util-zk/src/main/scala/com/twitter/zk", + "util/util-zk/src/main/scala/com/twitter/zk/coordination", ], exports = [ - "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", - "util/util-core/src/main/scala", + "util/util-zk/src/main/scala/com/twitter/zk", + "util/util-zk/src/main/scala/com/twitter/zk/coordination", ], ) diff --git a/util-zk/src/main/scala/com/twitter/zk/BUILD b/util-zk/src/main/scala/com/twitter/zk/BUILD new file mode 100644 index 0000000000..126654895e --- /dev/null +++ b/util-zk/src/main/scala/com/twitter/zk/BUILD @@ -0,0 +1,21 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-zk-zk", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-logging/src/main/scala/com/twitter/logging", + ], + exports = [ + "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + ], +) diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD new file mode 100644 index 0000000000..7df377b309 --- /dev/null +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD @@ -0,0 +1,20 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-zk-coordination", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-zk/src/main/scala/com/twitter/zk", + ], + exports = [ + "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-zk/src/main/scala/com/twitter/zk", + ], +) diff --git a/util-zk/src/test/scala/BUILD b/util-zk/src/test/scala/BUILD index 8a4fe40163..de068cd977 100644 --- a/util-zk/src/test/scala/BUILD +++ b/util-zk/src/test/scala/BUILD @@ -1,14 +1,6 @@ -junit_tests( - sources = rglobs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, +target( dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", - "3rdparty/jvm/org/jmock", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala", - "util/util-logging/src/main/scala", - "util/util-zk/src/main/scala", + "util/util-zk/src/test/scala/com/twitter/zk", + "util/util-zk/src/test/scala/com/twitter/zk/coordination", ], ) diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD new file mode 100644 index 0000000000..c235ad4abc --- /dev/null +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -0,0 +1,14 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-logging/src/main/scala/com/twitter/logging", + "util/util-zk/src/main/scala/com/twitter/zk", + ], +) diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD new file mode 100644 index 0000000000..d190c175cc --- /dev/null +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -0,0 +1,14 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-zk/src/main/scala/com/twitter/zk", + "util/util-zk/src/main/scala/com/twitter/zk/coordination", + ], +) From 174e1f76ca20c4676d7ed4260541612af1444d9c Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 23 Oct 2018 21:33:37 +0000 Subject: [PATCH 049/796] util: Workaround artifact publishing issue in Pants Problem Pants has an issue with published artifacts that don't havew any sources. See https://github.com/pantsbuild/pants/pull/6673 Solution Add a private and non-instantiable class, PantsWorkaround, to these targets. They will be removed following the next pants upgrade. JIRA Issues: CSL-7154 Differential Revision: https://phabricator.twitter.biz/D232304 --- util-cache/src/main/scala/BUILD | 2 +- util-cache/src/main/scala/PantsWorkaround.scala | 5 +++++ util-collection/src/main/scala/BUILD | 2 +- util-collection/src/main/scala/PantsWorkaround.scala | 5 +++++ util-core/src/main/scala/BUILD | 2 +- util-core/src/main/scala/PantsWorkaround.scala | 5 +++++ util-test/src/main/scala/BUILD | 2 +- util-test/src/main/scala/PantsWorkaround.scala | 5 +++++ util-zk/src/main/scala/BUILD | 2 +- util-zk/src/main/scala/PantsWorkaround.scala | 5 +++++ 10 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 util-cache/src/main/scala/PantsWorkaround.scala create mode 100644 util-collection/src/main/scala/PantsWorkaround.scala create mode 100644 util-core/src/main/scala/PantsWorkaround.scala create mode 100644 util-test/src/main/scala/PantsWorkaround.scala create mode 100644 util-zk/src/main/scala/PantsWorkaround.scala diff --git a/util-cache/src/main/scala/BUILD b/util-cache/src/main/scala/BUILD index 77874242b5..40c2f0c505 100644 --- a/util-cache/src/main/scala/BUILD +++ b/util-cache/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( # This target exists only to act as an aggregator jar. - sources = ["BUILD"], + sources = ["PantsWorkaround.scala"], provides = scala_artifact( org = "com.twitter", name = "util-cache", diff --git a/util-cache/src/main/scala/PantsWorkaround.scala b/util-cache/src/main/scala/PantsWorkaround.scala new file mode 100644 index 0000000000..78c256812b --- /dev/null +++ b/util-cache/src/main/scala/PantsWorkaround.scala @@ -0,0 +1,5 @@ +// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 +// will be removed following the next pants release. +private final class PantsWorkaround { + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") +} diff --git a/util-collection/src/main/scala/BUILD b/util-collection/src/main/scala/BUILD index 7bf3e97600..c63b628063 100644 --- a/util-collection/src/main/scala/BUILD +++ b/util-collection/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( # This target exists only to act as an aggregator jar. - sources = ["BUILD"], + sources = ["PantsWorkaround.scala"], provides = scala_artifact( org = "com.twitter", name = "util-collection", diff --git a/util-collection/src/main/scala/PantsWorkaround.scala b/util-collection/src/main/scala/PantsWorkaround.scala new file mode 100644 index 0000000000..78c256812b --- /dev/null +++ b/util-collection/src/main/scala/PantsWorkaround.scala @@ -0,0 +1,5 @@ +// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 +// will be removed following the next pants release. +private final class PantsWorkaround { + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") +} diff --git a/util-core/src/main/scala/BUILD b/util-core/src/main/scala/BUILD index aa50e43055..4c64da0f8e 100644 --- a/util-core/src/main/scala/BUILD +++ b/util-core/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( # This target exists only to act as an aggregator jar. - sources = ["BUILD"], + sources = ["PantsWorkaround.scala"], provides = scala_artifact( org = "com.twitter", name = "util-core", diff --git a/util-core/src/main/scala/PantsWorkaround.scala b/util-core/src/main/scala/PantsWorkaround.scala new file mode 100644 index 0000000000..78c256812b --- /dev/null +++ b/util-core/src/main/scala/PantsWorkaround.scala @@ -0,0 +1,5 @@ +// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 +// will be removed following the next pants release. +private final class PantsWorkaround { + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") +} diff --git a/util-test/src/main/scala/BUILD b/util-test/src/main/scala/BUILD index e899d2a652..3a67919cff 100644 --- a/util-test/src/main/scala/BUILD +++ b/util-test/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( # This target exists only to act as an aggregator jar. - sources = ["BUILD"], + sources = ["PantsWorkaround.scala"], provides = scala_artifact( org = "com.twitter", name = "util-test", diff --git a/util-test/src/main/scala/PantsWorkaround.scala b/util-test/src/main/scala/PantsWorkaround.scala new file mode 100644 index 0000000000..78c256812b --- /dev/null +++ b/util-test/src/main/scala/PantsWorkaround.scala @@ -0,0 +1,5 @@ +// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 +// will be removed following the next pants release. +private final class PantsWorkaround { + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") +} diff --git a/util-zk/src/main/scala/BUILD b/util-zk/src/main/scala/BUILD index 932bf03ff3..688347c739 100644 --- a/util-zk/src/main/scala/BUILD +++ b/util-zk/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( # This target exists only to act as an aggregator jar. - sources = ["BUILD"], + sources = ["PantsWorkaround.scala"], provides = scala_artifact( org = "com.twitter", name = "util-zk", diff --git a/util-zk/src/main/scala/PantsWorkaround.scala b/util-zk/src/main/scala/PantsWorkaround.scala new file mode 100644 index 0000000000..78c256812b --- /dev/null +++ b/util-zk/src/main/scala/PantsWorkaround.scala @@ -0,0 +1,5 @@ +// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 +// will be removed following the next pants release. +private final class PantsWorkaround { + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") +} From a970c5b5ddf9cb66a63693aa5cf7709d177a234a Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 24 Oct 2018 01:15:28 +0000 Subject: [PATCH 050/796] util-core: Rename ReaderDiscarded to ReaderDiscardedException Problem/Solution: ReaderDiscarded is an exception, let's make it more explicit. JIRA Issues: CSL-7090 TBR=true Differential Revision: https://phabricator.twitter.biz/D231969 --- CHANGELOG.rst | 6 ++++++ util-core/src/main/scala/com/twitter/io/BufReader.scala | 2 +- .../main/scala/com/twitter/io/InputStreamReader.scala | 4 ++-- util-core/src/main/scala/com/twitter/io/Pipe.scala | 2 +- util-core/src/main/scala/com/twitter/io/Reader.scala | 9 ++++++--- .../src/test/scala/com/twitter/io/BufReaderTest.scala | 4 +++- .../scala/com/twitter/io/InputStreamReaderTest.scala | 2 +- util-core/src/test/scala/com/twitter/io/PipeTest.scala | 8 ++++---- util-core/src/test/scala/com/twitter/io/ReaderTest.scala | 6 +++--- 9 files changed, 27 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9f45f1fbb1..09f2963ae9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,11 +7,17 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + * util-core: `c.t.u.Future.raiseWithin` methods now take the timeout exception as a call-by-name parameter instead of a strict exception. While Scala programs should compile as usual, Java users will need to use a `scala.Function0` as the second parameter. The helper `c.t.u.Function.func0` can be helpful. ``PHAB_ID=D229559`` +* util-core: Rename `c.t.io.Reader.ReaderDiscarded` to `c.t.io.ReaderDiscardedException`. + ``PHAB_ID=D231969`` + 18.10.0 ------- diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index 26c33962c5..e1773d8c69 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -20,7 +20,7 @@ private[io] final class BufReader(buf: Buf, chunkSize: Int) extends Reader[Buf] } final def discard(): Unit = synchronized { - state = Throw(new Reader.ReaderDiscarded) + state = Throw(new ReaderDiscardedException) } } diff --git a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala index 056b16358c..441e8e3742 100644 --- a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala +++ b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala @@ -37,7 +37,7 @@ class InputStreamReader private[io] ( */ def read(n: Int): Future[Option[Buf]] = { if (discarded) - return Future.exception(new Reader.ReaderDiscarded()) + return Future.exception(new ReaderDiscardedException()) if (n == 0) return Future.value(Some(Buf.Empty)) @@ -45,7 +45,7 @@ class InputStreamReader private[io] ( pool { try { if (discarded) - throw new Reader.ReaderDiscarded() + throw new ReaderDiscardedException() val buffer = new Array[Byte](chunkSize) val c = inputStream.read(buffer, 0, chunkSize) if (c == -1) { diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 7eb2c71ce1..6fe5d5bbc0 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -130,7 +130,7 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { result } - def discard(): Unit = fail(new Reader.ReaderDiscarded()) + def discard(): Unit = fail(new ReaderDiscardedException()) def write(buf: A): Future[Unit] = { val (waiter, value, result) = synchronized { diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 4480adeb19..faea0fdea7 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -95,9 +95,12 @@ trait Reader[+A] { def discard(): Unit } -object Reader { +/** + * Indicates that a given stream was discarded by the Reader's consumer. + */ +class ReaderDiscardedException extends Exception("Reader's consumer has discarded the stream") - class ReaderDiscarded extends Exception("This writer's reader has been discarded") +object Reader { val Null: Reader[Nothing] = new Reader[Nothing] { def read(n: Int): Future[Option[Nothing]] = Future.None @@ -292,7 +295,7 @@ object Reader { // // The computation r.read(..) will be interupted because we set an // interrupt handler in Reader.copy to discard `r`. - f.raise(new Reader.ReaderDiscarded()) + f.raise(new ReaderDiscardedException()) target.discard() } } diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index fdd69dd5ad..772c7ed915 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -28,7 +28,9 @@ class BufReaderTest extends FunSuite with Checkers { n < 0 || bytes.length == 0 || Await - .ready(r.read(n), 5.seconds).poll.exists(_.throwable.isInstanceOf[Reader.ReaderDiscarded]) + .ready(r.read(n), 5.seconds).poll.exists( + _.throwable.isInstanceOf[ReaderDiscardedException] + ) } } } diff --git a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala index 4a28e46555..67c5966baf 100644 --- a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala @@ -98,7 +98,7 @@ class InputStreamReaderTest extends FunSuite { r.discard() val f2 = r.read(10) - intercept[Reader.ReaderDiscarded] { Await.result(f2, 5.seconds) } + intercept[ReaderDiscardedException] { Await.result(f2, 5.seconds) } } test("discard closes InputStream") { diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index 5a046b7595..e550209295 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -1,7 +1,7 @@ package com.twitter.io import com.twitter.conversions.time._ -import com.twitter.io.Reader.ReaderDiscarded +import com.twitter.io.ReaderDiscardedException import com.twitter.util.{Await, Future, Return} import org.scalatest.{FunSuite, Matchers} @@ -144,7 +144,7 @@ class PipeTest extends FunSuite with Matchers { val rf = rw.read(10) assert(rf.isDefined) assert(closed) - intercept[Reader.ReaderDiscarded] { await(rf) } + intercept[ReaderDiscardedException] { await(rf) } } test("close") { @@ -303,7 +303,7 @@ class PipeTest extends FunSuite with Matchers { val rf = rw.read(1) rw.discard() - intercept[ReaderDiscarded] { + intercept[ReaderDiscardedException] { await(rf) } } @@ -314,7 +314,7 @@ class PipeTest extends FunSuite with Matchers { val wf = rw.write(buf(0, 1)) rw.discard() - intercept[ReaderDiscarded] { + intercept[ReaderDiscardedException] { await(wf) } } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index f5caa35293..78ddf74fc3 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -101,7 +101,7 @@ class ReaderTest val p = new Promise[Option[Buf]] val head = new Reader[Buf] { def read(n: Int): Future[Option[Buf]] = p - def discard(): Unit = p.setException(new Reader.ReaderDiscarded) + def discard(): Unit = p.setException(new ReaderDiscardedException) } val reader = Reader.concat(head +:: undefined) reader.discard() @@ -112,7 +112,7 @@ class ReaderTest val p = new Promise[Option[Buf]] val head = new Reader[Buf] { def read(n: Int): Future[Option[Buf]] = p - def discard(): Unit = p.setException(new Reader.ReaderDiscarded) + def discard(): Unit = p.setException(new ReaderDiscardedException) } val reader = Reader.concat(head +:: undefined) assertReadWhileReading(reader) @@ -122,7 +122,7 @@ class ReaderTest val p = new Promise[Option[Buf]] val head = new Reader[Buf] { def read(n: Int): Future[Option[Buf]] = p - def discard(): Unit = p.setException(new Reader.ReaderDiscarded) + def discard(): Unit = p.setException(new ReaderDiscardedException) } val reader = Reader.concat(head +:: undefined) assertFailed(reader, p) From 34ef850bac70ad3564653bbfe88d7660530f8051 Mon Sep 17 00:00:00 2001 From: Daniel Schobel Date: Fri, 26 Oct 2018 03:30:08 +0000 Subject: [PATCH 051/796] util-core: fix flaky Base64LongTest Problem / Solution Fix a disconnect in Base64LongTest between the alphabet which was being used in the function under test and the one which was being generated. JIRA Issues: CSL-7179 Differential Revision: https://phabricator.twitter.biz/D233915 --- .../src/test/scala/com/twitter/util/Base64LongTest.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala index 09fb65a9ee..b231b3e454 100644 --- a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala +++ b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala @@ -1,12 +1,10 @@ package com.twitter.util import com.twitter.util.Base64Long.{fromBase64, StandardBase64Alphabet, toBase64} -import org.junit.runner.RunWith import org.scalacheck.Arbitrary import org.scalacheck.Arbitrary.arbitrary import org.scalacheck.Gen import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks object Base64LongTest { @@ -54,7 +52,6 @@ object Base64LongTest { } } -@RunWith(classOf[JUnitRunner]) class Base64LongTest extends FunSuite with GeneratorDrivenPropertyChecks { import Base64LongTest._ @@ -116,7 +113,9 @@ class Base64LongTest extends FunSuite with GeneratorDrivenPropertyChecks { test("fromBase64 throws an IllegalArgumentException exception for characters out of range") { forAll { (s: String, a: Alphabet) => if (s.exists(!a.isDefinedAt(_))) { - assertThrows[IllegalArgumentException](fromBase64(s)) + assertThrows[IllegalArgumentException]( + fromBase64(s, 0, s.length, Base64Long.invertAlphabet(a)) + ) } } } From 6a8bc2238d44505bf317c338aaef5c2c998bc413 Mon Sep 17 00:00:00 2001 From: Nick Morgan Date: Fri, 26 Oct 2018 17:15:52 +0000 Subject: [PATCH 052/796] util-core: Return Some from Buf unapply methods that return Some Problem Custom `unapply` methods returning `Option` cause Scala to skip exhaustivity checking in pattern matching. Solution Modify the return types from `unapply` methods in com.twitter.io.Buf to return `Some` when the method is statically known to return `Some`. Result 1) Pattern matches containing the affected `unapply` methods will be checked for exhaustivity. E.g. val buf: com.twitter.io.Buf = ??? val opt: Option[String] = ??? (buf, opt) match { case (Buf.Utf8(str), Some(otherStr)) => str + otherStr } will now fail exhaustivity checking. 2) Code that was checking the `None` case from these `unapply` methods will now warn about unreachable code: val buf: com.twitter.io.Buf = ??? buf match { case Buf.Utf8(str) => str case _ => "" } Differential Revision: https://phabricator.twitter.biz/D229218 --- .../src/main/scala/com/twitter/io/Buf.scala | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 97dbcafa19..56a5586cd6 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -557,7 +557,7 @@ object Buf { } object Composite { - def unapply(buf: Composite): Option[IndexedSeq[Buf]] = + def unapply(buf: Composite): Some[IndexedSeq[Buf]] = Some(buf.bufs) /** @@ -776,7 +776,7 @@ object Buf { def apply(bytes: Array[Byte]): Buf = apply(bytes, 0, bytes.length) /** Extract the buffer's underlying offsets and array of bytes. */ - def unapply(buf: ByteArray): Option[(Array[Byte], Int, Int)] = + def unapply(buf: ByteArray): Some[(Array[Byte], Int, Int)] = Some((buf.bytes, buf.begin, buf.end)) /** @@ -815,7 +815,7 @@ object Buf { def apply(bytes: Array[Byte]): Buf = apply(bytes, 0, bytes.length) /** Extract a copy of the buffer's underlying array of bytes. */ - def unapply(ba: ByteArray): Option[Array[Byte]] = Some(ba.copiedByteArray) + def unapply(ba: ByteArray): Some[Array[Byte]] = Some(ba.copiedByteArray) /** Get a copy of a a Buf's data as an array of bytes. */ def extract(buf: Buf): Array[Byte] = Buf.ByteArray.coerce(buf).copiedByteArray @@ -898,7 +898,7 @@ object Buf { object ByteBuffer { /** Extract a read-only view of the underlying [[java.nio.ByteBuffer]]. */ - def unapply(buf: ByteBuffer): Option[java.nio.ByteBuffer] = + def unapply(buf: ByteBuffer): Some[java.nio.ByteBuffer] = Some(buf.underlying.asReadOnlyBuffer) /** Coerce a generic buffer to a Buf.ByteBuffer, potentially without copying data. */ @@ -928,7 +928,7 @@ object Buf { else new ByteBuffer(bb) /** Extract the buffer's underlying [[java.nio.ByteBuffer]]. */ - def unapply(buf: ByteBuffer): Option[java.nio.ByteBuffer] = Some(buf.underlying) + def unapply(buf: ByteBuffer): Some[java.nio.ByteBuffer] = Some(buf.underlying) /** * Get a reference to a Buf's data as a ByteBuffer. @@ -1103,7 +1103,7 @@ object Buf { /** * A [[StringCoder]] for a given [[java.nio.charset.Charset]] provides an * [[apply(String) encoder]]: `String` to [[Buf]] - * and an [[unapply(Buf) extractor]]: [[Buf]] to `Option[String]`. + * and an [[unapply(Buf) extractor]]: [[Buf]] to `Some[String]`. * * @note Malformed and unmappable input is silently replaced * see [[java.nio.charset.CodingErrorAction.REPLACE]] @@ -1125,10 +1125,9 @@ object Buf { * @note This extractor does *not* return None to indicate a failed * or impossible decoding. Malformed or unmappable bytes will * instead be silently replaced by the replacement character - * ("\uFFFD") in the returned String. This behavior may change - * in the future. + * ("\uFFFD") in the returned String. */ - def unapply(buf: Buf): Option[String] = Some(decodeString(buf, charset)) + def unapply(buf: Buf): Some[String] = Some(decodeString(buf, charset)) } /** From 5d020f2b7f3feb906b4debbc29382fa5c788c398 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Fri, 26 Oct 2018 18:48:19 +0000 Subject: [PATCH 053/796] util-core: clarify mapConcurrent ordering semantics Problem / Solution Make it clearer that mapConcurrent does not preserve order of the stream. JIRA Issues: CSL-7141 Differential Revision: https://phabricator.twitter.biz/D233747 --- .../com/twitter/concurrent/AsyncStream.scala | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 153b3359af..b6c25590a0 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -114,17 +114,19 @@ sealed abstract class AsyncStream[+A] { } /** - * Map over this stream with the given concurrency. The items will - * likely not be processed in order. `concurrencyLevel` specifies an - * "eagerness factor", and that many actions will be started when this - * method is called. Forcing the stream will yield the results of - * completed actions, and will block if none of the actions has yet - * completed. - * - * This method is useful for speeding up calculations over a stream - * where executing the actions in order is not important. To implement - * a concurrent fold, first call `mapConcurrent` and then fold that - * stream. Similarly, concurrent `foreachF` can be achieved by + * Map over this stream with the given concurrency. The items will likely be + * completed out of order, and if so, the stream of results will also be out + * of order. + * + * `concurrencyLevel` specifies an "eagerness factor", and that many actions + * will be started when this method is called. Forcing the stream will yield + * the results of completed actions, returning the results that finished + * first, and will block if none of the actions has yet completed. + * + * This method is useful for speeding up calculations over a stream where + * executing the actions (and processing their results) in order is not + * important. To implement a concurrent fold, first call `mapConcurrent` and + * then fold that stream. Similarly, concurrent `foreachF` can be achieved by * applying `mapConcurrent` and then `foreach`. * * @param concurrencyLevel: How many actions to execute concurrently. This From e670a087e0452527e77f9c468e5511174b29a3fb Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Sat, 27 Oct 2018 03:34:24 +0000 Subject: [PATCH 054/796] source: Reader.read() no longer takes size CSL is changing the Reader API such that it's more high-level (think polymorphic; has `flatMap`, etc). One of the conditions for this is dropping the `n` argument in `Reader.read(n)` such that every read returns all available bytes instead of `n`. JIRA Issues: CSL-6689 TBR=true Differential Revision: https://phabricator.twitter.biz/D228385 --- CHANGELOG.rst | 3 + .../concurrent/ConduitSpscBenchmark.scala | 4 +- .../scala/com/twitter/io/PipeBenchmark.scala | 4 +- .../src/main/java/com/twitter/io/Readers.java | 14 --- .../main/scala/com/twitter/io/BufReader.scala | 4 +- .../com/twitter/io/InputStreamReader.scala | 4 +- .../src/main/scala/com/twitter/io/Pipe.scala | 38 +++----- .../main/scala/com/twitter/io/Reader.scala | 75 ++++----------- .../twitter/io/exp/MinimumThroughput.scala | 6 +- .../scala/com/twitter/io/BufReaderTest.scala | 2 +- .../twitter/io/InputStreamReaderTest.scala | 31 +++--- .../test/scala/com/twitter/io/PipeTest.scala | 96 ++++--------------- .../scala/com/twitter/io/ReaderTest.scala | 65 ++++++++----- .../io/exp/MinimumThroughputTest.scala | 20 ++-- 14 files changed, 130 insertions(+), 236 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 09f2963ae9..46442a6e58 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -36,6 +36,9 @@ Breaking API Changes * util-core: `c.t.concurrent.AsyncStream.fromReader` has been moved to `c.t.io.Reader.toAsyncStream`. ``PHAB_ID=D228277`` +* util-core: `c.t.io.Reader.read()` no longer takes `n`, the maximum number of bytes to read off a + stream. ``PHAB_ID=D228385`` + New Features ~~~~~~~~~~~~ diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/ConduitSpscBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/ConduitSpscBenchmark.scala index 5c2f175bbf..ed43f656d1 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/ConduitSpscBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/ConduitSpscBenchmark.scala @@ -111,8 +111,8 @@ class ConduitSpscBenchmark extends StdBenchAnnotations { else source().flatMap(r.write) before feedReader(n - 1) private[this] def consumeReader(n: Int): Future[Boolean] = - if (n <= 1) r.read(Int.MaxValue).flatMap(sink) - else r.read(Int.MaxValue).flatMap(sink).flatMap(_ => consumeReader(n - 1)) + if (n <= 1) r.read().flatMap(sink) + else r.read().flatMap(sink).flatMap(_ => consumeReader(n - 1)) @Benchmark def reader: Boolean = diff --git a/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala index f620c131f7..cd58eca4e1 100644 --- a/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala @@ -15,13 +15,13 @@ class PipeBenchmark extends StdBenchAnnotations { def writeAndRead(): Future[Option[Buf]] = { val p = new Pipe[Buf] p.write(foo) - p.read(Int.MaxValue) + p.read() } @Benchmark def readAndWrite(): Future[Unit] = { val p = new Pipe[Buf] - p.read(Int.MaxValue) + p.read() p.write(foo) } } diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index 99d1bfad15..43641b9af5 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -54,13 +54,6 @@ public static Future copy(Reader r, Writer w) { return Reader$.MODULE$.copy(r, w); } - /** - * See {@code com.twitter.io.Reader.copy}. - */ - public static Future copy(Reader r, Writer w, int readSize) { - return Reader$.MODULE$.copy(r, w, readSize); - } - /** * See {@code com.twitter.io.Reader.copyMany}. */ @@ -68,13 +61,6 @@ public static Future copyMany(AsyncStream> readers, Write return Reader$.MODULE$.copyMany(readers, w); } - /** - * See {@code com.twitter.io.Reader.copyMany}. - */ - public static Future copyMany(AsyncStream> readers, Writer w, int readSize) { - return Reader$.MODULE$.copyMany(readers, w, readSize); - } - /** * See {@code com.twitter.io.Reader.fromFile()}. */ diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index e1773d8c69..a652398ce4 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -8,7 +8,7 @@ import com.twitter.util.{Future, Return, Try, Throw} private[io] final class BufReader(buf: Buf, chunkSize: Int) extends Reader[Buf] { private[this] var state: Try[Buf] = Return(buf) - final def read(n: Int): Future[Option[Buf]] = synchronized { + def read(): Future[Option[Buf]] = synchronized { state match { case Return(Buf.Empty) => Future.None case Return(b) => @@ -19,7 +19,7 @@ private[io] final class BufReader(buf: Buf, chunkSize: Int) extends Reader[Buf] } } - final def discard(): Unit = synchronized { + def discard(): Unit = synchronized { state = Throw(new ReaderDiscardedException) } } diff --git a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala index 441e8e3742..500f2cc35c 100644 --- a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala +++ b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala @@ -35,11 +35,9 @@ class InputStreamReader private[io] ( * * @note the underlying `InputStream` is closed on read of EOF. */ - def read(n: Int): Future[Option[Buf]] = { + def read(): Future[Option[Buf]] = { if (discarded) return Future.exception(new ReaderDiscardedException()) - if (n == 0) - return Future.value(Some(Buf.Empty)) mutex.acquire().flatMap { permit => pool { diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 6fe5d5bbc0..3c438f2f7d 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -71,7 +71,7 @@ import com.twitter.util.{Future, Promise, Return, Time} * - Closing an idle pipe returns a future that will be satisfied when a consumer observes the * closure (EOF) via read */ -final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { +final class Pipe[A] extends Reader[A] with Writer[A] { // Implementation Notes: // @@ -93,7 +93,7 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { // satisfied when a `read` observes the EOF (after calling close()) private[this] val closep: Promise[Unit] = new Promise[Unit]() - def read(n: Int): Future[Option[A]] = { + def read(): Future[Option[A]] = { val (waiter, result) = synchronized { state match { case State.Failed(exc) => @@ -108,20 +108,15 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { case State.Idle => val p = new Promise[Option[A]] - state = State.Reading(n, p) + state = State.Reading(p) (null, p) - case State.Writing(buf, p) if buf.length <= n => + case State.Writing(buf, p) => // pending write can fully fit into this requested read state = State.Idle (p, Future.value(Some(buf))) - case State.Writing(buf, p) => - // pending write is larger than the requested read - state = State.Writing(buf.slice(n, buf.length).asInstanceOf[A], p) - (null, Future.value(Some(buf.slice(0, n).asInstanceOf[A]))) - - case State.Reading(_, _) => + case State.Reading(_) => (null, Future.exception(new IllegalStateException("read() while read is pending"))) } } @@ -146,13 +141,7 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { state = State.Writing(buf, p) (null, null, p) - case State.Reading(n, p) if n < buf.length => - // pending reader doesn't have enough space for this write - val nextp = new Promise[Unit] - state = State.Writing(buf.slice(n, buf.length).asInstanceOf[A], nextp) - (p, Some(buf.slice(0, n).asInstanceOf[A]), nextp) - - case State.Reading(n, p) => + case State.Reading(p) => // pending reader has enough space for the full write state = State.Idle (p, Some(buf), Future.Done) @@ -180,7 +169,7 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { case State.Idle | State.Closing => state = State.Failed(cause) (closep, null, null) - case State.Reading(_, p) => + case State.Reading(p) => state = State.Failed(cause) (closep, p, null) case State.Writing(_, p) => @@ -205,7 +194,7 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { state = State.Closing (null, null, null) - case State.Reading(_, p) => + case State.Reading(p) => state = State.Closed (closep, p, null) @@ -232,7 +221,7 @@ final class Pipe[A <: Buf] extends Reader[A] with Writer[A] { object Pipe { - private sealed trait State[+A <: Buf] + private sealed trait State[+A] private object State { @@ -242,18 +231,17 @@ object Pipe { /** * Indicates a read is pending and is awaiting a `write`. * - * @param n number of bytes to read. * @param p when satisfied it indicates that this read has completed. */ - final case class Reading[A <: Buf](n: Int, p: Promise[Option[A]]) extends State[A] + final case class Reading[A](p: Promise[Option[A]]) extends State[A] /** - * Indicates a write of `buf` is pending to be `read`. + * Indicates a write of `value` is pending to be `read`. * - * @param buf the [[Buf]] to write. + * @param value the value to write. * @param p when satisfied it indicates that this write has been fully read. */ - final case class Writing[A <: Buf](buf: A, p: Promise[Unit]) extends State[A] + final case class Writing[A](value: A, p: Promise[Unit]) extends State[A] /** Indicates the pipe was failed. */ final case class Failed(exc: Throwable) extends State[Nothing] diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index faea0fdea7..55d33d6fc9 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -83,7 +83,7 @@ trait Reader[+A] { * * Stream failures are terminal such that all subsequent reads will resolve in failed [[Future]]s. */ - def read(n: Int): Future[Option[A]] + def read(): Future[Option[A]] /** * Discard this stream as its output is no longer required. This could be used to signal the @@ -103,7 +103,7 @@ class ReaderDiscardedException extends Exception("Reader's consumer has discarde object Reader { val Null: Reader[Nothing] = new Reader[Nothing] { - def read(n: Int): Future[Option[Nothing]] = Future.None + def read(): Future[Option[Nothing]] = Future.None def discard(): Unit = () } @@ -141,20 +141,20 @@ object Reader { in match { case Some(data) => frames = framer(data) - read(Int.MaxValue) + read() case None => Future.None } } - def read(n: Int): Future[Option[Buf]] = synchronized { + def read(): Future[Option[Buf]] = synchronized { frames match { case nextFrame :: rst => frames = rst Future.value(Some(nextFrame)) case _ => // flatMap to `this` to prevent allocating - r.read(Int.MaxValue).flatMap(this) + r.read().flatMap(this) } } @@ -169,7 +169,7 @@ object Reader { */ def readAll(r: Reader[Buf]): Future[Buf] = { def loop(left: Buf): Future[Buf] = - r.read(Int.MaxValue).flatMap { + r.read().flatMap { case Some(right) => loop(left concat right) case _ => Future.value(left) } @@ -180,9 +180,6 @@ object Reader { /** * Chunk the output of a given [[Reader]] by at most `chunkSize` (bytes). This consumes the * reader. - * - * @note The `n` (number of bytes to read) argument on the returned reader is ignored - * (`Int.MaxValue` is used instead). */ def chunked(r: Reader[Buf], chunkSize: Int): Reader[Buf] = new Framed(r, new ChunkedFramer(chunkSize)) @@ -190,8 +187,6 @@ object Reader { /** * Create a new [[Reader]] from a given [[Buf]]. The output of a returned reader is chunked by * a least `chunkSize` (bytes). - * - * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. */ def fromBuf(buf: Buf): Reader[Buf] = fromBuf(buf, Int.MaxValue) @@ -210,7 +205,6 @@ object Reader { * The resources held by the returned [[Reader]] are released on reading of EOF and * [[Reader.discard()]]. * - * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. * @see `Readers.fromFile` for a Java API */ def fromFile(f: File): Reader[Buf] = fromFile(f, InputStreamReader.DefaultMaxBufferSize) @@ -222,7 +216,6 @@ object Reader { * The resources held by the returned [[Reader]] are released on reading of EOF and * [[Reader.discard()]]. * - * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. * @see `Readers.fromFile` for a Java API */ def fromFile(f: File, chunkSize: Int): Reader[Buf] = fromStream(new FileInputStream(f), chunkSize) @@ -233,9 +226,6 @@ object Reader { * * The resources held by the returned [[Reader]] are released on reading of EOF and * [[Reader.discard()]]. - * - * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. - * @see `Readers.fromStream` for a Java API */ def fromStream(s: InputStream): Reader[Buf] = fromStream(s, InputStreamReader.DefaultMaxBufferSize) @@ -247,7 +237,6 @@ object Reader { * The resources held by the returned [[Reader]] are released on reading of EOF and * [[Reader.discard()]]. * - * @note The `n` (number of bytes to read) argument on the returned reader's `read` is ignored. * @see `Readers.fromStream` for a Java API */ def fromStream(s: InputStream, chunkSize: Int): Reader[Buf] = InputStreamReader(s, chunkSize) @@ -255,8 +244,8 @@ object Reader { /** * Allow [[AsyncStream]] to be consumed as a [[Reader]] */ - def fromAsyncStream[A <: Buf](as: AsyncStream[A]): Reader[A] = { - val pipe = new Pipe[A]() + def fromAsyncStream[A](as: AsyncStream[A]): Reader[A] = { + val pipe = new Pipe[A] // orphan the Future but allow it to clean up // the Pipe IF the stream ever finishes or fails as.foreachF(pipe.write).respond { @@ -269,9 +258,9 @@ object Reader { /** * Transformation (or lift) from [[Reader]] into `AsyncStream`. */ - def toAsyncStream[A <: Buf](r: Reader[A], chunkSize: Int = Int.MaxValue): AsyncStream[A] = - AsyncStream.fromFuture(r.read(chunkSize)).flatMap { - case Some(buf) => buf +:: Reader.toAsyncStream(r, chunkSize) + def toAsyncStream[A](r: Reader[A]): AsyncStream[A] = + AsyncStream.fromFuture(r.read()).flatMap { + case Some(buf) => buf +:: Reader.toAsyncStream(r) case None => AsyncStream.empty[A] } @@ -279,14 +268,14 @@ object Reader { * Convenient abstraction to read from a stream of Readers as if it were a * single Reader. */ - def concat(readers: AsyncStream[Reader[Buf]]): Reader[Buf] = { - val target = new Pipe[Buf]() + def concat[A](readers: AsyncStream[Reader[A]]): Reader[A] = { + val target = new Pipe[A] val f = copyMany(readers, target).respond { case Throw(exc) => target.fail(exc) case _ => target.close() } - new Reader[Buf] { - def read(n: Int): Future[Option[Buf]] = target.read(n) + new Reader[A] { + def read(): Future[Option[A]] = target.read() def discard(): Unit = { // We have to do this so that when the the target is discarded we can // interrupt the read operation. Consider the following: @@ -301,19 +290,6 @@ object Reader { } } - /** - * Copy bytes from many Readers to a Writer. The Writer is unmanaged, the - * caller is responsible for finalization and error handling, e.g.: - * - * {{{ - * Reader.copyMany(readers, writer) ensure writer.close() - * }}} - * - * @param bufsize The number of bytes to read each time. - */ - def copyMany(readers: AsyncStream[Reader[Buf]], target: Writer[Buf], bufsize: Int): Future[Unit] = - readers.foreachF(Reader.copy(_, target, bufsize)) - /** * Copy bytes from many Readers to a Writer. The Writer is unmanaged, the * caller is responsible for finalization and error handling, e.g.: @@ -322,8 +298,8 @@ object Reader { * Reader.copyMany(readers, writer) ensure writer.close() * }}} */ - def copyMany(readers: AsyncStream[Reader[Buf]], target: Writer[Buf]): Future[Unit] = - copyMany(readers, target, Writer.BufferSize) + def copyMany[A](readers: AsyncStream[Reader[A]], target: Writer[A]): Future[Unit] = + readers.foreachF(Reader.copy(_, target)) /** * Copy the bytes from a Reader to a Writer in chunks of size `n`. The Writer @@ -333,12 +309,10 @@ object Reader { * {{{ * Reader.copy(r, w, n) ensure w.close() * }}} - * - * @param n The number of bytes to read on each refill of the Writer. */ - def copy(r: Reader[Buf], w: Writer[Buf], n: Int): Future[Unit] = { + def copy[A](r: Reader[A], w: Writer[A]): Future[Unit] = { def loop(): Future[Unit] = - r.read(n).flatMap { + r.read().flatMap { case None => Future.Done case Some(buf) => w.write(buf) before loop() } @@ -350,17 +324,6 @@ object Reader { p } - /** - * Copy the bytes from a Reader to a Writer in chunks of size - * `Writer.BufferSize`. The Writer is unmanaged, the caller is responsible - * for finalization and error handling, e.g.: - * - * {{{ - * Reader.copy(r, w) ensure w.close() - * }}} - */ - def copy(r: Reader[Buf], w: Writer[Buf]): Future[Unit] = copy(r, w, Writer.BufferSize) - /** * Wraps a [[ Reader[Buf] ]] and emits frames as decided by `framer`. * diff --git a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala b/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala index 6e63ae6695..65c47fcc6a 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala +++ b/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala @@ -62,12 +62,12 @@ object MinimumThroughput { with Reader[Buf] { def discard(): Unit = reader.discard() - def read(n: Int): Future[Option[Buf]] = { - val deadline = nextDeadline(n) + def read(): Future[Option[Buf]] = { + val deadline = nextDeadline(0) val start = Time.now val read: Future[Option[Buf]] = - reader.read(n).raiseWithin(timer, deadline, MinThroughputTimeoutException) + reader.read().raiseWithin(timer, deadline, MinThroughputTimeoutException) read.transform { res => elapsed += Time.now - start diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index 772c7ed915..f098c2dc8d 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -28,7 +28,7 @@ class BufReaderTest extends FunSuite with Checkers { n < 0 || bytes.length == 0 || Await - .ready(r.read(n), 5.seconds).poll.exists( + .ready(r.read(), 5.seconds).poll.exists( _.throwable.isInstanceOf[ReaderDiscardedException] ) } diff --git a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala index 67c5966baf..7cb4a42df5 100644 --- a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala @@ -18,34 +18,25 @@ class InputStreamReaderTest extends FunSuite { val s = new ByteArrayInputStream(a) val r = new InputStreamReader(s, 4096) - val f = r.read(10) + val f = r.read() assert(Await.result(f, 5.seconds).isEmpty) } - test("InputStreamReader - read 0 bytes") { - val a = arr(0, 25) - val s = new ByteArrayInputStream(a) - val r = new InputStreamReader(s, 4096) - - val f1 = r.read(0) - assert(Await.result(f1, 5.seconds).get.isEmpty) - } - test("InputStreamReader - read positive bytes") { val a = arr(0, 25) val s = new ByteArrayInputStream(a) val r = new InputStreamReader(s, 10) - val f1 = r.read(10) + val f1 = r.read() assert(Await.result(f1, 5.seconds) == Some(buf(0, 10))) - val f2 = r.read(10) + val f2 = r.read() assert(Await.result(f2, 5.seconds) == Some(buf(10, 20))) - val f3 = r.read(10) + val f3 = r.read() assert(Await.result(f3, 5.seconds) == Some(buf(20, 25))) - val f4 = r.read(10) + val f4 = r.read() assert(Await.result(f4).isEmpty) } @@ -54,16 +45,16 @@ class InputStreamReaderTest extends FunSuite { val s = new ByteArrayInputStream(a) val r = new InputStreamReader(s, 100) - val f1 = r.read(1000) + val f1 = r.read() assert(Await.result(f1, 5.seconds) == Some(buf(0, 100))) - val f2 = r.read(1000) + val f2 = r.read() assert(Await.result(f2, 5.seconds) == Some(buf(100, 200))) - val f3 = r.read(1000) + val f3 = r.read() assert(Await.result(f3, 5.seconds) == Some(buf(200, 250))) - val f4 = r.read(1000) + val f4 = r.read() assert(Await.result(f4, 5.seconds).isEmpty) } @@ -92,12 +83,12 @@ class InputStreamReaderTest extends FunSuite { val s = new ByteArrayInputStream(a) val r = new InputStreamReader(s, 10) - val f1 = r.read(10) + val f1 = r.read() assert(Await.result(f1, 5.seconds) == Some(buf(0, 10))) r.discard() - val f2 = r.read(10) + val f2 = r.read() intercept[ReaderDiscardedException] { Await.result(f2, 5.seconds) } } diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index e550209295..e13b7ccd2b 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -14,7 +14,7 @@ class PipeTest extends FunSuite with Matchers { private def assertRead(r: Reader[Buf], i: Int, j: Int): Unit = { val n = j - i - val f = r.read(n) + val f = r.read() assertRead(f, i, j) } @@ -51,7 +51,7 @@ class PipeTest extends FunSuite with Matchers { } private def assertReadNone(r: Reader[Buf]): Unit = - assert(await(r.read(1)).isEmpty) + assert(await(r.read()).isEmpty) private val failedEx = new RuntimeException("ʕ •ᴥ•ʔ") @@ -66,10 +66,9 @@ class PipeTest extends FunSuite with Matchers { val rw = new Pipe[Buf] val wf = rw.write(buf(0, 6)) assert(!wf.isDefined) - assertRead(rw, 0, 3) - assert(!wf.isDefined) - assertRead(rw, 3, 6) + assertRead(rw, 0, 6) assert(wf.isDefined) + assert(await(wf.liftToTry) == Return(())) } test("Reader.readAll") { @@ -91,33 +90,16 @@ class PipeTest extends FunSuite with Matchers { val rw = new Pipe[Buf] val wf = rw.write(buf(0, 6)) assert(!wf.isDefined) - val rf = rw.read(6) + val rf = rw.read() assert(rf.isDefined) assert(toSeq(await(rf)) == Seq.range(0, 6)) } - test("partial read, then short read") { - val rw = new Pipe[Buf] - val wf = rw.write(buf(0, 6)) - assert(!wf.isDefined) - val rf = rw.read(4) - assert(rf.isDefined) - assert(toSeq(await(rf)) == Seq.range(0, 4)) - - assert(!wf.isDefined) - val rf2 = rw.read(4) - assert(rf2.isDefined) - assert(toSeq(await(rf2)) == Seq.range(4, 6)) - - assert(wf.isDefined) - assert(await(wf.liftToTry) == Return(())) - } - test("fail while reading") { val rw = new Pipe[Buf] var closed = false rw.onClose.ensure { closed = true } - val rf = rw.read(6) + val rf = rw.read() assert(!rf.isDefined) assert(!closed) val exc = new Exception @@ -131,7 +113,7 @@ class PipeTest extends FunSuite with Matchers { test("fail before reading") { val rw = new Pipe[Buf] rw.fail(new Exception) - val rf = rw.read(10) + val rf = rw.read() assert(rf.isDefined) intercept[Exception] { await(rf) } } @@ -141,7 +123,7 @@ class PipeTest extends FunSuite with Matchers { var closed = false rw.onClose.ensure { closed = true } rw.discard() - val rf = rw.read(10) + val rf = rw.read() assert(rf.isDefined) assert(closed) intercept[ReaderDiscardedException] { await(rf) } @@ -154,7 +136,7 @@ class PipeTest extends FunSuite with Matchers { val wf = rw.write(buf(0, 6)) before rw.close() assert(!wf.isDefined) assert(!closed) - assert(await(rw.read(6)) == Some(buf(0, 6))) + assert(await(rw.read()).contains(buf(0, 6))) assert(!wf.isDefined) assertReadEofAndClosed(rw) assert(closed) @@ -165,9 +147,7 @@ class PipeTest extends FunSuite with Matchers { val wf = rw.write(buf(0, 6)) assert(!wf.isDone) - assertRead(rw, 0, 3) - assert(!wf.isDone) - assertRead(rw, 3, 6) + assertRead(rw, 0, 6) assert(wf.isDone) assert(!rw.close().isDone) @@ -177,7 +157,7 @@ class PipeTest extends FunSuite with Matchers { test("read then write then close") { val rw = new Pipe[Buf] - val rf = rw.read(6) + val rf = rw.read() assert(!rf.isDefined) val wf = rw.write(buf(0, 6)) @@ -196,7 +176,7 @@ class PipeTest extends FunSuite with Matchers { val cf = rw.close() assert(!cf.isDone) - assertFailedEx(rw.read(1)) + assertFailedEx(rw.read()) assertFailedEx(cf) } @@ -212,37 +192,6 @@ class PipeTest extends FunSuite with Matchers { } } - test("write smaller buf than read is waiting for") { - val rw = new Pipe[Buf] - val rf = rw.read(6) - assert(!rf.isDefined) - - val wf = rw.write(buf(0, 5)) - assert(wf.isDone) - assertRead(rf, 0, 5) - - assert(!rw.read(1).isDefined) // nothing pending - rw.close() - assertReadEofAndClosed(rw) - } - - test("write larger buf than read is waiting for") { - val rw = new Pipe[Buf] - val rf = rw.read(3) - assert(!rf.isDefined) - - val wf = rw.write(buf(0, 6)) - assert(!wf.isDone) - assertRead(rf, 0, 3) - assert(!wf.isDone) - - assertRead(rw.read(5), 3, 6) // read the rest - assert(wf.isDone) - - assert(!rw.read(1).isDefined) // nothing pending to read - assert(rw.close().isDone) - } - test("write while write pending") { val rw = new Pipe[Buf] var closed = false @@ -262,7 +211,7 @@ class PipeTest extends FunSuite with Matchers { test("read after fail") { val rw = new Pipe[Buf] rw.fail(failedEx) - assertFailedEx(rw.read(1)) + assertFailedEx(rw.read()) } test("read after close with no pending reads") { @@ -289,9 +238,9 @@ class PipeTest extends FunSuite with Matchers { val rw = new Pipe[Buf] var closed = false rw.onClose.ensure { closed = true } - val rf = rw.read(1) + val rf = rw.read() intercept[IllegalStateException] { - await(rw.read(1)) + await(rw.read()) } assert(!rf.isDefined) assert(!closed) @@ -300,7 +249,7 @@ class PipeTest extends FunSuite with Matchers { test("discard with pending read") { val rw = new Pipe[Buf] - val rf = rw.read(1) + val rf = rw.read() rw.discard() intercept[ReaderDiscardedException] { @@ -324,17 +273,14 @@ class PipeTest extends FunSuite with Matchers { val cf = rw.write(buf(0, 6)).before(rw.close()) assert(!cf.isDone) - assertRead(rw, 0, 3) - assert(!cf.isDone) - - assertRead(rw, 3, 6) + assertRead(rw, 0, 6) assert(!cf.isDone) assertReadEofAndClosed(rw) } test("close not satisfied until reads are fulfilled") { val rw = new Pipe[Buf] - val rf = rw.read(6) + val rf = rw.read() val cf = rf.flatMap { _ => rw.close() } @@ -350,7 +296,7 @@ class PipeTest extends FunSuite with Matchers { test("close while read pending") { val rw = new Pipe[Buf] - val rf = rw.read(6) + val rf = rw.read() assert(!rf.isDefined) assert(rw.close().isDone) @@ -371,7 +317,7 @@ class PipeTest extends FunSuite with Matchers { val cf = rw.close() assert(!cf.isDone) - assertFailedEx(rw.read(1)) + assertFailedEx(rw.read()) assertFailedEx(cf) } @@ -383,7 +329,7 @@ class PipeTest extends FunSuite with Matchers { rw.fail(failedEx) assert(!cf.isDone) - assertFailedEx(rw.read(1)) + assertFailedEx(rw.read()) assertFailedEx(cf) } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 78ddf74fc3..d1d9f071a5 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -21,6 +21,7 @@ class ReaderTest with IntegrationPatience { private def arr(i: Int, j: Int) = Array.range(i, j).map(_.toByte) + private def buf(i: Int, j: Int) = Buf.ByteArray.Owned(arr(i, j)) private def await[T](t: Awaitable[T]): T = Await.result(t, 5.seconds) @@ -28,18 +29,26 @@ class ReaderTest def undefined: AsyncStream[Reader[Buf]] = throw new Exception private def assertReadWhileReading(r: Reader[Buf]): Unit = { - val f = r.read(1) - intercept[IllegalStateException] { await(r.read(1)) } + val f = r.read() + intercept[IllegalStateException] { + await(r.read()) + } assert(!f.isDefined) } private def assertFailed(r: Reader[Buf], p: Promise[Option[Buf]]): Unit = { - val f = r.read(1) + val f = r.read() assert(!f.isDefined) p.setException(new Exception) - intercept[Exception] { await(f) } - intercept[Exception] { await(r.read(0)) } - intercept[Exception] { await(r.read(1)) } + intercept[Exception] { + await(f) + } + intercept[Exception] { + await(r.read()) + } + intercept[Exception] { + await(r.read()) + } } test("Reader.copy - source and destination equality") { @@ -52,7 +61,9 @@ class ReaderTest val g = rw.write(Buf.ByteArray.Owned(p)).before { rw.write(Buf.ByteArray.Owned(q)).before { - rw.write(Buf.ByteArray.Owned(r)).before { rw.close() } + rw.write(Buf.ByteArray.Owned(r)).before { + rw.close() + } } } @@ -78,7 +89,7 @@ class ReaderTest case (s, i) => val r = Reader.chunked(Reader.fromBuf(Buf.Utf8(s), 32), i) - def readLoop(): Unit = await(r.read(Int.MaxValue)) match { + def readLoop(): Unit = await(r.read()) match { case Some(b) => assert(b.length <= i) readLoop() @@ -100,7 +111,7 @@ class ReaderTest test("Reader.concat - discard") { val p = new Promise[Option[Buf]] val head = new Reader[Buf] { - def read(n: Int): Future[Option[Buf]] = p + def read(): Future[Option[Buf]] = p def discard(): Unit = p.setException(new ReaderDiscardedException) } val reader = Reader.concat(head +:: undefined) @@ -111,7 +122,7 @@ class ReaderTest test("Reader.concat - read while reading") { val p = new Promise[Option[Buf]] val head = new Reader[Buf] { - def read(n: Int): Future[Option[Buf]] = p + def read(): Future[Option[Buf]] = p def discard(): Unit = p.setException(new ReaderDiscardedException) } val reader = Reader.concat(head +:: undefined) @@ -121,7 +132,7 @@ class ReaderTest test("Reader.concat - failed") { val p = new Promise[Option[Buf]] val head = new Reader[Buf] { - def read(n: Int): Future[Option[Buf]] = p + def read(): Future[Option[Buf]] = p def discard(): Unit = p.setException(new ReaderDiscardedException) } val reader = Reader.concat(head +:: undefined) @@ -130,17 +141,22 @@ class ReaderTest test("Reader.concat - lazy tail") { val head = new Reader[Buf] { - def read(n: Int): Future[Option[Buf]] = Future.exception(new Exception) + def read(): Future[Option[Buf]] = Future.exception(new Exception) + def discard(): Unit = () } val p = new Promise[Unit] + def tail: AsyncStream[Reader[Buf]] = { p.setDone() AsyncStream.empty } + val combined = Reader.concat(head +:: tail) val buf = Reader.readAll(combined) - intercept[Exception] { await(buf) } + intercept[Exception] { + await(buf) + } assert(!p.isDefined) } @@ -172,31 +188,35 @@ class ReaderTest val as = AsyncStream.exception(new Exception()) val r = Reader.fromAsyncStream(as) val f = Reader.readAll(r) - intercept[Exception] { await(f) } + intercept[Exception] { + await(f) + } } test("Reader.fromAsyncStream only evaluates tail when buffer is exhausted") { val tailEvaluated = new AtomicBoolean(false) + def tail: AsyncStream[Buf] = { tailEvaluated.set(true) AsyncStream.empty } - val as = AsyncStream.mk(buf(0, 10), tail) + + val as = AsyncStream.mk(buf(0, 10), AsyncStream.mk(buf(10, 20), tail)) val r = Reader.fromAsyncStream(as) // partially read the buffer - await(r.read(9)) + await(r.read()) assert(!tailEvaluated.get()) // read the rest of the buffer - await(r.read(2)) + await(r.read()) assert(tailEvaluated.get()) } test("Reader.toAsyncStream") { forAll { l: List[Byte] => val buf = Buf.ByteArray.Owned(l.toArray) - val as = Reader.toAsyncStream(Reader.fromBuf(buf, 1), chunkSize = 1) + val as = Reader.toAsyncStream(Reader.fromBuf(buf, 1)) assert(await(as.toSeq()).map(b => Buf.ByteArray.Owned.extract(b).head) == l) } @@ -218,20 +238,19 @@ class ReaderTest // read all of the frames buffers.foreach { buf => - assert(await(r.read(Int.MaxValue)).contains(buf)) + assert(await(r.read()).contains(buf)) } // make sure the reader signals EOF - assert(await(r.read(Int.MaxValue)).isEmpty) + assert(await(r.read()).isEmpty) } } test("Reader.framed reads empty frames") { val r = Reader.framed(BufReader(Buf.U32BE(0)), new ReaderTest.U32BEFramer()) - assert(await(r.read(Int.MaxValue)).contains(Buf.Empty)) - assert(await(r.read(Int.MaxValue)).isEmpty) + assert(await(r.read()).contains(Buf.Empty)) + assert(await(r.read()).isEmpty) } - } object ReaderTest { diff --git a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala b/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala index f6f4bc96f7..5815c94fa9 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala +++ b/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala @@ -23,12 +23,12 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { val reader = MinimumThroughput.reader(Reader.fromBuf(buf, 8), 0d, Timer.Nil) // read from the beginning - Await.result(reader.read(0)) match { + Await.result(reader.read()) match { case Some(b) => assert(b == buf.slice(0, 8)) case _ => fail() } - Await.result(reader.read(0)) match { + Await.result(reader.read()) match { case Some(b) => assert(b == buf.slice(8, 16)) case _ => fail() } @@ -43,7 +43,7 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { val underlying = new Reader[Buf] { private var reads = 0 def discard(): Unit = () - def read(n: Int): Future[Option[Buf]] = { + def read(): Future[Option[Buf]] = { reads += 1 if (reads == 2) { // now take 10 seconds to read 1 more byte, @@ -61,14 +61,14 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { ) // do a read of 1 byte in 0 time — which is ok. - Await.result(reader.read(1)) match { + Await.result(reader.read()) match { case Some(b) => assert(b == buf.slice(0, 1)) case _ => fail() } val ex = intercept[BelowThroughputException] { // note in the mock above, the 2nd read takes 10 seconds - Await.result(reader.read(1)) + Await.result(reader.read()) } assert(ex.elapsed == 10.seconds) assert(ex.expectedBps == 1d) @@ -78,7 +78,7 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { test("Reader - times out while reading") { val underlying = mock[Reader[Buf]] - when(underlying.read(1)) + when(underlying.read()) .thenReturn(Future.never) val timer = new MockTimer() @@ -89,7 +89,7 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { ) Time.withCurrentTimeFrozen { tc => - val f = reader.read(1) + val f = reader.read() tc.advance(10.seconds) timer.tick() @@ -105,7 +105,7 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { test("Reader - failures from underlying reader are untouched") { val ex = new RuntimeException("└[∵┌]└[ ∵ ]┘[┐∵]┘") val underlying = mock[Reader[Buf]] - when(underlying.read(1)) + when(underlying.read()) .thenReturn(Future.exception(ex)) val reader = MinimumThroughput.reader( @@ -115,7 +115,7 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { ) val thrown = intercept[RuntimeException] { - Await.result(reader.read(1)) + Await.result(reader.read()) } assert(thrown == ex) } @@ -127,7 +127,7 @@ class MinimumThroughputTest extends FunSuite with MockitoSugar { Timer.Nil ) - Await.result(reader.read(1)) match { + Await.result(reader.read()) match { case None => case _ => fail() } From ab669906baa4716b853c9beeeb2ce3817842f8ed Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Tue, 30 Oct 2018 02:43:04 +0000 Subject: [PATCH 055/796] util-core: simpler and faster mapConcurrent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem / Solution Simpler and faster mapConcurrent. Before [info] AsyncStreamBenchmark.mapConcurrent 10 avgt 10 33650.272 ± 796.158 ns/op [info] AsyncStreamBenchmark.mapConcurrent:·gc.alloc.rate.norm 10 avgt 10 46032.012 ± 0.019 B/op After [info] AsyncStreamBenchmark.mapConcurrent 10 avgt 10 18247.490 ± 3140.989 ns/op [info] AsyncStreamBenchmark.mapConcurrent:·gc.alloc.rate.norm 10 avgt 10 28144.006 ± 0.010 B/op Differential Revision: https://phabricator.twitter.biz/D234046 --- .../concurrent/AsyncStreamBenchmark.scala | 9 +- .../com/twitter/concurrent/AsyncStream.scala | 95 +++++-------------- .../twitter/concurrent/AsyncStreamTest.scala | 5 +- 3 files changed, 36 insertions(+), 73 deletions(-) diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala index d0dfc9844d..1b5f3e6115 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala @@ -1,7 +1,7 @@ package com.twitter.concurrent import com.twitter.conversions.time._ -import com.twitter.util.{Await, StdBenchAnnotations} +import com.twitter.util.{Await, Future, StdBenchAnnotations} import org.openjdk.jmh.annotations._ @State(Scope.Benchmark) @@ -47,6 +47,13 @@ class AsyncStreamBenchmark extends StdBenchAnnotations { def flatMap(): Seq[Int] = Await.result(as.flatMap(FlatMapFn).toSeq()) + private[this] val MapConcurrentFn: Int => Future[Int] = + x => Future.value(x + 1) + + @Benchmark + def mapConcurrent(): Seq[Int] = + Await.result(as.mapConcurrent(2)(MapConcurrentFn).toSeq()) + private[this] val FilterFn: Int => Boolean = x => x % 2 == 0 diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index b6c25590a0..7eecfb3d0d 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -1,6 +1,6 @@ package com.twitter.concurrent -import com.twitter.util.{Future, Promise, Return, Throw, Try} +import com.twitter.util.{Future, Return, Throw} import scala.annotation.varargs import scala.collection.mutable @@ -134,15 +134,8 @@ sealed abstract class AsyncStream[+A] { * the next item available in the stream. */ def mapConcurrent[B](concurrencyLevel: Int)(f: A => Future[B]): AsyncStream[B] = { - if (concurrencyLevel == 1) { - mapF(f) - } else if (concurrencyLevel < 1) { - throw new IllegalArgumentException( - s"concurrencyLevel must be at least one. got: $concurrencyLevel" - ) - } else { - embed(AsyncStream.mapConcStep(concurrencyLevel, f, Nil, () => this)) - } + require(concurrencyLevel > 0, s"concurrencyLevel must be at least one. got: $concurrencyLevel") + merge(fanout(this, concurrencyLevel).map(_.mapF(f)): _*) } /** @@ -719,65 +712,29 @@ object AsyncStream { } } - /** - * This must exist here, otherwise scalac will be greedy and hold a reference to the head of the stream - */ - private def mapConcStep[A, B]( - concurrencyLevel: Int, - f: (A => Future[B]), - pending: Seq[Future[Option[B]]], - inputs: () => AsyncStream[A] - ): Future[AsyncStream[B]] = { - // We only invoke inputs().uncons if there is space for more work - // to be started. - val inputReady: Future[Left[Option[(A, () => AsyncStream[A])], Nothing]] = - if (pending.size >= concurrencyLevel) Future.never - else - inputs().uncons.flatMap { - case None if pending.nonEmpty => Future.never - case other => Future.value(Left(other)) - } - - // The inputReady.isDefined check is an optimization to avoid the - // wasted allocation of calling Future.select when we know that - // there is more work that is ready to be started. - val workDone: Future[Right[Nothing, (Try[Option[B]], Seq[Future[Option[B]]])]] = - if (pending.isEmpty || inputReady.isDefined) Future.never - else Future.select(pending).map(Right(_)) - - // Wait for either the next input to be ready (Left) or for some pending - // work to complete (Right). - inputReady.or(workDone).flatMap { - case Left(None) => - // There is no work pending and we have exhausted the - // inputs, so we are done. - Future.value(empty) - - case Left(Some((a, tl))) => - // There is available concurrency and a new input is ready, - // so start the work and add it to `pending`. - mapConcStep(concurrencyLevel, f, f(a).map(Some(_)) +: pending, tl) - - case Right((Throw(t), _)) => - // Some work finished with failure, so terminate the stream. - Future.exception(t) - - case Right((Return(None), newPending)) => - // A cons cell was forced, freeing up a spot in `pending`. - mapConcStep(concurrencyLevel, f, newPending, inputs) - - case Right((Return(Some(a)), newPending)) => - // Some work finished. Eagerly start the next step, so - // that all available inputs have work started on them. In - // the next step, we replace the pending Future for the - // work that just completed with a Future that will be - // satisfied by forcing the next element of the result - // stream. Keeping `pending` full is the mechanism that we - // use to bound the evaluation depth at `concurrencyLevel` - // until the stream is forced. - val cellForced = new Promise[Option[B]] - val rest = mapConcStep(concurrencyLevel, f, cellForced +: newPending, inputs) - Future.value(mk(a, { cellForced.setValue(None); embed(rest) })) + // Oneshot turns an AsyncStream into a resource, where each read depletes the + // stream of that item. In other words, each item of the stream is observable + // only once. + private class Oneshot[A](var as: AsyncStream[A]) { + def read(): Future[Option[A]] = synchronized { + val head = as.head + as = as.drop(1) + head } + + // Recover the AsyncStream interface. We don't return `as` because we want + // to deplete the original stream with each read. With this we can create + // multiple AsyncStream views of the same Oneshot resource, thus "fanning + // out" the original stream into multiple distinct AsyncStreams. + def toAsyncStream: AsyncStream[A] = + AsyncStream.fromFuture(read()).flatMap { + case None => AsyncStream.empty + case Some(a) => a +:: toAsyncStream + } + } + + private def fanout[A](as: AsyncStream[A], n: Int): Seq[AsyncStream[A]] = { + val reader = new Oneshot(as) + 0.until(n).map(_ => reader.toAsyncStream) } } diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 490992a0fd..55d457c046 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -615,9 +615,8 @@ class AsyncStreamTest extends FunSuite with GeneratorDrivenPropertyChecks { // Make sure that all of the finished items are now // available. (As a side-effect, this will force more work to // be done if concurrency was the limiting factor.) - val completed = toSeq(result.take(workFinished)).sorted - val expectedCompleted = items.take(expectedFinished).sorted - assert(completed == expectedCompleted) + val completed = toSeq(result.take(workFinished)) + assert(completed.toSet.subsetOf(items.toSet)) } } From 886646eb73f79074bdbea066395f412b81366810 Mon Sep 17 00:00:00 2001 From: yic Date: Thu, 1 Nov 2018 20:46:51 +0000 Subject: [PATCH 056/796] util-core: move util-core-util's concurrent sources into a separate dir # Problem / Solution Portion of "util-core: Switch to 1:1:1 pants targets" commit introduced non-1:1:1 compliant target so code resolution in IntelliJ is breaking. Hence changing the directory structure so intellij is happy. The publishing structure should remain the same. JIRA Issues: DPB-10452 Differential Revision: https://phabricator.twitter.biz/D235472 --- build.sbt | 3 +++ util-core/BUILD | 25 +++++++++++++++++++ .../concurrent/NamedPoolThreadFactory.scala | 0 .../scala/com/twitter/concurrent/Offer.scala | 0 .../com/twitter/concurrent/Scheduler.scala | 0 .../scala/com/twitter/concurrent/Tx.scala | 0 .../main/scala/com/twitter/concurrent/BUILD | 12 --------- .../src/main/scala/com/twitter/util/BUILD | 25 +++---------------- 8 files changed, 31 insertions(+), 34 deletions(-) rename util-core/{ => concurrent-extra}/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala (100%) rename util-core/{ => concurrent-extra}/src/main/scala/com/twitter/concurrent/Offer.scala (100%) rename util-core/{ => concurrent-extra}/src/main/scala/com/twitter/concurrent/Scheduler.scala (100%) rename util-core/{ => concurrent-extra}/src/main/scala/com/twitter/concurrent/Tx.scala (100%) diff --git a/build.sbt b/build.sbt index fa7cf22819..46b7f3e220 100644 --- a/build.sbt +++ b/build.sbt @@ -222,6 +222,9 @@ lazy val utilCore = Project( sharedSettings ).settings( name := "util-core", + // Moved some code to 'concurrent-extra' to conform to Pants' 1:1:1 principle (https://www.pantsbuild.org/build_files.html#target-granularity) + // so that util-core would work better for Pants projects in IntelliJ. + unmanagedSourceDirectories in Compile += baseDirectory.value / "concurrent-extra", libraryDependencies ++= Seq( caffeineLib % "test", scalacheckLib, diff --git a/util-core/BUILD b/util-core/BUILD index bc41ada0da..9512034baa 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -12,3 +12,28 @@ target( "util/util-core/src/test/scala", ], ) + +# (DPB-10452) This target definition lives here because it needs to accommodate util-core +# code resolution in IntelliJ as well as `util-core-util` which contains packages +# com.twitter.concurrent and com.twitter.util +scala_library( + name = "util-core-util", + sources = globs( + "src/main/scala/com/twitter/util/*.scala", + "concurrent-extra/src/main/scala/com/twitter/concurrent/*.scala", + ), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-core-util", + repo = artifactory, + ), + dependencies = [ + "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", + "util/util-function/src/main/java", + ], + exports = [ + "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", + "util/util-function/src/main/java", + ], +) diff --git a/util-core/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala similarity index 100% rename from util-core/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala rename to util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala diff --git a/util-core/src/main/scala/com/twitter/concurrent/Offer.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala similarity index 100% rename from util-core/src/main/scala/com/twitter/concurrent/Offer.scala rename to util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala diff --git a/util-core/src/main/scala/com/twitter/concurrent/Scheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala similarity index 100% rename from util-core/src/main/scala/com/twitter/concurrent/Scheduler.scala rename to util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala diff --git a/util-core/src/main/scala/com/twitter/concurrent/Tx.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala similarity index 100% rename from util-core/src/main/scala/com/twitter/concurrent/Tx.scala rename to util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala diff --git a/util-core/src/main/scala/com/twitter/concurrent/BUILD b/util-core/src/main/scala/com/twitter/concurrent/BUILD index ceb2181ccb..d6406755a6 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/main/scala/com/twitter/concurrent/BUILD @@ -1,16 +1,4 @@ scala_library( - sources = globs( - "*.scala", - exclude = [ - # the com/twitter/util dependency already includes these files - globs( - "NamedPoolThreadFactory.scala", - "Offer.scala", - "Scheduler.scala", - "Tx.scala", - ), - ], - ), compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", diff --git a/util-core/src/main/scala/com/twitter/util/BUILD b/util-core/src/main/scala/com/twitter/util/BUILD index 8bcc4fd97c..3709da9555 100644 --- a/util-core/src/main/scala/com/twitter/util/BUILD +++ b/util-core/src/main/scala/com/twitter/util/BUILD @@ -1,23 +1,4 @@ -scala_library( - sources = globs( - "*.scala", - "../concurrent/NamedPoolThreadFactory.scala", - "../concurrent/Offer.scala", - "../concurrent/Scheduler.scala", - "../concurrent/Tx.scala", - ), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-core-util", - repo = artifactory, - ), - dependencies = [ - "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", - "util/util-function/src/main/java", - ], - exports = [ - "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", - "util/util-function/src/main/java", - ], +alias( + name = "util", + target = "util/util-core:util-core-util", ) From 8d35b496756172e9bb29c473d77e32f6414806e6 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 8 Nov 2018 16:49:56 +0000 Subject: [PATCH 057/796] util-core: Make Stopwatch.timeNanos monotone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem Stopwatch.timeNanos is intended to be used for measuring durations, but it's easy to shoot yourself in the foot with it since it's non-monotone. The comments give the caveat that it's only for us in testing, but even we screwed it up, which indicates it's better to be safe than sorry. Solution Make Stopwatch.timeNanos monotone when it's not using the supplied time control. Result Stopwatch.timeNanos is monotone, so it's usable in production. before ``` Benchmark Mode Cnt Score Error Units StopwatchBenchmark.timeSystemNanos avgt 5 33.117 ± 0.847 ns/op StopwatchBenchmark.timeSystemNanos:·gc.alloc.rate.norm avgt 5 ≈ 10⁻⁶ B/op StopwatchBenchmark.timeTimeNanos avgt 5 39.073 ± 3.524 ns/op StopwatchBenchmark.timeTimeNanos:·gc.alloc.rate.norm avgt 5 23.996 ± 0.036 B/op ``` after ``` Benchmark Mode Cnt Score Error Units StopwatchBenchmark.timeSystemNanos avgt 5 34.309 ± 2.924 ns/op StopwatchBenchmark.timeSystemNanos:·gc.alloc.rate.norm avgt 5 ≈ 10⁻⁶ B/op StopwatchBenchmark.timeTimeNanos avgt 5 40.920 ± 25.137 ns/op StopwatchBenchmark.timeTimeNanos:·gc.alloc.rate.norm avgt 5 ≈ 10⁻⁶ B/op ``` JIRA Issues: CSL-7233, CSL-7217 Differential Revision: https://phabricator.twitter.biz/D236629 --- CHANGELOG.rst | 5 ++++ .../com/twitter/util/StopwatchBenchmark.scala | 10 +++++++ .../scala/com/twitter/util/Stopwatch.scala | 30 +++++++++++++------ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 46442a6e58..cac459fc69 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,11 @@ Breaking API Changes * util-core: Rename `c.t.io.Reader.ReaderDiscarded` to `c.t.io.ReaderDiscardedException`. ``PHAB_ID=D231969`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-core: Made Stopwatch.timeNanos monotone. ``PHAB_ID=D236629`` + 18.10.0 ------- diff --git a/util-benchmark/src/main/scala/com/twitter/util/StopwatchBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/StopwatchBenchmark.scala index 5f2ae94381..02090d1ad0 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/StopwatchBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/StopwatchBenchmark.scala @@ -17,6 +17,16 @@ class StopwatchBenchmark { def timeTime(state: StopwatchState): Duration = { state.elapsed() } + + @Benchmark + def timeSystemNanos(): Long = { + Stopwatch.systemNanos() + } + + @Benchmark + def timeTimeNanos(): Long = { + Stopwatch.timeNanos() + } } object StopwatchBenchmark { diff --git a/util-core/src/main/scala/com/twitter/util/Stopwatch.scala b/util-core/src/main/scala/com/twitter/util/Stopwatch.scala index 1b39a4b8db..fc96d2c6d0 100644 --- a/util-core/src/main/scala/com/twitter/util/Stopwatch.scala +++ b/util-core/src/main/scala/com/twitter/util/Stopwatch.scala @@ -39,11 +39,15 @@ object Stopwatch extends Stopwatch { * A function that returns a Long that can be used for measuring elapsed time * in nanoseconds, that is `Time` manipulation compatible. * - * Useful for testing, but should not be used in production, since it uses a - * non-monotonic time under the hood, and entails a few allocations. For + * Useful for testing, but should not be used in a tight loop in production, + * since it entails a few allocations and does a [[Local]] lookup. For * production, see `systemNanos`. */ - val timeNanos: () => Long = () => Time.now.inNanoseconds + val timeNanos: () => Long = () => + Time.localGetTime() match { + case Some(local) => local().inNanoseconds + case None => systemNanos() + } /** * A function that returns a Long that can be used for measuring elapsed time @@ -59,11 +63,15 @@ object Stopwatch extends Stopwatch { * A function that returns a Long that can be used for measuring elapsed time * in microseconds, that is `Time` manipulation compatible. * - * Useful for testing, but should not be used in production, since it uses a - * non-monotonic time under the hood, and entails a few allocations. For + * Useful for testing, but should not be used in a tight loop in production, + * since it entails a few allocations and does a [[Local]] lookup. For * production, see `systemMicros`. */ - val timeMicros: () => Long = () => Time.now.inMicroseconds + val timeMicros: () => Long = () => + Time.localGetTime() match { + case Some(local) => local().inMicroseconds + case None => systemMicros() + } /** * A function that returns a Long that can be used for measuring elapsed time @@ -79,11 +87,15 @@ object Stopwatch extends Stopwatch { * A function that returns a Long that can be used for measuring elapsed time * in milliseconds, that is `Time` manipulation compatible. * - * Useful for testing, but should not be used in production, since it uses a - * non-monotonic time under the hood, and entails a few allocations. For + * Useful for testing, but should not be used in a tight loop in production, + * since it entails a few allocations and does a [[Local]] lookup. For * production, see `systemMillis`. */ - val timeMillis: () => Long = () => Time.now.inMilliseconds + val timeMillis: () => Long = () => + Time.localGetTime() match { + case Some(local) => local().inMilliseconds + case None => systemMillis() + } def start(): Elapsed = Time.localGetTime() match { case Some(local) => From 86ab24f53d4223e2578f568f6c3b3c1a84eb0531 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Mon, 12 Nov 2018 18:46:00 +0000 Subject: [PATCH 058/796] Twitter-oss: Prepare OSS libraries for release 18.11 Problem We want to release the next versions of our Twitter OSS libraries 18.11 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-7204 Differential Revision: https://phabricator.twitter.biz/D240782 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cac459fc69..9582363d69 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.11.0 +------- + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 74e5e6c316..aaea7700e2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "18.10.0" +val collUtils = "com.twitter" %% "util-collection" % "18.11.0" ``` # Units diff --git a/build.sbt b/build.sbt index 46b7f3e220..ceaf9f76a3 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.11.0-SNAPSHOT" +val releaseVersion = "18.11.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 1e311ac4f98e1e904b33dbea3deb294136e11b93 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 13 Nov 2018 16:34:08 +0000 Subject: [PATCH 059/796] Twitter-oss: update OSS libraries post-release to 18.12 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D241161 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aaea7700e2..6b7d5d0b8f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index ceaf9f76a3..af8905b0c4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.11.0" +val releaseVersion = "18.12.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From ba9c5def0b7ff9ab3dc7fba740b2ad4fc5b8b788 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 13 Nov 2018 23:57:38 +0000 Subject: [PATCH 060/796] util-jvm: Remove unsupported JVM versions Problem / Solution Code supporting JDK versions before 8 is not necessary. JIRA Issues: CSL-7092 Differential Revision: https://phabricator.twitter.biz/D240951 --- .../main/scala/com/twitter/jvm/Hotspot.scala | 357 +----------------- 1 file changed, 14 insertions(+), 343 deletions(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index d7b0259033..d502fcfd01 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -9,7 +9,6 @@ import javax.management.openmbean.CompositeDataSupport import javax.management.{ObjectName, RuntimeMBeanException} import scala.collection.JavaConverters._ import scala.language.reflectiveCalls -import scala.util.control.NonFatal class Hotspot extends Jvm { private[this] val epoch = @@ -29,14 +28,9 @@ class Hotspot extends Jvm { ObjectName.getInstance("com.sun.management:type=HotSpotDiagnostic") private[this] val jvm: VMManagement = { - val fld = try { - // jdk5/6 have jvm field in ManagementFactory class - Class.forName("sun.management.ManagementFactory").getDeclaredField("jvm") - } catch { - case NonFatal(_) => - // jdk7 moves jvm field to ManagementFactoryHelper class - Class.forName("sun.management.ManagementFactoryHelper").getDeclaredField("jvm") - } + val fld = Class + .forName("sun.management.ManagementFactoryHelper") + .getDeclaredField("jvm") fld.setAccessible(true) fld.get(null).asInstanceOf[VMManagement] } @@ -67,11 +61,11 @@ class Hotspot extends Jvm { counters(name).get(name) object opts extends Opts { - def compileThresh = opt("CompileThreshold") map (_.toInt) + def compileThresh: Option[Int] = opt("CompileThreshold").map(_.toInt) } private[this] def ticksToDuration(ticks: Long, freq: Long) = - (1000000 * ticks / freq).microseconds + (1000000L * ticks / freq).microseconds private[this] def getGc(which: Int, cs: Map[String, Counter]) = { def get(what: String) = cs.get("sun.gc.collector.%d.%s".format(which, what)) @@ -103,7 +97,7 @@ class Hotspot extends Jvm { val ageHisto = for { thresh <- tenuringThreshold.toSeq - i <- 1L to thresh + i <- 1L.to(thresh) bucket <- cs.get("sun.gc.generation.0.agetable.bytes.%02d".format(i)) } yield long(bucket) @@ -118,7 +112,7 @@ class Hotspot extends Jvm { // TODO: include causes for GCs? Snapshot( timestamp.getOrElse(Time.epoch), - heap.getOrElse(Heap(0, 0, Seq())), + heap.getOrElse(Heap(0, 0, Nil)), getGc(0, cs).toSeq ++ getGc(1, cs).toSeq ) } @@ -132,26 +126,15 @@ class Hotspot extends Jvm { private val log = Logger.getLogger(getClass.getName) private[this] val safepointBean = { - val runtimeBean = - try { - Class - .forName("sun.management.ManagementFactory") - .getMethod("getHotspotRuntimeMBean") - .invoke(null) - // jdk 6 has HotspotRuntimeMBean in the ManagementFactory class - } catch { - case _: Throwable => - Class - .forName("sun.management.ManagementFactoryHelper") - .getMethod("getHotspotRuntimeMBean") - .invoke(null) - // jdks 7 and 8 have HotspotRuntimeMBean in the ManagementFactoryHelper class - } + val runtimeBean = Class + .forName("sun.management.ManagementFactoryHelper") + .getMethod("getHotspotRuntimeMBean") + .invoke(null) def asSafepointBean(x: AnyRef) = { x.asInstanceOf[{ - def getSafepointSyncTime: Long; - def getTotalSafepointTime: Long; + def getSafepointSyncTime: Long + def getTotalSafepointTime: Long def getSafepointCount: Long } ] @@ -200,319 +183,7 @@ class Hotspot extends Jvm { def tenuringThreshold: Long = counter("sun.gc.policy.tenuringThreshold").map(long).getOrElse(0L) def snapCounters: Map[String, String] = - counters("") mapValues (_.getValue().toString) + counters("").mapValues(_.getValue().toString) def forceGc(): Unit = System.gc() } - -/* - -A sample of counters available on hotspot: - -scala> res0 foreach { case (k, v) => printf("%s = %s\n", k, v) } -java.ci.totalTime = 4453822466 -java.cls.loadedClasses = 3459 -java.cls.sharedLoadedClasses = 0 -java.cls.sharedUnloadedClasses = 0 -java.cls.unloadedClasses = 0 -java.property.java.class.path = . -java.property.java.endorsed.dirs = /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Home/lib/endorsed -java.property.java.ext.dirs = /Library/Java/Extensions:/System/Library/Java/Extensions:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Home/lib/ext -java.property.java.home = /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Home -java.property.java.library.path = /Applications/YourKit_Java_Profiler_9.5.6.app/bin/mac:.:/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java -java.property.java.version = 1.6.0_29 -java.property.java.vm.info = mixed mode -java.property.java.vm.name = Java HotSpot(TM) 64-Bit Server VM -java.property.java.vm.specification.name = Java Virtual Machine Specification -java.property.java.vm.specification.vendor = Sun Microsystems Inc. -java.property.java.vm.specification.version = 1.0 -java.property.java.vm.vendor = Apple Inc. -java.property.java.vm.version = 20.4-b02-402 -java.rt.vmArgs = -Xserver -Djava.net.preferIPv4Stack=true -Dsbt.log.noformat=true -Xmx2G -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:MaxTenuringThreshold=3 -Xbootclasspath/a:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/jline.jar:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/scala-compiler.jar:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/scala-dbc.jar:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/scala-library.jar:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/scala-swing.jar:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/scalap.jar -Dscala.usejavacp=true -Dscala.home=/Users/marius/t/src/.scala/scala-2.8.1.final -Denv.emacs= -java.rt.vmFlags = -java.threads.daemon = 3 -java.threads.live = 4 -java.threads.livePeak = 5 -java.threads.started = 9 -sun.ci.compilerThread.0.compiles = 389 -sun.ci.compilerThread.0.method = -sun.ci.compilerThread.0.time = 52805 -sun.ci.compilerThread.0.type = 1 -sun.ci.compilerThread.1.compiles = 442 -sun.ci.compilerThread.1.method = java/lang/String compareTo -sun.ci.compilerThread.1.time = 49709 -sun.ci.compilerThread.1.type = 1 -sun.ci.lastFailedMethod = -sun.ci.lastFailedType = 0 -sun.ci.lastInvalidatedMethod = -sun.ci.lastInvalidatedType = 0 -sun.ci.lastMethod = java/nio/DirectByteBuffer ix -sun.ci.lastSize = 10 -sun.ci.lastType = 1 -sun.ci.nmethodCodeSize = 793568 -sun.ci.nmethodSize = 2272496 -sun.ci.osrBytes = 3470 -sun.ci.osrCompiles = 11 -sun.ci.osrTime = 75314565 -sun.ci.standardBytes = 262871 -sun.ci.standardCompiles = 819 -sun.ci.standardTime = 4394092506 -sun.ci.threads = 2 -sun.ci.totalBailouts = 0 -sun.ci.totalCompiles = 830 -sun.ci.totalInvalidates = 0 -sun.cls.appClassBytes = 174471 -sun.cls.appClassLoadCount = 165 -sun.cls.appClassLoadTime = 16577583 -sun.cls.appClassLoadTime.self = 8760586 -sun.cls.classInitTime = 153888518 -sun.cls.classInitTime.self = 75974400 -sun.cls.classLinkedTime = 76083753 -sun.cls.classLinkedTime.self = 61672107 -sun.cls.classVerifyTime = 14198241 -sun.cls.classVerifyTime.self = 8650545 -sun.cls.defineAppClassTime = 4361539 -sun.cls.defineAppClassTime.self = 579811 -sun.cls.defineAppClasses = 35 -sun.cls.initializedClasses = 2994 -sun.cls.isUnsyncloadClassSet = 0 -sun.cls.jniDefineClassNoLockCalls = 0 -sun.cls.jvmDefineClassNoLockCalls = 3 -sun.cls.jvmFindLoadedClassNoLockCalls = 0 -sun.cls.linkedClasses = 3353 -sun.cls.loadInstanceClassFailRate = 0 -sun.cls.loadedBytes = 9556504 -sun.cls.lookupSysClassTime = 165493723 -sun.cls.methodBytes = 7128160 -sun.cls.nonSystemLoaderLockContentionRate = 0 -sun.cls.parseClassTime = 260534425 -sun.cls.parseClassTime.self = 225060288 -sun.cls.sharedClassLoadTime = 259627 -sun.cls.sharedLoadedBytes = 0 -sun.cls.sharedUnloadedBytes = 0 -sun.cls.sysClassBytes = 16107656 -sun.cls.sysClassLoadTime = 518807017 -sun.cls.systemLoaderLockContentionRate = 0 -sun.cls.time = 547792239 -sun.cls.unloadedBytes = 0 -sun.cls.unsafeDefineClassCalls = 3 -sun.cls.verifiedClasses = 3353 -sun.gc.cause = No GC -sun.gc.collector.0.invocations = 13 -sun.gc.collector.0.lastEntryTime = 3235357990 -sun.gc.collector.0.lastExitTime = 3242811518 -sun.gc.collector.0.name = PCopy -sun.gc.collector.0.time = 74741924 -sun.gc.collector.1.invocations = 0 -sun.gc.collector.1.lastEntryTime = 0 -sun.gc.collector.1.lastExitTime = 0 -sun.gc.collector.1.name = CMS -sun.gc.collector.1.time = 0 -sun.gc.generation.0.agetable.bytes.00 = 0 -sun.gc.generation.0.agetable.bytes.01 = 2154984 -sun.gc.generation.0.agetable.bytes.02 = 0 -sun.gc.generation.0.agetable.bytes.03 = 0 -sun.gc.generation.0.agetable.bytes.04 = 0 -sun.gc.generation.0.agetable.bytes.05 = 0 -sun.gc.generation.0.agetable.bytes.06 = 0 -sun.gc.generation.0.agetable.bytes.07 = 0 -sun.gc.generation.0.agetable.bytes.08 = 0 -sun.gc.generation.0.agetable.bytes.09 = 0 -sun.gc.generation.0.agetable.bytes.10 = 0 -sun.gc.generation.0.agetable.bytes.11 = 0 -sun.gc.generation.0.agetable.bytes.12 = 0 -sun.gc.generation.0.agetable.bytes.13 = 0 -sun.gc.generation.0.agetable.bytes.14 = 0 -sun.gc.generation.0.agetable.bytes.15 = 0 -sun.gc.generation.0.agetable.size = 16 -sun.gc.generation.0.capacity = 21757952 -sun.gc.generation.0.maxCapacity = 174456832 -sun.gc.generation.0.minCapacity = 21757952 -sun.gc.generation.0.name = new -sun.gc.generation.0.space.0.capacity = 17432576 -sun.gc.generation.0.space.0.initCapacity = 0 -sun.gc.generation.0.space.0.maxCapacity = 139591680 -sun.gc.generation.0.space.0.name = eden -sun.gc.generation.0.space.0.used = 6954760 -sun.gc.generation.0.space.1.capacity = 2162688 -sun.gc.generation.0.space.1.initCapacity = 0 -sun.gc.generation.0.space.1.maxCapacity = 17432576 -sun.gc.generation.0.space.1.name = s0 -sun.gc.generation.0.space.1.used = 0 -sun.gc.generation.0.space.2.capacity = 2162688 -sun.gc.generation.0.space.2.initCapacity = 0 -sun.gc.generation.0.space.2.maxCapacity = 17432576 -sun.gc.generation.0.space.2.name = s1 -sun.gc.generation.0.space.2.used = 2162688 -sun.gc.generation.0.spaces = 3 -sun.gc.generation.0.threads = 8 -sun.gc.generation.1.capacity = 65404928 -sun.gc.generation.1.maxCapacity = 1973026816 -sun.gc.generation.1.minCapacity = 65404928 -sun.gc.generation.1.name = old -sun.gc.generation.1.space.0.capacity = 65404928 -sun.gc.generation.1.space.0.initCapacity = 65404928 -sun.gc.generation.1.space.0.maxCapacity = 1973026816 -sun.gc.generation.1.space.0.name = old -sun.gc.generation.1.space.0.used = 25122784 -sun.gc.generation.1.spaces = 1 -sun.gc.generation.2.capacity = 33161216 -sun.gc.generation.2.maxCapacity = 268435456 -sun.gc.generation.2.minCapacity = 21757952 -sun.gc.generation.2.name = perm -sun.gc.generation.2.space.0.capacity = 33161216 -sun.gc.generation.2.space.0.initCapacity = 21757952 -sun.gc.generation.2.space.0.maxCapacity = 268435456 -sun.gc.generation.2.space.0.name = perm -sun.gc.generation.2.space.0.used = 32439520 -sun.gc.generation.2.spaces = 1 -sun.gc.lastCause = unknown GCCause -sun.gc.policy.collectors = 2 -sun.gc.policy.desiredSurvivorSize = 1081344 -sun.gc.policy.generations = 3 -sun.gc.policy.maxTenuringThreshold = 3 -sun.gc.policy.name = ParNew:CMS -sun.gc.policy.tenuringThreshold = 1 -sun.gc.tlab.alloc = 756560 -sun.gc.tlab.allocThreads = 1 -sun.gc.tlab.fastWaste = 0 -sun.gc.tlab.fills = 28 -sun.gc.tlab.gcWaste = 0 -sun.gc.tlab.maxFastWaste = 0 -sun.gc.tlab.maxFills = 28 -sun.gc.tlab.maxGcWaste = 0 -sun.gc.tlab.maxSlowAlloc = 4 -sun.gc.tlab.maxSlowWaste = 116 -sun.gc.tlab.slowAlloc = 4 -sun.gc.tlab.slowWaste = 116 -sun.os.hrt.frequency = 1000000000 -sun.os.hrt.ticks = 6949680691 -sun.property.sun.boot.class.path = /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/jsfd.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/classes.jar:/System/Library/Frameworks/JavaVM.framework/Frameworks/JavaRuntimeSupport.framework/Resources/Java/JavaRuntimeSupport.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/ui.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/laf.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/sunrsasign.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/jsse.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/jce.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/charsets.jar:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/jline.jar:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/scala-compiler.jar:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/scala-dbc.jar:/Users/marius/t/src/.scala/scala-2.8.1.final/lib/scala-library.jar:/Users/marius/t/src/ -sun.property.sun.boot.library.path = /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Libraries -sun.rt._sync_ContendedLockAttempts = 53 -sun.rt._sync_Deflations = 11 -sun.rt._sync_EmptyNotifications = 0 -sun.rt._sync_FailedSpins = 0 -sun.rt._sync_FutileWakeups = 4 -sun.rt._sync_Inflations = 16 -sun.rt._sync_MonExtant = 256 -sun.rt._sync_MonInCirculation = 0 -sun.rt._sync_MonScavenged = 0 -sun.rt._sync_Notifications = 15 -sun.rt._sync_Parks = 43 -sun.rt._sync_PrivateA = 0 -sun.rt._sync_PrivateB = 0 -sun.rt._sync_SlowEnter = 0 -sun.rt._sync_SlowExit = 0 -sun.rt._sync_SlowNotify = 0 -sun.rt._sync_SlowNotifyAll = 0 -sun.rt._sync_SuccessfulSpins = 0 -sun.rt.applicationTime = 6804389569 -sun.rt.createVmBeginTime = 1320946078890 -sun.rt.createVmEndTime = 1320946078960 -sun.rt.internalVersion = Java HotSpot(TM) 64-Bit Server VM (20.4-b02-402) for macosx-amd64 JRE (1.6.0), built on Nov 1 2011 10:50:41 by "root" with gcc 4.2.1 (Based on Apple Inc. build 5658) (LLVM build 2335.15.00) -sun.rt.interruptedBeforeIO = 0 -sun.rt.interruptedDuringIO = 0 -sun.rt.javaCommand = scala.tools.nsc.MainGenericRunner -classpath /Users/marius/t/src/util/util-core/target/classes:/Users/marius/t/src/util/util-jvm/target/classes:/Users/marius/t/src/project/boot/scala-2.8.1/lib/scala-library.jar -sun.rt.jvmCapabilities = 1000000000000000000000000000000000000000000000000000000000000000 -sun.rt.jvmVersion = 335806466 -sun.rt.safepointSyncTime = 3107558 -sun.rt.safepointTime = 82605199 -sun.rt.safepoints = 21 -sun.rt.threadInterruptSignaled = 0 -sun.rt.vmInitDoneTime = 1320946078941 -sun.threads.vmOperationTime = 76932327 - - -## after a System.gc() - -sun.gc.cause = No GC -sun.gc.collector.0.invocations = 13 -sun.gc.collector.0.lastEntryTime = 3235357990 -sun.gc.collector.0.lastExitTime = 3242811518 -sun.gc.collector.0.name = PCopy -sun.gc.collector.0.time = 74741924 -sun.gc.collector.1.invocations = 0 -sun.gc.collector.1.lastEntryTime = 0 -sun.gc.collector.1.lastExitTime = 0 -sun.gc.collector.1.name = CMS -sun.gc.collector.1.time = 0 -sun.gc.generation.0.agetable.bytes.00 = 0 -sun.gc.generation.0.agetable.bytes.01 = 2154984 -sun.gc.generation.0.agetable.bytes.02 = 0 -sun.gc.generation.0.agetable.bytes.03 = 0 -sun.gc.generation.0.agetable.bytes.04 = 0 -sun.gc.generation.0.agetable.bytes.05 = 0 -sun.gc.generation.0.agetable.bytes.06 = 0 -sun.gc.generation.0.agetable.bytes.07 = 0 -sun.gc.generation.0.agetable.bytes.08 = 0 -sun.gc.generation.0.agetable.bytes.09 = 0 -sun.gc.generation.0.agetable.bytes.10 = 0 -sun.gc.generation.0.agetable.bytes.11 = 0 -sun.gc.generation.0.agetable.bytes.12 = 0 -sun.gc.generation.0.agetable.bytes.13 = 0 -sun.gc.generation.0.agetable.bytes.14 = 0 -sun.gc.generation.0.agetable.bytes.15 = 0 -sun.gc.generation.0.agetable.size = 16 -sun.gc.generation.0.capacity = 21757952 -sun.gc.generation.0.maxCapacity = 174456832 -sun.gc.generation.0.minCapacity = 21757952 -sun.gc.generation.0.name = new -sun.gc.generation.0.space.0.capacity = 17432576 -sun.gc.generation.0.space.0.initCapacity = 0 -sun.gc.generation.0.space.0.maxCapacity = 139591680 -sun.gc.generation.0.space.0.name = eden -sun.gc.generation.0.space.0.used = 6954760 -sun.gc.generation.0.space.1.capacity = 2162688 -sun.gc.generation.0.space.1.initCapacity = 0 -sun.gc.generation.0.space.1.maxCapacity = 17432576 -sun.gc.generation.0.space.1.name = s0 -sun.gc.generation.0.space.1.used = 0 -sun.gc.generation.0.space.2.capacity = 2162688 -sun.gc.generation.0.space.2.initCapacity = 0 -sun.gc.generation.0.space.2.maxCapacity = 17432576 -sun.gc.generation.0.space.2.name = s1 -sun.gc.generation.0.space.2.used = 2162688 -sun.gc.generation.0.spaces = 3 -sun.gc.generation.0.threads = 8 -sun.gc.generation.1.capacity = 65404928 -sun.gc.generation.1.maxCapacity = 1973026816 -sun.gc.generation.1.minCapacity = 65404928 -sun.gc.generation.1.name = old -sun.gc.generation.1.space.0.capacity = 65404928 -sun.gc.generation.1.space.0.initCapacity = 65404928 -sun.gc.generation.1.space.0.maxCapacity = 1973026816 -sun.gc.generation.1.space.0.name = old -sun.gc.generation.1.space.0.used = 25122784 -sun.gc.generation.1.spaces = 1 -sun.gc.generation.2.capacity = 33161216 -sun.gc.generation.2.maxCapacity = 268435456 -sun.gc.generation.2.minCapacity = 21757952 -sun.gc.generation.2.name = perm -sun.gc.generation.2.space.0.capacity = 33161216 -sun.gc.generation.2.space.0.initCapacity = 21757952 -sun.gc.generation.2.space.0.maxCapacity = 268435456 -sun.gc.generation.2.space.0.name = perm -sun.gc.generation.2.space.0.used = 32439520 -sun.gc.generation.2.spaces = 1 -sun.gc.lastCause = unknown GCCause -sun.gc.policy.collectors = 2 -sun.gc.policy.desiredSurvivorSize = 1081344 -sun.gc.policy.generations = 3 -sun.gc.policy.maxTenuringThreshold = 3 -sun.gc.policy.name = ParNew:CMS -sun.gc.policy.tenuringThreshold = 1 -sun.gc.tlab.alloc = 756560 -sun.gc.tlab.allocThreads = 1 -sun.gc.tlab.fastWaste = 0 -sun.gc.tlab.fills = 28 -sun.gc.tlab.gcWaste = 0 -sun.gc.tlab.maxFastWaste = 0 -sun.gc.tlab.maxFills = 28 -sun.gc.tlab.maxGcWaste = 0 -sun.gc.tlab.maxSlowAlloc = 4 -sun.gc.tlab.maxSlowWaste = 116 -sun.gc.tlab.slowAlloc = 4 -sun.gc.tlab.slowWaste = 116 - - */ From 361c81f8ff3bd9eb9fb967be90a5867dbc8c82ab Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 14 Nov 2018 00:13:47 +0000 Subject: [PATCH 061/796] util: Rephrase comment regarding pants workaround Problem / Solution This workaround is no longer temporary and the comments now reflect that. JIRA Issues: CSL-7168 Differential Revision: https://phabricator.twitter.biz/D240956 --- util-cache/src/main/scala/PantsWorkaround.scala | 5 ++--- util-collection/src/main/scala/PantsWorkaround.scala | 5 ++--- util-core/src/main/scala/PantsWorkaround.scala | 5 ++--- util-test/src/main/scala/PantsWorkaround.scala | 5 ++--- util-zk/src/main/scala/PantsWorkaround.scala | 5 ++--- 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/util-cache/src/main/scala/PantsWorkaround.scala b/util-cache/src/main/scala/PantsWorkaround.scala index 78c256812b..9800d15646 100644 --- a/util-cache/src/main/scala/PantsWorkaround.scala +++ b/util-cache/src/main/scala/PantsWorkaround.scala @@ -1,5 +1,4 @@ -// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 -// will be removed following the next pants release. +// workaround for https://github.com/pantsbuild/pants/issues/6678 private final class PantsWorkaround { - throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") } diff --git a/util-collection/src/main/scala/PantsWorkaround.scala b/util-collection/src/main/scala/PantsWorkaround.scala index 78c256812b..9800d15646 100644 --- a/util-collection/src/main/scala/PantsWorkaround.scala +++ b/util-collection/src/main/scala/PantsWorkaround.scala @@ -1,5 +1,4 @@ -// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 -// will be removed following the next pants release. +// workaround for https://github.com/pantsbuild/pants/issues/6678 private final class PantsWorkaround { - throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") } diff --git a/util-core/src/main/scala/PantsWorkaround.scala b/util-core/src/main/scala/PantsWorkaround.scala index 78c256812b..9800d15646 100644 --- a/util-core/src/main/scala/PantsWorkaround.scala +++ b/util-core/src/main/scala/PantsWorkaround.scala @@ -1,5 +1,4 @@ -// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 -// will be removed following the next pants release. +// workaround for https://github.com/pantsbuild/pants/issues/6678 private final class PantsWorkaround { - throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") } diff --git a/util-test/src/main/scala/PantsWorkaround.scala b/util-test/src/main/scala/PantsWorkaround.scala index 78c256812b..9800d15646 100644 --- a/util-test/src/main/scala/PantsWorkaround.scala +++ b/util-test/src/main/scala/PantsWorkaround.scala @@ -1,5 +1,4 @@ -// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 -// will be removed following the next pants release. +// workaround for https://github.com/pantsbuild/pants/issues/6678 private final class PantsWorkaround { - throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") } diff --git a/util-zk/src/main/scala/PantsWorkaround.scala b/util-zk/src/main/scala/PantsWorkaround.scala index 78c256812b..9800d15646 100644 --- a/util-zk/src/main/scala/PantsWorkaround.scala +++ b/util-zk/src/main/scala/PantsWorkaround.scala @@ -1,5 +1,4 @@ -// Temporary workaround for https://github.com/pantsbuild/pants/pull/6673 -// will be removed following the next pants release. +// workaround for https://github.com/pantsbuild/pants/issues/6678 private final class PantsWorkaround { - throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/pull/6673") + throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") } From c9bc2b41cd4ac367b31de94432dbbd483e41a229 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 14 Nov 2018 17:11:11 +0000 Subject: [PATCH 062/796] util-core: Strip some overhead from AsyncQueue Problem There is some unnecessary overhead in AsyncQueue, a common primitive in performance critical code, that can be removed. Solution Remove the overhead and simplify the internal state machine. No bankers queue please Differential Revision: https://phabricator.twitter.biz/D237063 --- .../concurrent/AsyncQueueBenchmark.scala | 5 + .../com/twitter/concurrent/AsyncQueue.scala | 113 +++++++++--------- .../twitter/concurrent/AsyncQueueTest.scala | 25 +++- 3 files changed, 84 insertions(+), 59 deletions(-) diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncQueueBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncQueueBenchmark.scala index f62f8ae009..2627939158 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncQueueBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncQueueBenchmark.scala @@ -9,6 +9,11 @@ class AsyncQueueBenchmark extends StdBenchAnnotations { private[this] val q = new AsyncQueue[String] + @Benchmark + def allocation(): AsyncQueue[String] = { + new AsyncQueue[String] + } + @Benchmark def offersThenPolls: Future[String] = { q.offer("hello") diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala index b01156ca08..865848c154 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala @@ -1,15 +1,22 @@ package com.twitter.concurrent -import com.twitter.util.{Future, Promise, Try, Return, Throw} -import java.util.{Queue => JQueue, ArrayDeque} +import com.twitter.util.{Future, Promise, Return, Throw, Try} +import java.util.ArrayDeque import scala.collection.immutable.Queue object AsyncQueue { private sealed trait State - private case object Idle extends State - private case object Offering extends State + + // In the `OfferingOrIdle` state we can have 0 or more chunks of data ready for + // consumption. If we have 0 data chunks and the queue is `poll`ed we can transition + // to the `Polling` state. + private case object OfferingOrIdle extends State + + // If we're in a `Polling` state we must have at least one polling `Promise`. private case object Polling extends State + + // In the `Excepting` state the queue will only contain offers. private case class Excepting(exc: Throwable) extends State /** Indicates there is no max capacity */ @@ -33,12 +40,10 @@ class AsyncQueue[T](maxPendingOffers: Int) { require(maxPendingOffers > 0) - // synchronize all access to state, offers, and pollers - private[this] var state: State = Idle + // synchronize all access to `state` and `queue` + private[this] var state: State = OfferingOrIdle - // these aren't part of the state machine for performance - private[this] val offers: JQueue[T] = new ArrayDeque[T] - private[this] val pollers: JQueue[Promise[T]] = new ArrayDeque[Promise[T]] + private[this] val queue = new ArrayDeque[Any] /** * An asynchronous, unbounded, FIFO queue. In addition to providing [[offer]] @@ -51,7 +56,10 @@ class AsyncQueue[T](maxPendingOffers: Int) { * Returns the current number of pending elements. */ final def size: Int = synchronized { - offers.size + state match { + case OfferingOrIdle | Excepting(_) => queue.size + case Polling => 0 // If we're `Polling` we're empty + } } /** @@ -60,28 +68,26 @@ class AsyncQueue[T](maxPendingOffers: Int) { */ final def poll(): Future[T] = synchronized { state match { - case Idle => + case OfferingOrIdle if queue.isEmpty => val p = new Promise[T] state = Polling - pollers.offer(p) + queue.offer(p) p + case OfferingOrIdle => + val elem = queue.poll() + Future.value(elem.asInstanceOf[T]) + case Polling => val p = new Promise[T] - pollers.offer(p) + queue.offer(p) p - case Offering => - val elem = offers.poll() - if (offers.isEmpty) - state = Idle - Future.value(elem) - - case Excepting(t) if offers.isEmpty => + case Excepting(t) if queue.isEmpty => Future.exception(t) case Excepting(_) => - Future.value(offers.poll()) + Future.value(queue.poll().asInstanceOf[T]) } } @@ -94,22 +100,17 @@ class AsyncQueue[T](maxPendingOffers: Int) { var waiter: Promise[T] = null val result = synchronized { state match { - case Idle => - state = Offering - offers.offer(elem) - true - - case Offering if offers.size >= maxPendingOffers => + case OfferingOrIdle if queue.size >= maxPendingOffers => false - case Offering => - offers.offer(elem) + case OfferingOrIdle => + queue.offer(elem) true case Polling => - waiter = pollers.poll() - if (pollers.isEmpty) - state = Idle + waiter = queue.poll().asInstanceOf[Promise[T]] + if (queue.isEmpty) + state = OfferingOrIdle true case Excepting(_) => @@ -132,22 +133,24 @@ class AsyncQueue[T](maxPendingOffers: Int) { */ final def drain(): Try[Queue[T]] = synchronized { state match { - case Offering => - state = Idle + case OfferingOrIdle => var q = Queue.empty[T] - while (!offers.isEmpty) { - q :+= offers.poll() + while (!queue.isEmpty) { + q :+= queue.poll().asInstanceOf[T] } Return(q) - case Excepting(e) if !offers.isEmpty => + + case Excepting(_) if !queue.isEmpty => var q = Queue.empty[T] - while (!offers.isEmpty) { - q :+= offers.poll() + while (!queue.isEmpty) { + q :+= queue.poll().asInstanceOf[T] } Return(q) + case Excepting(e) => Throw(e) - case _ => + + case Polling => Return(Queue.empty) } } @@ -166,37 +169,39 @@ class AsyncQueue[T](maxPendingOffers: Int) { * No new elements are admitted to the queue after it has been failed. */ def fail(exc: Throwable, discard: Boolean): Unit = { - var q: Queue[Promise[T]] = null - synchronized { + // CAUTION: `q` may be `null` + val q: Array[AnyRef] = synchronized { state match { - case Idle => - state = Excepting(exc) - case Polling => state = Excepting(exc) - q = Queue.empty[Promise[T]] - while (!pollers.isEmpty) { - val waiter = pollers.poll() - q :+= waiter + if (queue.isEmpty) null + else { + val data = queue.toArray + queue.clear() + data } - case Offering => + case OfferingOrIdle => if (discard) - offers.clear() + queue.clear() state = Excepting(exc) + null case Excepting(_) => // Just take the first one. + null } } // we do this to avoid satisfaction while synchronized, which could lead to // lock contention if closures on the promise are slow or there are a lot of // them if (q != null) { - q.foreach { p => - p.setException(exc) + var i = 0 + while (i < q.length) { + q(i).asInstanceOf[Promise[_]].setException(exc) + i += 1 } } } - override def toString = "AsyncQueue<%s>".format(synchronized(state)) + override def toString = s"AsyncQueue<${synchronized(state)}>" } diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncQueueTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncQueueTest.scala index 66c6dbd60d..71a2d4669b 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncQueueTest.scala @@ -1,13 +1,13 @@ package com.twitter.concurrent -import com.twitter.util.{Await, Return, Throw} -import org.junit.runner.RunWith +import com.twitter.util.{Await, Awaitable, Duration, Return, Throw} import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import scala.collection.immutable.Queue -@RunWith(classOf[JUnitRunner]) class AsyncQueueTest extends FunSuite { + + private def await[T](t: Awaitable[T]): T = Await.result(t, Duration.fromSeconds(5)) + test("queue pollers") { val q = new AsyncQueue[Int] @@ -132,7 +132,7 @@ class AsyncQueueTest extends FunSuite { q.offer(2) q.offer(3) - assert(1 == Await.result(q.poll())) + assert(1 == await(q.poll())) assert(Return(Queue(2, 3)) == q.drain()) assert(!q.poll().isDefined) @@ -161,4 +161,19 @@ class AsyncQueueTest extends FunSuite { assert(0 == q.size) } + test("size") { + val q = new AsyncQueue[Int]() + assert(q.size == 0) + val f1 = q.poll() + assert(q.size == 0) + q.offer(0) + assert(await(f1) == 0) + assert(q.size == 0) + q.offer(1) + assert(q.size == 1) + q.fail(new Exception(), false) + assert(q.size == 1) + assert(await(q.poll()) == 1) + assert(q.size == 0) + } } From ff7440788f8ff3b22362d82ea6b62d3d038c5dbb Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 14 Nov 2018 17:49:56 +0000 Subject: [PATCH 063/796] util-app: Tweak pants BUILD target Problem IDEA and other tools do not work well with the parent directory paths used for sources. Solution Move the BUILD target back to the top-level. JIRA Issues: CSL-7254 Differential Revision: https://phabricator.twitter.biz/D241586 --- util-app/src/main/java/BUILD | 2 +- util-app/src/main/scala/BUILD | 18 +++++++++++++-- util-app/src/main/scala/com/twitter/app/BUILD | 23 ------------------- util-app/src/test/java/com/twitter/app/BUILD | 2 +- util-app/src/test/scala/com/twitter/app/BUILD | 2 +- util-jvm/src/main/scala/com/twitter/jvm/BUILD | 2 +- util-jvm/src/test/scala/com/twitter/jvm/BUILD | 2 +- .../src/main/scala/com/twitter/logging/BUILD | 4 ++-- .../src/test/scala/com/twitter/logging/BUILD | 2 +- .../main/scala/com/twitter/util/tunable/BUILD | 2 +- .../com/twitter/util/tunable/linter/BUILD | 2 +- 11 files changed, 26 insertions(+), 35 deletions(-) delete mode 100644 util-app/src/main/scala/com/twitter/app/BUILD diff --git a/util-app/src/main/java/BUILD b/util-app/src/main/java/BUILD index f7e60703a5..135fed783f 100644 --- a/util-app/src/main/java/BUILD +++ b/util-app/src/main/java/BUILD @@ -1,6 +1,6 @@ target( dependencies = [ "util/util-app/src/main/java/com/twitter/app", - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", ], ) diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index f7e60703a5..73997192be 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -1,6 +1,20 @@ -target( +scala_library( + sources = rglobs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-app", + repo = artifactory, + ), dependencies = [ "util/util-app/src/main/java/com/twitter/app", - "util/util-app/src/main/scala/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", + "util/util-registry/src/main/scala/com/twitter/util/registry", + ], + exports = [ + "util/util-app/src/main/java/com/twitter/app", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/util", ], ) diff --git a/util-app/src/main/scala/com/twitter/app/BUILD b/util-app/src/main/scala/com/twitter/app/BUILD deleted file mode 100644 index 8f4a5ab625..0000000000 --- a/util-app/src/main/scala/com/twitter/app/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - sources = globs( - "*.scala", - "../finagle/util/flags.scala", - ), - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-app", - repo = artifactory, - ), - dependencies = [ - "util/util-app/src/main/java/com/twitter/app", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", - "util/util-registry/src/main/scala/com/twitter/util/registry", - ], - exports = [ - "util/util-app/src/main/java/com/twitter/app", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", - ], -) diff --git a/util-app/src/test/java/com/twitter/app/BUILD b/util-app/src/test/java/com/twitter/app/BUILD index 6fbe02072d..cbd35024d7 100644 --- a/util-app/src/test/java/com/twitter/app/BUILD +++ b/util-app/src/test/java/com/twitter/app/BUILD @@ -5,7 +5,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", "util/util-app/src/main/java/com/twitter/app", - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/util", "util/util-function/src/main/java/com/twitter/function", ], diff --git a/util-app/src/test/scala/com/twitter/app/BUILD b/util-app/src/test/scala/com/twitter/app/BUILD index 1f4e7e0faa..b26f04ffde 100644 --- a/util-app/src/test/scala/com/twitter/app/BUILD +++ b/util-app/src/test/scala/com/twitter/app/BUILD @@ -5,7 +5,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", "util/util-app/src/test/resources", "util/util-core/src/main/scala/com/twitter/util", "util/util-registry/src/main/scala/com/twitter/util/registry", diff --git a/util-jvm/src/main/scala/com/twitter/jvm/BUILD b/util-jvm/src/main/scala/com/twitter/jvm/BUILD index 67957f1aac..a3931e358d 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/main/scala/com/twitter/jvm/BUILD @@ -6,7 +6,7 @@ scala_library( repo = artifactory, ), dependencies = [ - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/main/scala/com/twitter/util", "util/util-stats/src/main/scala/com/twitter/finagle/stats", diff --git a/util-jvm/src/test/scala/com/twitter/jvm/BUILD b/util-jvm/src/test/scala/com/twitter/jvm/BUILD index 71e4f440ed..396867d4eb 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/test/scala/com/twitter/jvm/BUILD @@ -10,7 +10,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/util", "util/util-jvm/src/main/scala/com/twitter/jvm", "util/util-logging/src/main/scala", diff --git a/util-logging/src/main/scala/com/twitter/logging/BUILD b/util-logging/src/main/scala/com/twitter/logging/BUILD index 67f2e55270..c9214067a1 100644 --- a/util-logging/src/main/scala/com/twitter/logging/BUILD +++ b/util-logging/src/main/scala/com/twitter/logging/BUILD @@ -7,13 +7,13 @@ scala_library( repo = artifactory, ), dependencies = [ - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/main/scala/com/twitter/util", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], exports = [ - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", ], ) diff --git a/util-logging/src/test/scala/com/twitter/logging/BUILD b/util-logging/src/test/scala/com/twitter/logging/BUILD index f0920cd015..2d6e110096 100644 --- a/util-logging/src/test/scala/com/twitter/logging/BUILD +++ b/util-logging/src/test/scala/com/twitter/logging/BUILD @@ -5,7 +5,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/io", "util/util-core/src/main/scala/com/twitter/util", "util/util-logging/src/main/scala/com/twitter/logging", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD index 03c25bf40f..fcb4206f5a 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD @@ -9,7 +9,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/util", ], diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD index a26d6ad8ed..0584e836b5 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD @@ -4,7 +4,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", - "util/util-app/src/main/scala/com/twitter/app", + "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/util", "util/util-tunable/src/main/scala/com/twitter/util/tunable", ], From 06dbfa9efc4961b7962d2e72e16f8f4891104237 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Wed, 14 Nov 2018 19:57:16 +0000 Subject: [PATCH 064/796] scalafix-rewrite: Fixing rules for RichU64 Two problems: - `java.lang.Long.toHexString` doesn't padd string with leading zeros - Missed base for `parseUnsignedLong`; should be 16 JIRA Issues: CSL-6030 Differential Revision: https://phabricator.twitter.biz/D242094 --- .../test/scala/com/twitter/util/U64Test.scala | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/util/U64Test.scala b/util-core/src/test/scala/com/twitter/util/U64Test.scala index c3e0961415..6273473424 100644 --- a/util-core/src/test/scala/com/twitter/util/U64Test.scala +++ b/util-core/src/test/scala/com/twitter/util/U64Test.scala @@ -1,12 +1,14 @@ package com.twitter.util import scala.util.Random - import org.junit.runner.RunWith +import org.scalacheck.Gen import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner +import org.scalatest.prop.GeneratorDrivenPropertyChecks + @RunWith(classOf[JUnitRunner]) -class U64Test extends WordSpec { +class U64Test extends WordSpec with GeneratorDrivenPropertyChecks { import U64._ "comparable" in { @@ -151,5 +153,16 @@ class U64Test extends WordSpec { // this is what Long.parseLong("") does intercept[NumberFormatException] { "".toU64Long } } + + "Java8 migrations" in { + forAll { l: Long => + assert(new RichU64Long(l).toU64HexString == "%016x".format(l)) + } + + forAll { l: Long => + val s = "%016x".format(l) + assert(new RichU64String(s).toU64Long == java.lang.Long.parseUnsignedLong(s, 16)) + } + } } } From ce41d10ef2cd72d252984e6f0f99991543d814f5 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 14 Nov 2018 20:16:56 +0000 Subject: [PATCH 065/796] util: Distinct PantsWorkaround class names Problem Util API docs disappeared, this is reported here https://github.com/twitter/util/issues/237. `./sbt unidoc` complains about same names of PantsWorkaround. Solution Rename those. Differential Revision: https://phabricator.twitter.biz/D241608 --- util-cache/src/main/scala/BUILD | 2 +- .../src/main/scala/PantsWorkaroundCache.scala | 2 +- util-collection/src/main/scala/BUILD | 2 +- .../scala/{PantsWorkaround.scala => PantsWorkaroundColl.scala} | 2 +- util-core/src/main/scala/BUILD | 2 +- .../scala/{PantsWorkaround.scala => PantsWorkaroundCore.scala} | 2 +- util-zk/src/main/scala/BUILD | 2 +- .../src/main/scala/PantsWorkaroundZk.scala | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename util-zk/src/main/scala/PantsWorkaround.scala => util-cache/src/main/scala/PantsWorkaroundCache.scala (79%) rename util-collection/src/main/scala/{PantsWorkaround.scala => PantsWorkaroundColl.scala} (80%) rename util-core/src/main/scala/{PantsWorkaround.scala => PantsWorkaroundCore.scala} (80%) rename util-cache/src/main/scala/PantsWorkaround.scala => util-zk/src/main/scala/PantsWorkaroundZk.scala (80%) diff --git a/util-cache/src/main/scala/BUILD b/util-cache/src/main/scala/BUILD index 40c2f0c505..b563354662 100644 --- a/util-cache/src/main/scala/BUILD +++ b/util-cache/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( # This target exists only to act as an aggregator jar. - sources = ["PantsWorkaround.scala"], + sources = ["PantsWorkaroundCache.scala"], provides = scala_artifact( org = "com.twitter", name = "util-cache", diff --git a/util-zk/src/main/scala/PantsWorkaround.scala b/util-cache/src/main/scala/PantsWorkaroundCache.scala similarity index 79% rename from util-zk/src/main/scala/PantsWorkaround.scala rename to util-cache/src/main/scala/PantsWorkaroundCache.scala index 9800d15646..2dccfb94cd 100644 --- a/util-zk/src/main/scala/PantsWorkaround.scala +++ b/util-cache/src/main/scala/PantsWorkaroundCache.scala @@ -1,4 +1,4 @@ // workaround for https://github.com/pantsbuild/pants/issues/6678 -private final class PantsWorkaround { +private final class PantsWorkaroundCache { throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") } diff --git a/util-collection/src/main/scala/BUILD b/util-collection/src/main/scala/BUILD index c63b628063..8857ace469 100644 --- a/util-collection/src/main/scala/BUILD +++ b/util-collection/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( # This target exists only to act as an aggregator jar. - sources = ["PantsWorkaround.scala"], + sources = ["PantsWorkaroundColl.scala"], provides = scala_artifact( org = "com.twitter", name = "util-collection", diff --git a/util-collection/src/main/scala/PantsWorkaround.scala b/util-collection/src/main/scala/PantsWorkaroundColl.scala similarity index 80% rename from util-collection/src/main/scala/PantsWorkaround.scala rename to util-collection/src/main/scala/PantsWorkaroundColl.scala index 9800d15646..b4b36f594b 100644 --- a/util-collection/src/main/scala/PantsWorkaround.scala +++ b/util-collection/src/main/scala/PantsWorkaroundColl.scala @@ -1,4 +1,4 @@ // workaround for https://github.com/pantsbuild/pants/issues/6678 -private final class PantsWorkaround { +private final class PantsWorkaroundColl { throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") } diff --git a/util-core/src/main/scala/BUILD b/util-core/src/main/scala/BUILD index 4c64da0f8e..afe099a500 100644 --- a/util-core/src/main/scala/BUILD +++ b/util-core/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( # This target exists only to act as an aggregator jar. - sources = ["PantsWorkaround.scala"], + sources = ["PantsWorkaroundCore.scala"], provides = scala_artifact( org = "com.twitter", name = "util-core", diff --git a/util-core/src/main/scala/PantsWorkaround.scala b/util-core/src/main/scala/PantsWorkaroundCore.scala similarity index 80% rename from util-core/src/main/scala/PantsWorkaround.scala rename to util-core/src/main/scala/PantsWorkaroundCore.scala index 9800d15646..8aa6cef74e 100644 --- a/util-core/src/main/scala/PantsWorkaround.scala +++ b/util-core/src/main/scala/PantsWorkaroundCore.scala @@ -1,4 +1,4 @@ // workaround for https://github.com/pantsbuild/pants/issues/6678 -private final class PantsWorkaround { +private final class PantsWorkaroundCore { throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") } diff --git a/util-zk/src/main/scala/BUILD b/util-zk/src/main/scala/BUILD index 688347c739..67043ece8a 100644 --- a/util-zk/src/main/scala/BUILD +++ b/util-zk/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( # This target exists only to act as an aggregator jar. - sources = ["PantsWorkaround.scala"], + sources = ["PantsWorkaroundZk.scala"], provides = scala_artifact( org = "com.twitter", name = "util-zk", diff --git a/util-cache/src/main/scala/PantsWorkaround.scala b/util-zk/src/main/scala/PantsWorkaroundZk.scala similarity index 80% rename from util-cache/src/main/scala/PantsWorkaround.scala rename to util-zk/src/main/scala/PantsWorkaroundZk.scala index 9800d15646..202482a88b 100644 --- a/util-cache/src/main/scala/PantsWorkaround.scala +++ b/util-zk/src/main/scala/PantsWorkaroundZk.scala @@ -1,4 +1,4 @@ // workaround for https://github.com/pantsbuild/pants/issues/6678 -private final class PantsWorkaround { +private final class PantsWorkaroundZk { throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") } From 1dc614bc7adf236ffe1e92b45fac35ea3241f6b0 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Thu, 15 Nov 2018 11:59:29 +0000 Subject: [PATCH 066/796] util-core: test all AsyncStream ctors and fix tail for Embed Problem AsyncStream has several implementations, but not all of them are covered by unit tests. The new tests revealed a small bug in the Embed case for tail. The future returned by tail should always resolve to Future.None if the AsyncStream is empty. This property aids clarity: None means there is nothing left to process, and Some(x) means that there is. Specifically, in the Some(x) case, x must not be empty. The bug in the Embed case is that it's possible to get Some(x), where x is empty. Solution In the Embed case, we should wait for the embedded (within the future) AsyncStream to materialize, then check if it's empty. And if the embedded AsyncStream resolves to another embedded AsyncStream, we should recurse until we can get our hands on a non-Embed AsyncStream. Differential Revision: https://phabricator.twitter.biz/D241513 --- CHANGELOG.rst | 6 + .../com/twitter/concurrent/AsyncStream.scala | 9 +- .../twitter/concurrent/AsyncStreamTest.scala | 1168 +++++++++-------- 3 files changed, 615 insertions(+), 568 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9582363d69..8de66e4cfe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Bug Fixes +~~~~~~~~~ + +* util-core: Fixed a bug where tail would sometimes return Some empty AsyncStream instead of None. + ``PHAB_ID=D241513`` + 18.11.0 ------- diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 7eecfb3d0d..9e41f301d9 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -55,7 +55,7 @@ sealed abstract class AsyncStream[+A] { */ def tail: Future[Option[AsyncStream[A]]] = this match { case Empty | FromFuture(_) => Future.None - case Cons(_, more) => Future.value(Some(more())) + case Cons(_, more) => extract(more()) case Embed(fas) => fas.flatMap(_.tail) } @@ -677,6 +677,13 @@ object AsyncStream { private[concurrent] def embed[A](fas: Future[AsyncStream[A]]): AsyncStream[A] = Embed(fas) + private def extract[A](as: AsyncStream[A]): Future[Option[AsyncStream[A]]] = + as match { + case Empty => Future.None + case Embed(fas) => fas.flatMap(extract) + case _ => Future.value(Some(as)) + } + /** * Java friendly [[AsyncStream.flatten]]. */ diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 55d457c046..20a8cb2a13 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -13,719 +13,726 @@ class AsyncStreamTest extends FunSuite with GeneratorDrivenPropertyChecks { import AsyncStream.{mk, of} import AsyncStreamTest._ - test("strict head") { - intercept[Exception] { (undefined: Unit) +:: AsyncStream.empty } - intercept[Exception] { mk(undefined, AsyncStream.empty) } - intercept[Exception] { of(undefined) } - } - - test("lazy tail") { - var forced = false - val s = () +:: { forced = true; AsyncStream.empty[Unit] } - assert(await(s.head) == Some(())) - assert(!forced) - await(s.tail) - assert(forced) - - var forced1 = false - val t = mk((), { forced1 = true; AsyncStream.empty[Unit] }) - assert(await(t.head) == Some(())) - assert(!forced1) - await(t.tail) - assert(forced1) - } + // Test all AsyncStream constructors: Empty, FromFuture, Cons, Embed. + seqImpl.foreach { impl => + def fromSeq[A](seq: Seq[A]): AsyncStream[A] = impl.apply(seq) - test("call-by-name tail evaluated at most once") { - val p = new Promise[Unit] - val s = () +:: { - if (p.setDone()) of(()) - else AsyncStream.empty[Unit] + test(s"$impl: strict head") { + intercept[Exception] { (undefined: Unit) +:: AsyncStream.empty } + intercept[Exception] { mk(undefined, AsyncStream.empty) } + intercept[Exception] { of(undefined) } } - assert(toSeq(s) == toSeq(s)) - } - test("ops that force tail evaluation") { - def isForced(f: AsyncStream[_] => Future[_]): Unit = { + test(s"$impl: lazy tail") { var forced = false - Await.ready(f(() +:: { forced = true; AsyncStream.empty })) + val s = () +:: { forced = true; AsyncStream.empty[Unit] } + assert(await(s.head) == Some(())) + assert(!forced) + await(s.tail) assert(forced) + + var forced1 = false + val t = mk((), { forced1 = true; AsyncStream.empty[Unit] }) + assert(await(t.head) == Some(())) + assert(!forced1) + await(t.tail) + assert(forced1) } - isForced(_.foldLeft(0)((_, _) => 0)) - isForced(_.foldLeftF(0)((_, _) => Future.value(0))) - isForced(_.tail) - } + test(s"$impl: call-by-name tail evaluated at most once") { + val p = new Promise[Unit] + val s = () +:: { + if (p.setDone()) of(()) + else AsyncStream.empty[Unit] + } + assert(toSeq(s) == toSeq(s)) + } - test("observe: failure") { - val s = 1 +:: 2 +:: (undefined: AsyncStream[Int]) - val (x +: y +: Nil, exc) = await(s.observe()) + test(s"$impl: ops that force tail evaluation") { + def isForced(f: AsyncStream[_] => Future[_]): Unit = { + var forced = false + Await.ready(f(() +:: { forced = true; AsyncStream.empty })) + assert(forced) + } - assert(x == 1) - assert(y == 2) - assert(exc.isDefined) - } + isForced(_.foldLeft(0)((_, _) => 0)) + isForced(_.foldLeftF(0)((_, _) => Future.value(0))) + isForced(_.tail) + } - test("observe: no failure") { - val s = 1 +:: 2 +:: AsyncStream.empty[Int] - val (x +: y +: Nil, exc) = await(s.observe()) + test(s"$impl: observe: failure") { + val s = 1 +:: 2 +:: (undefined: AsyncStream[Int]) + val (x +: y +: Nil, exc) = await(s.observe()) - assert(x == 1) - assert(y == 2) - assert(exc.isEmpty) - } + assert(x == 1) + assert(y == 2) + assert(exc.isDefined) + } - test("fromSeq works on infinite streams") { - def ones: Stream[Int] = 1 #:: ones - assert(toSeq(fromSeq(ones).take(3)) == Seq(1, 1, 1)) - } + test(s"$impl: observe: no failure") { + val s = 1 +:: 2 +:: AsyncStream.empty[Int] + val (x +: y +: Nil, exc) = await(s.observe()) - test("foreach") { - val x = new Promise[Unit] - val y = new Promise[Unit] + assert(x == 1) + assert(y == 2) + assert(exc.isEmpty) + } - def f() = { x.setDone(); () } - def g() = { y.setDone(); () } + test(s"$impl: fromSeq works on infinite streams") { + def ones: Stream[Int] = 1 #:: ones + assert(toSeq(fromSeq(ones).take(3)) == Seq(1, 1, 1)) + } - val s = () +:: f() +:: g() +:: AsyncStream.empty[Unit] - assert(!x.isDefined) - assert(!y.isDefined) + test(s"$impl: foreach") { + val x = new Promise[Unit] + val y = new Promise[Unit] - s.foreach(_ => ()) - assert(x.isDefined) - assert(y.isDefined) - } + def f() = { x.setDone(); () } + def g() = { y.setDone(); () } - test("lazy ops") { - val p = new Promise[Unit] - val s = () +:: { - p.setDone() - undefined: AsyncStream[Unit] + val s = () +:: f() +:: g() +:: AsyncStream.empty[Unit] + assert(!x.isDefined) + assert(!y.isDefined) + + s.foreach(_ => ()) + assert(x.isDefined) + assert(y.isDefined) } - s.map(x => 0) - assert(!p.isDefined) + test(s"$impl: lazy ops") { + val p = new Promise[Unit] + val s = () +:: { + p.setDone() + undefined: AsyncStream[Unit] + } - s.mapF(x => Future.True) - assert(!p.isDefined) + s.map(x => 0) + assert(!p.isDefined) - s.flatMap(x => of(x)) - assert(!p.isDefined) + s.mapF(x => Future.True) + assert(!p.isDefined) - s.filter(_ => true) - assert(!p.isDefined) + s.flatMap(x => of(x)) + assert(!p.isDefined) - s.withFilter(_ => true) - assert(!p.isDefined) + s.filter(_ => true) + assert(!p.isDefined) - s.take(Int.MaxValue) - assert(!p.isDefined) + s.withFilter(_ => true) + assert(!p.isDefined) - assert(toSeq(s.take(1)) == Seq(())) - assert(!p.isDefined) + s.take(Int.MaxValue) + assert(!p.isDefined) - s.takeWhile(_ => true) - assert(!p.isDefined) + assert(toSeq(s.take(1)) == Seq(())) + assert(!p.isDefined) - s.uncons - assert(!p.isDefined) + s.takeWhile(_ => true) + assert(!p.isDefined) - s.foldRight(Future.Done) { (_, _) => - Future.Done - } - assert(!p.isDefined) + s.uncons + assert(!p.isDefined) - s.scanLeft(Future.Done) { (_, _) => - Future.Done - } - assert(!p.isDefined) + s.foldRight(Future.Done) { (_, _) => + Future.Done + } + assert(!p.isDefined) - s ++ s - assert(!p.isDefined) + s.scanLeft(Future.Done) { (_, _) => + Future.Done + } + assert(!p.isDefined) - assert(await(s.head) == Some(())) - assert(!p.isDefined) + s ++ s + assert(!p.isDefined) - intercept[Exception] { await(s.tail).isEmpty } - assert(p.isDefined) - } + assert(await(s.head) == Some(())) + assert(!p.isDefined) - class Ctx[A](ops: AsyncStream[Int] => AsyncStream[A]) { - var once = 0 - val s: AsyncStream[Int] = 2 +:: { - once = once + 1 - if (once > 1) throw new Exception("evaluated more than once") - AsyncStream.of(1) + intercept[Exception] { await(s.tail).isEmpty } + assert(p.isDefined) } - val ss = ops(s) - ss.foreach(_ => ()) - // does not throw - ss.foreach(_ => ()) - } + class Ctx[A](ops: AsyncStream[Int] => AsyncStream[A]) { + var once = 0 + val s: AsyncStream[Int] = 2 +:: { + once = once + 1 + if (once > 1) throw new Exception("evaluated more than once") + AsyncStream.of(1) + } - test("memoized stream") { - new Ctx(s => s.map(_ => 0)) - new Ctx(s => s.mapF(_ => Future.value(1))) - new Ctx(s => s.flatMap(of(_))) - new Ctx(s => s.filter(_ => true)) - new Ctx(s => s.withFilter(_ => true)) - new Ctx(s => s.take(2)) - new Ctx(s => s.takeWhile(_ => true)) - new Ctx( - s => - s.scanLeft(Future.Done) { (_, _) => - Future.Done - } - ) - new Ctx(s => s ++ s) - } + val ss = ops(s) + ss.foreach(_ => ()) + // does not throw + ss.foreach(_ => ()) + } + + test(s"$impl: memoized stream") { + new Ctx(s => s.map(_ => 0)) + new Ctx(s => s.mapF(_ => Future.value(1))) + new Ctx(s => s.flatMap(of(_))) + new Ctx(s => s.filter(_ => true)) + new Ctx(s => s.withFilter(_ => true)) + new Ctx(s => s.take(2)) + new Ctx(s => s.takeWhile(_ => true)) + new Ctx( + s => + s.scanLeft(Future.Done) { (_, _) => + Future.Done + } + ) + new Ctx(s => s ++ s) + } - // Note: We could use ScalaCheck's Arbitrary[Function1] for some of the tests - // below, however ScalaCheck generates only constant functions which return - // the same value for any input. This makes it quite useless to us. We'll take - // another look since https://github.com/rickynils/scalacheck/issues/136 might - // have solved this issue. + // Note: We could use ScalaCheck's Arbitrary[Function1] for some of the tests + // below, however ScalaCheck generates only constant functions which return + // the same value for any input. This makes it quite useless to us. We'll take + // another look since https://github.com/rickynils/scalacheck/issues/136 might + // have solved this issue. - test("map") { - forAll { (s: List[Int]) => - def f(n: Int) = n.toString - assert(toSeq(fromSeq(s).map(f)) == s.map(f)) + test(s"$impl: map") { + forAll { (s: List[Int]) => + def f(n: Int) = n.toString + assert(toSeq(fromSeq(s).map(f)) == s.map(f)) + } } - } - test("mapF") { - forAll { (s: List[Int]) => - def f(n: Int) = n.toString - val g = f _ andThen Future.value - assert(toSeq(fromSeq(s).mapF(g)) == s.map(f)) + test(s"$impl: mapF") { + forAll { (s: List[Int]) => + def f(n: Int) = n.toString + val g = f _ andThen Future.value + assert(toSeq(fromSeq(s).mapF(g)) == s.map(f)) + } } - } - test("flatMap") { - forAll { (s: List[Int]) => - def f(n: Int) = n.toString - def g(a: Int): AsyncStream[String] = of(f(a)) - def h(a: Int): List[String] = List(f(a)) - assert(toSeq(fromSeq(s).flatMap(g)) == s.flatMap(h)) + test(s"$impl: flatMap") { + forAll { (s: List[Int]) => + def f(n: Int) = n.toString + def g(a: Int): AsyncStream[String] = of(f(a)) + def h(a: Int): List[String] = List(f(a)) + assert(toSeq(fromSeq(s).flatMap(g)) == s.flatMap(h)) + } } - } - test("filter") { - forAll { (s: List[Int]) => - def f(n: Int) = n % 3 == 0 - assert(toSeq(fromSeq(s).filter(f)) == s.filter(f)) + test(s"$impl: filter") { + forAll { (s: List[Int]) => + def f(n: Int) = n % 3 == 0 + assert(toSeq(fromSeq(s).filter(f)) == s.filter(f)) + } } - } - test("++") { - forAll { (a: List[Int], b: List[Int]) => - assert(toSeq(fromSeq(a) ++ fromSeq(b)) == a ++ b) + test(s"$impl: ++") { + forAll { (a: List[Int], b: List[Int]) => + assert(toSeq(fromSeq(a) ++ fromSeq(b)) == a ++ b) + } } - } - - test("++ with a long stream") { - var count = 0 - def genLongStream(len: Int): AsyncStream[Int] = - if (len == 0) { - AsyncStream.of(1) - } else { - count = count + 1 - 1 +:: genLongStream(len - 1) - } - // concat a long stream does not stack overflow - val s = genLongStream(1000000) ++ genLongStream(3) - s.foreach(_ => ()) - val first = count - s.foreach(_ => ()) - // the values are evaluated once - assert(count == first) - } - test("foldRight") { - forAll { (a: List[Int]) => - def f(n: Int, s: String) = (s.toLong + n).toString - def g(q: Int, p: => Future[String]): Future[String] = p.map(f(q, _)) - val m = fromSeq(a).foldRight(Future.value("0"))(g) - assert(await(m) == a.foldRight("0")(f)) + test(s"$impl: ++ with a long stream") { + var count = 0 + def genLongStream(len: Int): AsyncStream[Int] = + if (len == 0) { + AsyncStream.of(1) + } else { + count = count + 1 + 1 +:: genLongStream(len - 1) + } + // concat a long stream does not stack overflow + val s = genLongStream(1000000) ++ genLongStream(3) + s.foreach(_ => ()) + val first = count + s.foreach(_ => ()) + // the values are evaluated once + assert(count == first) + } + + test(s"$impl: foldRight") { + forAll { (a: List[Int]) => + def f(n: Int, s: String) = (s.toLong + n).toString + def g(q: Int, p: => Future[String]): Future[String] = p.map(f(q, _)) + val m = fromSeq(a).foldRight(Future.value("0"))(g) + assert(await(m) == a.foldRight("0")(f)) + } } - } - test("scanLeft") { - forAll { (a: List[Int]) => - def f(s: String, n: Int) = (s.toLong + n).toString - assert(toSeq(fromSeq(a).scanLeft("0")(f)) == a.scanLeft("0")(f)) + test(s"$impl: scanLeft") { + forAll { (a: List[Int]) => + def f(s: String, n: Int) = (s.toLong + n).toString + assert(toSeq(fromSeq(a).scanLeft("0")(f)) == a.scanLeft("0")(f)) + } } - } - test("scanLeft is eager") { - val never = AsyncStream.fromFuture(Future.never) - val hd = never.scanLeft("hi")((_, _) => ???).head - assert(hd.isDefined) - assert(await(hd) == Some("hi")) - } + test(s"$impl: scanLeft is eager") { + val never = AsyncStream.fromFuture(Future.never) + val hd = never.scanLeft("hi")((_, _) => ???).head + assert(hd.isDefined) + assert(await(hd) == Some("hi")) + } - test("scanLeft is eager for embed") { - val embed = AsyncStream.embed(Future.never) - val hd = embed.scanLeft("hi")((_, _) => ???).head - assert(hd.isDefined) - assert(await(hd) == Some("hi")) - } + test(s"$impl: scanLeft is eager for embed") { + val embed = AsyncStream.embed(Future.never) + val hd = embed.scanLeft("hi")((_, _) => ???).head + assert(hd.isDefined) + assert(await(hd) == Some("hi")) + } - test("foldLeft") { - forAll { (a: List[Int]) => - def f(s: String, n: Int) = (s.toLong + n).toString - assert(await(fromSeq(a).foldLeft("0")(f)) == a.foldLeft("0")(f)) + test(s"$impl: foldLeft") { + forAll { (a: List[Int]) => + def f(s: String, n: Int) = (s.toLong + n).toString + assert(await(fromSeq(a).foldLeft("0")(f)) == a.foldLeft("0")(f)) + } } - } - test("foldLeftF") { - forAll { (a: List[Int]) => - def f(s: String, n: Int) = (s.toLong + n).toString - val g: (String, Int) => Future[String] = (q, p) => Future.value(f(q, p)) - assert(await(fromSeq(a).foldLeftF("0")(g)) == a.foldLeft("0")(f)) + test(s"$impl: foldLeftF") { + forAll { (a: List[Int]) => + def f(s: String, n: Int) = (s.toLong + n).toString + val g: (String, Int) => Future[String] = (q, p) => Future.value(f(q, p)) + assert(await(fromSeq(a).foldLeftF("0")(g)) == a.foldLeft("0")(f)) + } } - } - test("flatten") { - val small = Gen.resize(10, Arbitrary.arbitrary[List[List[Int]]]) - forAll(small) { s => - assert(toSeq(fromSeq(s.map(fromSeq)).flatten) == s.flatten) + test(s"$impl: flatten") { + val small = Gen.resize(10, Arbitrary.arbitrary[List[List[Int]]]) + forAll(small) { s => + assert(toSeq(fromSeq(s.map(fromSeq)).flatten) == s.flatten) + } } - } - test("head") { - forAll { (a: List[Int]) => - assert(await(fromSeq(a).head) == a.headOption) + test(s"$impl: head") { + forAll { (a: List[Int]) => + assert(await(fromSeq(a).head) == a.headOption) + } } - } - test("isEmpty") { - val s = AsyncStream.of(1) - val tail = await(s.tail) - assert(tail == None) - } + test(s"$impl: isEmpty") { + val s = AsyncStream.of(1) + val tail = await(s.tail) + assert(tail == None) + } - test("tail") { - forAll(Gen.nonEmptyListOf(Arbitrary.arbitrary[Int])) { (a: List[Int]) => - val tail = await(fromSeq(a).tail) - a.tail match { - case Nil => assert(tail == None) - case _ => assert(toSeq(tail.get) == a.tail) + test(s"$impl: tail") { + forAll(Gen.nonEmptyListOf(Arbitrary.arbitrary[Int])) { (a: List[Int]) => + val tail = await(fromSeq(a).tail) + a.tail match { + case Nil => assert(tail == None) + case _ => assert(toSeq(tail.get) == a.tail) + } } } - } - test("uncons") { - assert(await(AsyncStream.empty.uncons) == None) - forAll(Gen.nonEmptyListOf(Arbitrary.arbitrary[Int])) { (a: List[Int]) => - val Some((h, t)) = await(fromSeq(a).uncons) - assert(h == a.head) - assert(toSeq(t()) == a.tail) + test(s"$impl: uncons") { + assert(await(AsyncStream.empty.uncons) == None) + forAll(Gen.nonEmptyListOf(Arbitrary.arbitrary[Int])) { (a: List[Int]) => + val Some((h, t)) = await(fromSeq(a).uncons) + assert(h == a.head) + assert(toSeq(t()) == a.tail) + } } - } - test("take") { - forAll(genListAndN) { - case (as, n) => - assert(toSeq(fromSeq(as).take(n)) == as.take(n)) + test(s"$impl: take") { + forAll(genListAndN) { + case (as, n) => + assert(toSeq(fromSeq(as).take(n)) == as.take(n)) + } } - } - test("drop") { - forAll(genListAndN) { - case (as, n) => - assert(toSeq(fromSeq(as).drop(n)) == as.drop(n)) + test(s"$impl: drop") { + forAll(genListAndN) { + case (as, n) => + assert(toSeq(fromSeq(as).drop(n)) == as.drop(n)) + } } - } - test("takeWhile") { - forAll(genListAndSentinel) { - case (as, x) => - assert(toSeq(fromSeq(as).takeWhile(_ != x)) == as.takeWhile(_ != x)) + test(s"$impl: takeWhile") { + forAll(genListAndSentinel) { + case (as, x) => + assert(toSeq(fromSeq(as).takeWhile(_ != x)) == as.takeWhile(_ != x)) + } } - } - test("dropWhile") { - forAll(genListAndSentinel) { - case (as, x) => - assert(toSeq(fromSeq(as).dropWhile(_ != x)) == as.dropWhile(_ != x)) + test(s"$impl: dropWhile") { + forAll(genListAndSentinel) { + case (as, x) => + assert(toSeq(fromSeq(as).dropWhile(_ != x)) == as.dropWhile(_ != x)) + } } - } - test("toSeq") { - forAll { (as: List[Int]) => - assert(await(fromSeq(as).toSeq()) == as) + test(s"$impl: toSeq") { + forAll { (as: List[Int]) => + assert(await(fromSeq(as).toSeq()) == as) + } } - } - test("identity") { - val small = Gen.resize(10, Arbitrary.arbitrary[List[Int]]) - forAll(small) { s => - val a = fromSeq(s) - def f(x: Int) = x +:: a + test(s"$impl: identity") { + val small = Gen.resize(10, Arbitrary.arbitrary[List[Int]]) + forAll(small) { s => + val a = fromSeq(s) + def f(x: Int) = x +:: a - assert(toSeq(of(1).flatMap(f)) == toSeq(f(1))) - assert(toSeq(a.flatMap(of)) == toSeq(a)) + assert(toSeq(of(1).flatMap(f)) == toSeq(f(1))) + assert(toSeq(a.flatMap(of)) == toSeq(a)) + } } - } - test("associativity") { - val small = Gen.resize(10, Arbitrary.arbitrary[List[Int]]) - forAll(small, small, small) { (s, t, u) => - val a = fromSeq(s) - val b = fromSeq(t) - val c = fromSeq(u) + test(s"$impl: associativity") { + val small = Gen.resize(10, Arbitrary.arbitrary[List[Int]]) + forAll(small, small, small) { (s, t, u) => + val a = fromSeq(s) + val b = fromSeq(t) + val c = fromSeq(u) - def f(x: Int) = x +:: b - def g(x: Int) = x +:: c + def f(x: Int) = x +:: b + def g(x: Int) = x +:: c - val v = a.flatMap(f).flatMap(g) - val w = a.flatMap(x => f(x).flatMap(g)) - assert(toSeq(v) == toSeq(w)) + val v = a.flatMap(f).flatMap(g) + val w = a.flatMap(x => f(x).flatMap(g)) + assert(toSeq(v) == toSeq(w)) + } } - } - test("buffer() works like Seq.splitAt") { - forAll { (items: List[Char], bufferSize: Int) => - val (expectedBuffer, expectedRest) = items.splitAt(bufferSize) - val (buffer, rest) = await(fromSeq(items).buffer(bufferSize)) - assert(expectedBuffer == buffer) - assert(expectedRest == toSeq(rest())) + test(s"$impl: buffer() works like Seq.splitAt") { + forAll { (items: List[Char], bufferSize: Int) => + val (expectedBuffer, expectedRest) = items.splitAt(bufferSize) + val (buffer, rest) = await(fromSeq(items).buffer(bufferSize)) + assert(expectedBuffer == buffer) + assert(expectedRest == toSeq(rest())) + } } - } - test("buffer() has the same properties as take() and drop()") { - // We need items to be non-empty, because AsyncStream.empty ++ - // forces the future to be created. - val gen = Gen.zip(Gen.nonEmptyListOf(Arbitrary.arbitrary[Char]), Arbitrary.arbitrary[Int]) - - forAll(gen) { - case (items, n) => - var forced1 = false - val stream1 = fromSeq(items) ++ { forced1 = true; AsyncStream.empty[Char] } - var forced2 = false - val stream2 = fromSeq(items) ++ { forced2 = true; AsyncStream.empty[Char] } - - val takeResult = toSeq(stream2.take(n)) - val (bufferResult, bufferRest) = await(stream1.buffer(n)) - assert(takeResult == bufferResult) - - // Strictness property: we should only need to force the full - // stream if we asked for more items that were present in the - // stream. - assert(forced1 == (n > items.size)) - assert(forced1 == forced2) - val wasForced = forced1 - - // Strictness property: Since AsyncStream contains a Future - // rather than a thunk, we need to evaluate the next element in - // order to get the result of drop and the rest of the stream - // after buffering. - val bufferTail = bufferRest() - val dropTail = stream2.drop(n) - assert(forced1 == (n >= items.size)) - assert(forced1 == forced2) - - // This is the only case that should have caused the item to be forced. - assert((wasForced == forced1) || n == items.size) - - // Forcing the rest of the sequence should always cause evaluation. - assert(toSeq(bufferTail) == toSeq(dropTail)) - assert(forced1) - assert(forced2) + test(s"$impl: buffer() has the same properties as take() and drop()") { + // We need items to be non-empty, because AsyncStream.empty ++ + // forces the future to be created. + val gen = Gen.zip(Gen.nonEmptyListOf(Arbitrary.arbitrary[Char]), Arbitrary.arbitrary[Int]) + + forAll(gen) { + case (items, n) => + var forced1 = false + val stream1 = fromSeq(items) ++ { forced1 = true; AsyncStream.empty[Char] } + var forced2 = false + val stream2 = fromSeq(items) ++ { forced2 = true; AsyncStream.empty[Char] } + + val takeResult = toSeq(stream2.take(n)) + val (bufferResult, bufferRest) = await(stream1.buffer(n)) + assert(takeResult == bufferResult) + + // Strictness property: we should only need to force the full + // stream if we asked for more items that were present in the + // stream. + assert(forced1 == (n > items.size)) + assert(forced1 == forced2) + val wasForced = forced1 + + // Strictness property: Since AsyncStream contains a Future + // rather than a thunk, we need to evaluate the next element in + // order to get the result of drop and the rest of the stream + // after buffering. + val bufferTail = bufferRest() + val dropTail = stream2.drop(n) + assert(forced1 == (n >= items.size)) + assert(forced1 == forced2) + + // This is the only case that should have caused the item to be forced. + assert((wasForced == forced1) || n == items.size) + + // Forcing the rest of the sequence should always cause evaluation. + assert(toSeq(bufferTail) == toSeq(dropTail)) + assert(forced1) + assert(forced2) + } } - } - test("grouped() works like Seq.grouped") { - forAll { (items: Seq[Char], groupSize: Int) => - // This is a Try so that we can test that bad inputs act the - // same. (Zero or negative group sizes throw the same - // exception.) - val expected = Try(items.grouped(groupSize).toSeq) - val actual = Try(toSeq(fromSeq(items).grouped(groupSize))) - - // If they are both exceptions, then pass if the exceptions are - // the same type (don't require them to define equality or have - // the same exception message) - (actual, expected) match { - case (Throw(e1), Throw(e2)) => assert(e1.getClass == e2.getClass) - case _ => assert(actual == expected) + test(s"$impl: grouped() works like Seq.grouped") { + forAll { (items: Seq[Char], groupSize: Int) => + // This is a Try so that we can test that bad inputs act the + // same. (Zero or negative group sizes throw the same + // exception.) + val expected = Try(items.grouped(groupSize).toSeq) + val actual = Try(toSeq(fromSeq(items).grouped(groupSize))) + + // If they are both exceptions, then pass if the exceptions are + // the same type (don't require them to define equality or have + // the same exception message) + (actual, expected) match { + case (Throw(e1), Throw(e2)) => assert(e1.getClass == e2.getClass) + case _ => assert(actual == expected) + } } } - } - test("grouped should be lazy") { - val gen = - for { - // We need items to be non-empty, because AsyncStream.empty ++ - // forces the future to be created. - items <- Gen.nonEmptyListOf(Arbitrary.arbitrary[Char]) - - // We need to make sure that the chunk size (1) is valid and (2) - // is short enough that forcing the first group does not force - // the exception. - groupSize <- Gen.chooseNum(1, items.size) - } yield (items, groupSize) - - forAll(gen) { - case (items, groupSize) => - var forced = false - val stream: AsyncStream[Char] = fromSeq(items) ++ { forced = true; AsyncStream.empty } - - val expected = items.grouped(groupSize).toSeq.headOption - // This will take up to items.size items from the stream. This - // does not require forcing the tail. - val actual = await(stream.grouped(groupSize).head) - assert(actual == expected) - assert(!forced) - val expectedChunks = items.grouped(groupSize).toSeq - val allChunks = toSeq(stream.grouped(groupSize)) - assert(allChunks == expectedChunks) - assert(forced) + test(s"$impl: grouped should be lazy") { + val gen = + for { + // We need items to be non-empty, because AsyncStream.empty ++ + // forces the future to be created. + items <- Gen.nonEmptyListOf(Arbitrary.arbitrary[Char]) + + // We need to make sure that the chunk size (1) is valid and (2) + // is short enough that forcing the first group does not force + // the exception. + groupSize <- Gen.chooseNum(1, items.size) + } yield (items, groupSize) + + forAll(gen) { + case (items, groupSize) => + var forced = false + val stream: AsyncStream[Char] = fromSeq(items) ++ { forced = true; AsyncStream.empty } + + val expected = items.grouped(groupSize).toSeq.headOption + // This will take up to items.size items from the stream. This + // does not require forcing the tail. + val actual = await(stream.grouped(groupSize).head) + assert(actual == expected) + assert(!forced) + val expectedChunks = items.grouped(groupSize).toSeq + val allChunks = toSeq(stream.grouped(groupSize)) + assert(allChunks == expectedChunks) + assert(forced) + } } - } - test("mapConcurrent preserves items") { - forAll(Arbitrary.arbitrary[List[Int]], Gen.choose(1, 10)) { (xs, conc) => - assert(toSeq(AsyncStream.fromSeq(xs).mapConcurrent(conc)(Future.value)).sorted == xs.sorted) + test(s"$impl: mapConcurrent preserves items") { + forAll(Arbitrary.arbitrary[List[Int]], Gen.choose(1, 10)) { (xs, conc) => + assert(toSeq(fromSeq(xs).mapConcurrent(conc)(Future.value)).sorted == xs.sorted) + } } - } - test("mapConcurrent makes progress when an item is blocking") { - forAll(Arbitrary.arbitrary[List[Int]], Gen.choose(2, 10)) { (xs, conc) => - // This promise is not satisfied, which would block the evaluation - // of .map, and should not block .mapConcurrent when conc > 1 - val first = new Promise[Int] - - // This function will return a blocking future the first time it - // is called and an immediately-available future thereafter. - var used = false - def f(x: Int) = - if (used) { - Future.value(x) - } else { - used = true - first - } + test(s"$impl: mapConcurrent makes progress when an item is blocking") { + forAll(Arbitrary.arbitrary[List[Int]], Gen.choose(2, 10)) { (xs, conc) => + // This promise is not satisfied, which would block the evaluation + // of .map, and should not block .mapConcurrent when conc > 1 + val first = new Promise[Int] + + // This function will return a blocking future the first time it + // is called and an immediately-available future thereafter. + var used = false + def f(x: Int) = + if (used) { + Future.value(x) + } else { + used = true + first + } - // Concurrently map over the stream. The whole stream should be - // available, except for one item which is still blocked. - val mapped = AsyncStream.fromSeq(xs).mapConcurrent(conc)(f) + // Concurrently map over the stream. The whole stream should be + // available, except for one item which is still blocked. + val mapped = fromSeq(xs).mapConcurrent(conc)(f) - // All but the first value, which is still blocking, has been returned - assert(toSeq(mapped.take(xs.length - 1)).sorted == xs.drop(1).sorted) + // All but the first value, which is still blocking, has been returned + assert(toSeq(mapped.take(xs.length - 1)).sorted == xs.drop(1).sorted) - if (xs.nonEmpty) { - // The stream as a whole is still blocking on the unsatisfied promise - assert(!mapped.foreach(_ => ()).isDefined) + if (xs.nonEmpty) { + // The stream as a whole is still blocking on the unsatisfied promise + assert(!mapped.foreach(_ => ()).isDefined) - // Unblock the first value - first.setValue(xs.head) - } + // Unblock the first value + first.setValue(xs.head) + } - // Now the whole stream should be available and should contain all - // of the items, ignoring order (but preserving repetition) - assert(mapped.foreach(_ => ()).isDefined) - assert(toSeq(mapped).sorted == xs.sorted) + // Now the whole stream should be available and should contain all + // of the items, ignoring order (but preserving repetition) + assert(mapped.foreach(_ => ()).isDefined) + assert(toSeq(mapped).sorted == xs.sorted) + } } - } - test("mapConcurrent is lazy once it reaches its concurrency limit") { - forAll(Gen.choose(2, 10), Arbitrary.arbitrary[Seq[Int]]) { (conc, xs) => - val q = new scala.collection.mutable.Queue[Promise[Unit]] + test(s"$impl: mapConcurrent is lazy once it reaches its concurrency limit") { + forAll(Gen.choose(2, 10), Arbitrary.arbitrary[Seq[Int]]) { (conc, xs) => + val q = new scala.collection.mutable.Queue[Promise[Unit]] - val mapped = - AsyncStream.fromSeq(xs).mapConcurrent(conc) { _ => - val p = new Promise[Unit] - q.enqueue(p) - p - } + val mapped = + fromSeq(xs).mapConcurrent(conc) { _ => + val p = new Promise[Unit] + q.enqueue(p) + p + } - // If there are at least `conc` items in the queue, then we should - // have started exactly `conc` of them. Otherwise, we should have - // started all of them. - assert(q.size == conc.min(xs.size)) + // If there are at least `conc` items in the queue, then we should + // have started exactly `conc` of them. Otherwise, we should have + // started all of them. + assert(q.size == conc.min(xs.size)) - if (xs.nonEmpty) { - assert(!mapped.head.isDefined) + if (xs.nonEmpty) { + assert(!mapped.head.isDefined) - val p = q.dequeue() - p.setDone() - } + val p = q.dequeue() + p.setDone() + } - // Satisfying that promise makes the head of the queue available. - assert(mapped.head.isDefined) + // Satisfying that promise makes the head of the queue available. + assert(mapped.head.isDefined) - if (xs.size > 1) { - // We do not add another element to the queue until the next - // element is forced. - assert(q.size == (conc.min(xs.size) - 1)) + if (xs.size > 1) { + // We do not add another element to the queue until the next + // element is forced. + assert(q.size == (conc.min(xs.size) - 1)) - val tl = mapped.drop(1) - assert(!tl.head.isDefined) + val tl = mapped.drop(1) + assert(!tl.head.isDefined) - // Forcing the next element of the queue causes us to enqueue - // one more element (if there are more elements to enqueue) - assert(q.size == conc.min(xs.size - 1)) + // Forcing the next element of the queue causes us to enqueue + // one more element (if there are more elements to enqueue) + assert(q.size == conc.min(xs.size - 1)) - val p = q.dequeue() - p.setDone() + val p = q.dequeue() + p.setDone() - // Satisfying that promise causes the head to be available. - assert(tl.head.isDefined) + // Satisfying that promise causes the head to be available. + assert(tl.head.isDefined) + } } } - } - test("mapConcurrent makes progress, even with blocking streams and blocking work") { - val gen = - Gen.zip( - Gen.choose(0, 10).label("numActions"), - Gen.choose(0, 10).flatMap(Gen.listOfN(_, Arbitrary.arbitrary[Int])), - Gen.choose(1, 11).label("concurrency") - ) - - forAll(gen) { - case (numActions, items, concurrency) => - val input: AsyncStream[Int] = - AsyncStream.fromSeq(items) ++ AsyncStream.fromFuture(Future.never) - - var workStarted = 0 - var workFinished = 0 - val result = - input.mapConcurrent(concurrency) { i => - workStarted += 1 - if (workFinished < numActions) { - workFinished += 1 - Future.value(i) - } else { - // After numActions evaluations, return a Future that - // will never be satisfied. - Future.never + test(s"$impl: mapConcurrent makes progress, even with blocking streams and blocking work") { + val gen = + Gen.zip( + Gen.choose(0, 10).label("numActions"), + Gen.choose(0, 10).flatMap(Gen.listOfN(_, Arbitrary.arbitrary[Int])), + Gen.choose(1, 11).label("concurrency") + ) + + forAll(gen) { + case (numActions, items, concurrency) => + val input: AsyncStream[Int] = + fromSeq(items) ++ AsyncStream.fromFuture(Future.never) + + var workStarted = 0 + var workFinished = 0 + val result = + input.mapConcurrent(concurrency) { i => + workStarted += 1 + if (workFinished < numActions) { + workFinished += 1 + Future.value(i) + } else { + // After numActions evaluations, return a Future that + // will never be satisfied. + Future.never + } } - } - // How much work should have been started by mapConcurrent. - val expectedStarted = items.size.min(concurrency) - assert(workStarted == expectedStarted, "work started") + // How much work should have been started by mapConcurrent. + val expectedStarted = items.size.min(concurrency) + assert(workStarted == expectedStarted, "work started") - val expectedFinished = numActions.min(expectedStarted) - assert(workFinished == expectedFinished, "expected finished") + val expectedFinished = numActions.min(expectedStarted) + assert(workFinished == expectedFinished, "expected finished") - // Make sure that all of the finished items are now - // available. (As a side-effect, this will force more work to - // be done if concurrency was the limiting factor.) - val completed = toSeq(result.take(workFinished)) - assert(completed.toSet.subsetOf(items.toSet)) + // Make sure that all of the finished items are now + // available. (As a side-effect, this will force more work to + // be done if concurrency was the limiting factor.) + val completed = toSeq(result.take(workFinished)).sorted + val expectedCompleted = items.take(expectedFinished).sorted + assert(completed == expectedCompleted) + } } - } - test("sum") { - forAll { xs: List[Int] => - assert(xs.sum == await(AsyncStream.fromSeq(xs).sum)) + test(s"$impl: sum") { + forAll { xs: List[Int] => + assert(xs.sum == await(fromSeq(xs).sum)) + } } - } - test("size") { - forAll { xs: List[Int] => - assert(xs.size == await(AsyncStream.fromSeq(xs).size)) + test(s"$impl: size") { + forAll { xs: List[Int] => + assert(xs.size == await(fromSeq(xs).size)) + } } - } - test("force") { - forAll { xs: List[Int] => - val p = new Promise[Unit] - // The promise will be defined iff the tail is forced. - val s = AsyncStream.fromSeq(xs) ++ { p.setDone(); AsyncStream.empty } + test(s"$impl: force") { + forAll { xs: List[Int] => + val p = new Promise[Unit] + // The promise will be defined iff the tail is forced. + val s = fromSeq(xs) ++ { p.setDone(); AsyncStream.empty } - // If the input is empty, then the tail will be forced right away. - assert(p.isDefined == xs.isEmpty) + // If the input is empty, then the tail will be forced right away. + assert(p.isDefined == xs.isEmpty) - // Unconditionally force the whole stream - await(s.force) - assert(p.isDefined) + // Unconditionally force the whole stream + await(s.force) + assert(p.isDefined) + } } - } - test("withEffect") { - forAll(genListAndN) { - case (xs, n) => - var i = 0 - val s = AsyncStream.fromSeq(xs).withEffect(_ => i += 1) + test(s"$impl: withEffect") { + forAll(genListAndN) { + case (xs, n) => + var i = 0 + val s = fromSeq(xs).withEffect(_ => i += 1) - // Is lazy on initial application (with the exception of the first element) - assert(i == (if (xs.isEmpty) 0 else 1)) + // Is lazy on initial application (with the exception of the first element) + assert(i == (if (xs.isEmpty) 0 else 1)) - // Is lazy when consuming the stream - await(s.take(n).force) + // Is lazy when consuming the stream + await(s.take(n).force) - // If the list is empty, no effects should occur. If the list is - // non-empty, the effect will occur for the first item right away, - // since the head is not lazy. Otherwise, we expect the same - // number of effects as items demanded. - val expected = if (xs.isEmpty) 0 else 1.max(xs.length.min(n)) - assert(i == expected) + // If the list is empty, no effects should occur. If the list is + // non-empty, the effect will occur for the first item right away, + // since the head is not lazy. Otherwise, we expect the same + // number of effects as items demanded. + val expected = if (xs.isEmpty) 0 else 1.max(xs.length.min(n)) + assert(i == expected) - // Preserves the elements in the stream - assert(toSeq(s) == xs) + // Preserves the elements in the stream + assert(toSeq(s) == xs) + } } - } - test("merge generates a stream equal to all input streams") { - forAll { (lists: Seq[List[Int]]) => - val streams = lists.map(fromSeq) - val merged = AsyncStream.merge(streams: _*) + test(s"$impl: merge generates a stream equal to all input streams") { + forAll { (lists: Seq[List[Int]]) => + val streams = lists.map(fromSeq) + val merged = AsyncStream.merge(streams: _*) - val input = AsyncStream(streams: _*).flatten + val input = AsyncStream(streams: _*).flatten - assert(toSeq(input).sorted == toSeq(merged).sorted) + assert(toSeq(input).sorted == toSeq(merged).sorted) + } } - } - test("merge fails the result stream if an input stream fails") { - forAll() { (lists: Seq[List[Int]]) => - val s = mk(1, undefined: AsyncStream[Int]) - val streams = s +: lists.map(fromSeq) - val merged = AsyncStream.merge(streams: _*) + test(s"$impl: merge fails the result stream if an input stream fails") { + forAll() { (lists: Seq[List[Int]]) => + val s = mk(1, undefined: AsyncStream[Int]) + val streams = s +: lists.map(fromSeq) + val merged = AsyncStream.merge(streams: _*) - intercept[Exception](toSeq(merged)) + intercept[Exception](toSeq(merged)) + } } - } - test("merged stream contains elements as they become available from input streams") { - forAll { (input: List[Int]) => - val promises = List.fill(input.size)(Promise[Int]()) + test(s"$impl: merged stream contains elements as they become available from input streams") { + forAll { (input: List[Int]) => + val promises = List.fill(input.size)(Promise[Int]()) - // grouped into lists of 10 elements each - val grouped = promises.grouped(10).toList + // grouped into lists of 10 elements each + val grouped = promises.grouped(10).toList - // merged list of streams - val streams = grouped.map(fromSeq(_).flatMap(AsyncStream.fromFuture)) - val merged = AsyncStream.merge(streams: _*).toSeq() + // merged list of streams + val streams = grouped.map(fromSeq(_).flatMap(AsyncStream.fromFuture)) + val merged = AsyncStream.merge(streams: _*).toSeq() - // build an interleaved list of the promises for the stream - // [s1(1), s2(1), s3(1), s1(2), s2(2), s3(2), ...] - val interleavedHeads = grouped.flatMap(_.zipWithIndex).sortBy(_._2).map(_._1) - interleavedHeads.zip(input).foreach { - case (p, i) => - p.update(Return(i)) - } + // build an interleaved list of the promises for the stream + // [s1(1), s2(1), s3(1), s1(2), s2(2), s3(2), ...] + val interleavedHeads = grouped.flatMap(_.zipWithIndex).sortBy(_._2).map(_._1) + interleavedHeads.zip(input).foreach { + case (p, i) => + p.update(Return(i)) + } - assert(Await.result(merged) == input) + assert(Await.result(merged) == input) + } } - } - test("exception produces a failed stream") { - intercept[Exception]( - toSeq(AsyncStream.exception(new Exception())) - ) - } + test(s"$impl: exception produces a failed stream") { + intercept[Exception]( + toSeq(AsyncStream.exception(new Exception())) + ) + } - test("exception eventually produces a failed stream") { - forAll { (xs: List[Int]) => - val stream = fromSeq(xs) ++ AsyncStream.exception(new Exception()) - intercept[Exception](toSeq(stream)) + test(s"$impl: exception eventually produces a failed stream") { + forAll { (xs: List[Int]) => + val stream = fromSeq(xs) ++ AsyncStream.exception(new Exception()) + intercept[Exception](toSeq(stream)) + } } + } } @@ -748,12 +755,39 @@ private object AsyncStreamTest { def toSeq[A](s: AsyncStream[A]): Seq[A] = await(s.toSeq()) - def fromSeq[A](s: Seq[A]): AsyncStream[A] = - // Test all AsyncStream constructors: Empty, FromFuture, Cons, Embed. - s match { + sealed trait FromSeq { + def apply[A](seq: Seq[A]): AsyncStream[A] + } + + private case object Cons extends FromSeq { + def apply[A](seq: Seq[A]): AsyncStream[A] = seq match { + case Nil => AsyncStream.empty + case a +: as => a +:: apply(as) + } + } + + private case object EmbeddedCons extends FromSeq { + def apply[A](seq: Seq[A]): AsyncStream[A] = seq match { + case Nil => AsyncStream.embed(Future.value(AsyncStream.empty)) + case a +: as => AsyncStream.embed(Future.value(a +:: apply(as))) + } + } + + private case object OfFuture extends FromSeq { + def apply[A](seq: Seq[A]): AsyncStream[A] = seq match { case Nil => AsyncStream.empty case a +: Nil => AsyncStream.of(a) - case a +: b +: Nil => AsyncStream.embed(Future.value(a +:: AsyncStream.of(b))) - case a +: as => a +:: fromSeq(as) + case a +: as => a +:: apply(as) } + } + + private case object EmbeddedFuture extends FromSeq { + def apply[A](seq: Seq[A]): AsyncStream[A] = seq match { + case Nil => AsyncStream.embed(Future.value(AsyncStream.empty)) + case a +: Nil => AsyncStream.embed(Future.value(AsyncStream.of(a))) + case a +: as => AsyncStream.embed(Future.value(a +:: apply(as))) + } + } + + def seqImpl: Seq[FromSeq] = Seq(Cons, EmbeddedCons, OfFuture, EmbeddedFuture) } From 5e0e1073f0694940b94e6e335df2657e400e3a1b Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Fri, 16 Nov 2018 22:26:43 +0000 Subject: [PATCH 067/796] util-core: simpler and faster AsyncStream.mapConcurrent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem / Solution Simpler and faster AsyncStream.mapConcurrent. Before Benchmark (concurrency) (impl) (size) Mode Cnt Score Error Units AsyncStreamBenchmark.mapConcurrent 5 cons 20 avgt 10 98032.513 ± 2201.248 ns/op AsyncStreamBenchmark.mapConcurrent:·gc.alloc.rate.norm 5 cons 20 avgt 10 119544.052 ± 0.059 B/op AsyncStreamBenchmark.mapConcurrent 5 embed 20 avgt 10 89167.779 ± 7870.823 ns/op AsyncStreamBenchmark.mapConcurrent:·gc.alloc.rate.norm 5 embed 20 avgt 10 111960.046 ± 0.048 B/op After Benchmark (concurrency) (impl) (size) Mode Cnt Score Error Units AsyncStreamBenchmark.mapConcurrent 5 cons 20 avgt 10 48305.789 ± 1843.003 ns/op AsyncStreamBenchmark.mapConcurrent:·gc.alloc.rate.norm 5 cons 20 avgt 10 67432.025 ± 0.028 B/op AsyncStreamBenchmark.mapConcurrent 5 embed 20 avgt 10 53308.644 ± 1296.209 ns/op AsyncStreamBenchmark.mapConcurrent:·gc.alloc.rate.norm 5 embed 20 avgt 10 75736.028 ± 0.032 B/op JIRA Issues: CSL-7226 Differential Revision: https://phabricator.twitter.biz/D240056 --- .../concurrent/AsyncStreamBenchmark.scala | 21 ++++- .../com/twitter/concurrent/AsyncStream.scala | 94 +++++++++++++++++-- .../twitter/concurrent/AsyncStreamTest.scala | 13 ++- 3 files changed, 113 insertions(+), 15 deletions(-) diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala index 1b5f3e6115..89d1c1854d 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala @@ -11,6 +11,14 @@ class AsyncStreamBenchmark extends StdBenchAnnotations { @Param(Array("10")) var size: Int = _ + /** Implementation */ + @Param(Array("cons", "embed")) + var impl: String = _ + + /** Concurrency Level */ + @Param(Array("2")) + var concurrency: Int = _ + private[this] val timeout = 5.seconds private[this] var as: AsyncStream[Int] = _ @@ -25,7 +33,14 @@ class AsyncStreamBenchmark extends StdBenchAnnotations { @Setup(Level.Iteration) def setup(): Unit = { - as = AsyncStream.fromSeq(0.until(size)) + as = impl match { + case "cons" => + AsyncStream.fromSeq(0.until(size)) + case "embed" => + 0.until(size).foldRight(AsyncStream.empty[Int]) { (i, tail) => + AsyncStream.embed(Future.value(i +:: tail)) + } + } longs = genLongStream(1000) } @@ -52,7 +67,7 @@ class AsyncStreamBenchmark extends StdBenchAnnotations { @Benchmark def mapConcurrent(): Seq[Int] = - Await.result(as.mapConcurrent(2)(MapConcurrentFn).toSeq()) + Await.result(as.mapConcurrent(concurrency)(MapConcurrentFn).toSeq()) private[this] val FilterFn: Int => Boolean = x => x % 2 == 0 @@ -71,4 +86,4 @@ class AsyncStreamBenchmark extends StdBenchAnnotations { @Benchmark def ++(): Int = Await.result((longs ++ as).foldLeft(0)(_ + _)) -} +} \ No newline at end of file diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 9e41f301d9..564221df3e 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -1,6 +1,6 @@ package com.twitter.concurrent -import com.twitter.util.{Future, Return, Throw} +import com.twitter.util.{Future, Return, Throw, Promise} import scala.annotation.varargs import scala.collection.mutable @@ -722,13 +722,86 @@ object AsyncStream { // Oneshot turns an AsyncStream into a resource, where each read depletes the // stream of that item. In other words, each item of the stream is observable // only once. - private class Oneshot[A](var as: AsyncStream[A]) { - def read(): Future[Option[A]] = synchronized { - val head = as.head - as = as.drop(1) - head + // + // more must be guarded by synchronization on this. + private final class Oneshot[A](var more: () => AsyncStream[A]) { + def read(): Future[Option[A]] = { + // References to readEmbed args for the Embed case. + var fas: Future[AsyncStream[A]] = null + var headp: Promise[Option[A]] = null + var tailp: Promise[() => AsyncStream[A]] = null + + // Reference to head for the default case. + var headf: Future[Option[A]] = null + + synchronized { + more() match { + case Embed(embedded) => + // The Embed case is special because + // + // val head = as.head + // as = as.drop(1) + // + // reduces to a chain of maps, which can be very long, as many as the + // number of times read is called while waiting for the embedded + // AsyncStream. + // + // val head = fas.map(_.drop(1)).map(_.drop(1))....flatMap(_.head) + // as = Embed(fas.map(_.drop(1)).map(_.drop(1))...) + // + // In other words, for AsyncStreams with Embed tails, the naive + // implementation multiplies the work of each item by the concurrency + // level. + // + // The optimization here is straight forward: create a head and tail + // promise and that wait for the embedded AsyncStream. Subsequent + // calls to read will create new head and tail promises that wait on + // the previous tail promise. + fas = embedded + headp = new Promise[Option[A]] + tailp = new Promise[() => AsyncStream[A]] + more = () => AsyncStream.embed(tailp.map(_())) + headp + + case as => + headf = as.head + more = () => as.drop(1) + } + } + + // Non-null fas implies Embed case. + if (fas != null) { + readEmbed(fas, headp, tailp) + headp + } else { + headf + } } + private[this] def readEmbed( + fas: Future[AsyncStream[A]], + headp: Promise[Option[A]], + tailp: Promise[() => AsyncStream[A]] + ): Unit = + fas.respond { + case Throw(e) => + headp.setException(e) + tailp.setException(e) + case Return(v) => v match { + case Empty => + headp.become(Future.None) + tailp.setValue(Oneshot.empty) + case FromFuture(fa) => + headp.become(fa.map(Some(_))) + tailp.setValue(Oneshot.empty) + case Cons(fa, more2) => + headp.become(fa.map(Some(_))) + tailp.setValue(more2) + case Embed(newFas) => + readEmbed(newFas, headp, tailp) + } + } + // Recover the AsyncStream interface. We don't return `as` because we want // to deplete the original stream with each read. With this we can create // multiple AsyncStream views of the same Oneshot resource, thus "fanning @@ -740,8 +813,13 @@ object AsyncStream { } } + private object Oneshot { + val emptyVal: () => AsyncStream[Nothing] = () => AsyncStream.empty[Nothing] + def empty[A]: () => AsyncStream[A] = emptyVal.asInstanceOf[() => AsyncStream[A]] + } + private def fanout[A](as: AsyncStream[A], n: Int): Seq[AsyncStream[A]] = { - val reader = new Oneshot(as) + val reader = new Oneshot(() => as) 0.until(n).map(_ => reader.toAsyncStream) } -} +} \ No newline at end of file diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 20a8cb2a13..1a4c648822 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -619,12 +619,17 @@ class AsyncStreamTest extends FunSuite with GeneratorDrivenPropertyChecks { // Make sure that all of the finished items are now // available. (As a side-effect, this will force more work to // be done if concurrency was the limiting factor.) - val completed = toSeq(result.take(workFinished)).sorted - val expectedCompleted = items.take(expectedFinished).sorted - assert(completed == expectedCompleted) + val completed = toSeq(result.take(workFinished)) + assert(completed.toSet.subsetOf(items.toSet)) } } + test(s"$impl: mapConcurrent tail laziness") { + val failAfter1 = 1 +:: undefined[AsyncStream[Int]] + val justOne = failAfter1.mapConcurrent(1)(Future.value).take(1) + assert(await(justOne.toSeq()) == Seq(1)) + } + test(s"$impl: sum") { forAll { xs: List[Int] => assert(xs.sum == await(fromSeq(xs).sum)) @@ -790,4 +795,4 @@ private object AsyncStreamTest { } def seqImpl: Seq[FromSeq] = Seq(Cons, EmbeddedCons, OfFuture, EmbeddedFuture) -} +} \ No newline at end of file From d9c5e4a3fbb383114bc5aad73754ed9fef6f4dd2 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 19 Nov 2018 20:34:36 +0000 Subject: [PATCH 068/796] util-core: Remove MinimumThroughput utils # Problem / Solution We introduced some experimental APIs into Util that rate-limit `Reader`s and `Writer`s. As they were never adopted, we can just remove them. JIRA Issues: CSL-7220 Differential Revision: https://phabricator.twitter.biz/D240944 --- CHANGELOG.rst | 7 + .../io/exp/BelowThroughputException.scala | 15 -- .../twitter/io/exp/MinimumThroughput.scala | 161 ----------- .../io/exp/MinimumThroughputTest.scala | 253 ------------------ 4 files changed, 7 insertions(+), 429 deletions(-) delete mode 100644 util-core/src/main/scala/com/twitter/io/exp/BelowThroughputException.scala delete mode 100644 util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala delete mode 100644 util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8de66e4cfe..5d786acc50 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,13 @@ Bug Fixes * util-core: Fixed a bug where tail would sometimes return Some empty AsyncStream instead of None. ``PHAB_ID=D241513`` +======= + +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: Experimental `c.t.io.exp.MinThroughput` utilities were removed. ``PHAB_ID=D240944`` + 18.11.0 ------- diff --git a/util-core/src/main/scala/com/twitter/io/exp/BelowThroughputException.scala b/util-core/src/main/scala/com/twitter/io/exp/BelowThroughputException.scala deleted file mode 100644 index 3667a54489..0000000000 --- a/util-core/src/main/scala/com/twitter/io/exp/BelowThroughputException.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.twitter.io.exp - -import com.twitter.util.{Duration, TimeoutException} - -/** - * Indicates a [[com.twitter.io.Reader reader]] or - * [[com.twitter.io.Writer writer]] dropped below - * the minimum required bytes per second threshold. - * - * @param elapsed total time spent reading or writing. - * - * @see [[MinimumThroughput]] - */ -case class BelowThroughputException(elapsed: Duration, currentBps: Double, expectedBps: Double) - extends TimeoutException(s"current bps $currentBps below minimum $expectedBps") diff --git a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala b/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala deleted file mode 100644 index 65c47fcc6a..0000000000 --- a/util-core/src/main/scala/com/twitter/io/exp/MinimumThroughput.scala +++ /dev/null @@ -1,161 +0,0 @@ -package com.twitter.io.exp - -import com.twitter.io.{Writer, Buf, Reader} -import com.twitter.util._ -import scala.util.control.NoStackTrace - -/** - * [[Reader Readers]] and [[Writer Writers]] that have minimum bytes - * per second throughput limits. - */ -object MinimumThroughput { - - private val BpsNoElapsed = -1d - - private val MinDeadline = Duration.fromSeconds(1) - - /** internal marker used to raise on read/write timeouts. */ - private case object MinThroughputTimeoutException extends Exception with NoStackTrace - - private abstract class Min(minBps: Double) { - assert(minBps >= 0, s"minBps must be non-negative: $minBps") - - // we rely on the `reader.read` and `writer.write` contract that only - // one read or write can ever be outstanding, so volatile provides - // us thread-safety. - @volatile protected[this] var bytes = 0L - @volatile protected[this] var elapsed = Duration.Zero - - /** Calculate and return the current bps */ - def bps: Double = { - val elapsedSecs = elapsed.inSeconds - if (elapsedSecs == 0) { - BpsNoElapsed - } else { - bytes.toDouble / elapsedSecs.toDouble - } - } - - /** - * Return a cap on how long we're willing to wait for the operation to - * complete processing `numBytes` while staying above the min bps requirements. - */ - def nextDeadline(numBytes: Int): Duration = - if (minBps == 0) - Duration.Top - else - Duration.fromSeconds(((bytes + numBytes) / minBps).toInt).max(MinDeadline) - - /** Verify if we are over the min bps requirements. */ - def throughputOk: Boolean = { - val newBps = bps - newBps >= minBps || newBps == BpsNoElapsed - } - - def newBelowThroughputException: BelowThroughputException = - BelowThroughputException(elapsed, bps, minBps) - - } - - private class MinReader(reader: Reader[Buf], minBps: Double, timer: Timer) - extends Min(minBps) - with Reader[Buf] { - def discard(): Unit = reader.discard() - - def read(): Future[Option[Buf]] = { - val deadline = nextDeadline(0) - - val start = Time.now - val read: Future[Option[Buf]] = - reader.read().raiseWithin(timer, deadline, MinThroughputTimeoutException) - - read.transform { res => - elapsed += Time.now - start - res match { - case Throw(MinThroughputTimeoutException) => - discard() - Future.exception(newBelowThroughputException) - case Throw(_) => - read // pass through other failures untouched - case Return(None) => - read // hit EOF, so we're fine - case Return(Some(buf)) => - // read some bytes, see if we're still good in terms of bps - bytes += buf.length - if (throughputOk) { - read - } else { - discard() - Future.exception(newBelowThroughputException) - } - } - } - } - } - - private class MinWriter(writer: Writer[Buf], minBps: Double, timer: Timer) - extends Min(minBps) - with Writer[Buf] { - def fail(cause: Throwable): Unit = writer.fail(cause) - - override def write(buf: Buf): Future[Unit] = { - val numBytes = buf.length - val deadline = nextDeadline(numBytes) - - val start = Time.now - val write: Future[Unit] = - writer.write(buf).raiseWithin(timer, deadline, MinThroughputTimeoutException) - - write.transform { res => - elapsed += Time.now - start - res match { - case Throw(MinThroughputTimeoutException) => - val ex = newBelowThroughputException - fail(ex) - Future.exception(ex) - case Throw(_) => - write // pass through other failures untouched - case Return(_) => - // wrote some bytes, see if we're still good in terms of bps - bytes += numBytes - if (throughputOk) { - write - } else { - val ex = newBelowThroughputException - fail(ex) - Future.exception(ex) - } - } - } - } - - def close(deadline: Time): Future[Unit] = writer.close(deadline) - - def onClose: Future[Unit] = writer.onClose - } - - /** - * Ensures that the throughput of [[Reader.read reads]] happen - * above a minimum specified bytes per second rate. - * - * Reads will return a Future of a [[BelowThroughputException]] when the - * throughput is not met. - * - * @note the minimum time allotted to any read is 1 second. - */ - def reader(reader: Reader[Buf], minBps: Double, timer: Timer): Reader[Buf] = - new MinReader(reader, minBps, timer) - - /** - * Ensures that the throughput of [[Writer.write writes]] happen - * above a minimum specified bytes per second rate. - * - * Writes will return a Future of a [[BelowThroughputException]] when the - * throughput is not met. - * - * @note the minimum time allotted to any write is 1 second. - */ - def writer(writer: Writer[Buf], minBps: Double, timer: Timer): Writer[Buf] = - new MinWriter(writer, minBps, timer) - -} diff --git a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala b/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala deleted file mode 100644 index 5815c94fa9..0000000000 --- a/util-core/src/test/scala/com/twitter/io/exp/MinimumThroughputTest.scala +++ /dev/null @@ -1,253 +0,0 @@ -package com.twitter.io.exp - -import com.twitter.conversions.time._ -import com.twitter.io.{Writer, Buf, BufReader, Reader} -import com.twitter.util._ -import java.io.ByteArrayOutputStream -import org.junit.runner.RunWith -import org.mockito.Mockito._ -import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -import org.scalatest.mockito.MockitoSugar - -@RunWith(classOf[JUnitRunner]) -class MinimumThroughputTest extends FunSuite with MockitoSugar { - test("Reader - negative minBps") { - intercept[AssertionError] { - MinimumThroughput.reader(mock[Reader[Buf]], -1, Timer.Nil) - } - } - - test("Reader - faster than min") { - val buf = Buf.UsAscii("soylent green is") // 16 bytes - val reader = MinimumThroughput.reader(Reader.fromBuf(buf, 8), 0d, Timer.Nil) - - // read from the beginning - Await.result(reader.read()) match { - case Some(b) => assert(b == buf.slice(0, 8)) - case _ => fail() - } - - Await.result(reader.read()) match { - case Some(b) => assert(b == buf.slice(8, 16)) - case _ => fail() - } - } - - test("Reader - below threshold after read") { - val buf = Buf.UsAscii("0") - - Time.withCurrentTimeFrozen { tc => - // it'd be great if we could just use a mock. - // the problem was that stubs are evaluated once, eagerly, at creation time. - val underlying = new Reader[Buf] { - private var reads = 0 - def discard(): Unit = () - def read(): Future[Option[Buf]] = { - reads += 1 - if (reads == 2) { - // now take 10 seconds to read 1 more byte, - // which would put us at 0.2 bps, and thus below the threshold. - tc.advance(10.seconds) - } - Future.value(Some(buf)) - } - } - - val reader = MinimumThroughput.reader( - underlying, - 1d, // min bytes per second - Timer.Nil - ) - - // do a read of 1 byte in 0 time — which is ok. - Await.result(reader.read()) match { - case Some(b) => assert(b == buf.slice(0, 1)) - case _ => fail() - } - - val ex = intercept[BelowThroughputException] { - // note in the mock above, the 2nd read takes 10 seconds - Await.result(reader.read()) - } - assert(ex.elapsed == 10.seconds) - assert(ex.expectedBps == 1d) - assert(ex.currentBps == 0.2d) - } - } - - test("Reader - times out while reading") { - val underlying = mock[Reader[Buf]] - when(underlying.read()) - .thenReturn(Future.never) - - val timer = new MockTimer() - val reader = MinimumThroughput.reader( - underlying, - 1d, // min bytes per second - timer - ) - - Time.withCurrentTimeFrozen { tc => - val f = reader.read() - tc.advance(10.seconds) - timer.tick() - - val ex = intercept[BelowThroughputException] { - Await.result(f) - } - assert(ex.elapsed == 10.seconds) - assert(ex.expectedBps == 1d) - assert(ex.currentBps == 0d) - } - } - - test("Reader - failures from underlying reader are untouched") { - val ex = new RuntimeException("└[∵┌]└[ ∵ ]┘[┐∵]┘") - val underlying = mock[Reader[Buf]] - when(underlying.read()) - .thenReturn(Future.exception(ex)) - - val reader = MinimumThroughput.reader( - underlying, - 1d, // min bytes per second - Timer.Nil - ) - - val thrown = intercept[RuntimeException] { - Await.result(reader.read()) - } - assert(thrown == ex) - } - - test("Reader - pass through EOFs from underlying") { - val reader = MinimumThroughput.reader( - Reader.empty, - 1d, // min bytes per second - Timer.Nil - ) - - Await.result(reader.read()) match { - case None => - case _ => fail() - } - } - - test("Reader - discard is passed through to underlying") { - val underlying = mock[Reader[Buf]] - val reader = MinimumThroughput.reader(underlying, 1, Timer.Nil) - - reader.discard() - verify(underlying).discard() - } - - test("Writer - faster than min") { - val buf = Buf.UsAscii("0") - - val writer = - MinimumThroughput.writer(Writer.fromOutputStream(new ByteArrayOutputStream()), 0d, Timer.Nil) - - val w1 = writer.write(buf) - Await.ready(w1) - assert(w1.isDone) - - val w2 = writer.write(buf) - Await.ready(w2) - assert(w2.isDone) - } - - test("Writer - below threshold after write") { - val buf = Buf.UsAscii("0") - - Time.withCurrentTimeFrozen { tc => - // it'd be great if we could just use a mock. - // the problem was that stubs are evaluated once, eagerly, at creation time. - val underlying = new Writer[Buf] { - private var writes = 0 - def fail(cause: Throwable): Unit = () - def write(buf: Buf): Future[Unit] = { - writes += 1 - if (writes == 2) { - // now take 10 seconds to write 1 more byte, - // which would put us at 0.2 bps, and thus below the threshold. - tc.advance(10.seconds) - } - Future.Done - } - - def close(deadline: Time): Future[Unit] = Future.Done - - def onClose: Future[Unit] = Future.never - } - - val writer = MinimumThroughput.writer( - underlying, - 1d, // min bytes per second - Timer.Nil - ) - - // do a write of 1 byte in 0 time — which is ok. - val w1 = writer.write(buf) - Await.ready(w1) - assert(w1.isDone) - - val ex = intercept[BelowThroughputException] { - // note in the mock above, the 2nd write takes 10 seconds - Await.result(writer.write(buf)) - } - assert(ex.elapsed == 10.seconds) - assert(ex.expectedBps == 1d) - assert(ex.currentBps == 0.2d) - } - } - - test("Writer - times out while writing") { - val buf = Buf.UsAscii("0") - val underlying = mock[Writer[Buf]] - when(underlying.write(buf)) - .thenReturn(Future.never) - - val timer = new MockTimer() - val writer = MinimumThroughput.writer(underlying, 1d, timer) - - Time.withCurrentTimeFrozen { tc => - val f = writer.write(buf) - tc.advance(10.seconds) - timer.tick() - - val ex = intercept[BelowThroughputException] { - Await.result(f) - } - assert(ex.elapsed == 10.seconds) - assert(ex.expectedBps == 1d) - assert(ex.currentBps == 0d) - } - } - - test("Writer - failures from underlying writer are untouched") { - val buf = Buf.UsAscii("0") - val ex = new RuntimeException("ᕕ( ᐛ )ᕗ") - val underlying = mock[Writer[Buf]] - when(underlying.write(buf)) - .thenReturn(Future.exception(ex)) - - val writer = MinimumThroughput.writer(underlying, 1d, Timer.Nil) - - val thrown = intercept[RuntimeException] { - Await.result(writer.write(buf)) - } - assert(thrown == ex) - } - - test("Writer - fail is passed through to underlying") { - val underlying = mock[Writer[Buf]] - - val writer = MinimumThroughput.writer(underlying, 1d, Timer.Nil) - - val ex = new RuntimeException() - writer.fail(ex) - - verify(underlying).fail(ex) - } - -} From f7d5eef112765cfcb67a9daf7cd685ac97f43a90 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 26 Nov 2018 16:05:04 +0000 Subject: [PATCH 069/796] util-core: Make WaitQueue and derived types abstract classes Problem We use `trait`s as the foundations for data strutures used in our `Promise` implementation and quite a few megamorphic methods. This can require the JVM to perform invokeinterface calls, which in contrast to invokevirtual calls, required searching through the list of implemented interfaces to find the interface of interest. This can result in meaningful overhead for very high throughput code dealing in classes that implement many interfaces. Solution Restruture the `WaitQueue` class and its subtypes as abstract classes. Differential Revision: https://phabricator.twitter.biz/D242570 --- .../com/twitter/util/FutureBenchmark.scala | 32 +++++++++++++++++++ .../main/scala/com/twitter/util/Promise.scala | 9 +++--- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala index 02967a1f9a..9316880f72 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala @@ -3,6 +3,7 @@ package com.twitter.util import com.twitter.concurrent.Offer import org.openjdk.jmh.annotations._ +// ./sbt 'project util-benchmark' 'jmh:run FutureBenchmark' class FutureBenchmark extends StdBenchAnnotations { import FutureBenchmark._ @@ -198,6 +199,37 @@ class FutureBenchmark extends StdBenchAnnotations { Await.result(p) } + def buildChain(root: Promise[Int], state: RunqState): Future[Int] = { + var next: Future[Int] = root + var sum = 0 + var i = 0 + + val fmap = { i: Int => + Future.value(i + 1) + } + val respond = { _: Try[Int] => + sum += 1 + } + + while (i < state.depth) { + next = i % 2 match { + case 0 => next.flatMap(fmap) + case 1 => next.respond(respond) + } + i += 1 + } + next + } + + @Benchmark + def runChain(state: RunqState): Int = { + val p = new Promise[Int]() + val f = buildChain(p, state) + // trigger the callbacks + p.setValue(5) + Await.result(f) + } + @Benchmark def monitored(): String = { val f = Future.monitored { diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 457178c278..faf6bba1fd 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -54,7 +54,7 @@ object Promise { /** * A persistent queue of continuations (i.e., `K`). */ - private[util] sealed trait WaitQueue[-A] { + private[util] sealed abstract class WaitQueue[-A] { def first: K[A] def rest: WaitQueue[A] @@ -124,9 +124,10 @@ object Promise { * it will make "Linked" and "Waiting" state cases ambiguous. This, however, * may change following the further performance improvements. */ - private[util] trait K[-A] extends (Try[A] => Unit) with WaitQueue[A] { + private[util] abstract class K[-A] extends WaitQueue[A] { final def first: K[A] = this final def rest: WaitQueue[A] = WaitQueue.empty + def apply(r: Try[A]): Unit } /** @@ -192,7 +193,7 @@ object Promise { * @param k the closure to invoke in the saved context, with the * provided result */ - private class Monitored[A](saved: Local.Context, k: Try[A] => Unit) extends K[A] { + private final class Monitored[A](saved: Local.Context, k: Try[A] => Unit) extends K[A] { def apply(result: Try[A]): Unit = { val current = Local.save() if (current ne saved) @@ -209,7 +210,7 @@ object Promise { protected[this] def k(r: Try[A]): Unit - def apply(result: Try[A]): Unit = { + final def apply(result: Try[A]): Unit = { val current = Local.save() if (current ne saved) Local.restore(saved) From 3b1434e2b59a4f8df89c408ed442abb738e3a6a6 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Mon, 26 Nov 2018 19:47:07 +0000 Subject: [PATCH 070/796] util-core: Provide a way to listen for stream termination Problem It can be useful to know when a stream is closed for implementing many common questions, like "how long has my stream taken to finish" or "did my stream finish successfully?" but there is not currently a good way to tell. Solution Provide Reader#onClose which will be satisfied after the stream is finished, discarded, or failed. JIRA Issues: CSL-6813 TBR=true Differential Revision: https://phabricator.twitter.biz/D236311 --- CHANGELOG.rst | 11 ++- .../src/main/java/com/twitter/io/Readers.java | 5 -- .../main/scala/com/twitter/io/BufReader.scala | 39 ++++++--- .../com/twitter/io/InputStreamReader.scala | 16 +++- .../com/twitter/io/OutputStreamWriter.scala | 23 +++-- .../src/main/scala/com/twitter/io/Pipe.scala | 36 +++++--- .../main/scala/com/twitter/io/Reader.scala | 29 +++++-- .../com/twitter/io/StreamTermination.scala | 30 +++++++ .../main/scala/com/twitter/io/Writer.scala | 14 +-- util-core/src/test/java/com/twitter/io/BUILD | 2 +- .../com/twitter/io/ReaderCompilationTest.java | 5 -- .../com/twitter/io/ReaderSemanticsTest.scala | 86 +++++++++++++++++++ .../scala/com/twitter/io/ReaderTest.scala | 4 + 13 files changed, 240 insertions(+), 60 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/io/StreamTermination.scala create mode 100644 util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5d786acc50..6fb207679a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,19 +7,26 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-core: Provide a way to listen for stream termination to `c.t.util.Reader`, `Reader#onClose` + which is satisfied when the stream is discarded or read until the end. ``PHAB_ID=D236311`` + Bug Fixes ~~~~~~~~~ * util-core: Fixed a bug where tail would sometimes return Some empty AsyncStream instead of None. ``PHAB_ID=D241513`` -======= - Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ * util-core: Experimental `c.t.io.exp.MinThroughput` utilities were removed. ``PHAB_ID=D240944`` +* util-core: Deleted `c.t.io.Reader.Null`, which was incompatible with `Reader#onClose` semantics. + `c.t.io.Reader#empty[Nothing]` is a drop-in replacement. ``PHAB_ID=D236311`` + 18.11.0 ------- diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index 43641b9af5..b8a669e3f9 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -17,11 +17,6 @@ public final class Readers { private Readers() { throw new IllegalStateException(); } - /** - * See {@code com.twitter.io.Reader.Null}. - */ - public static final Reader NULL = Reader$.MODULE$.Null(); - public static Reader newEmptyReader() { return Reader$.MODULE$.empty(); } diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index a652398ce4..357afca546 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -1,27 +1,40 @@ package com.twitter.io -import com.twitter.util.{Future, Return, Try, Throw} +import com.twitter.util.{Future, Return, Try, Throw, ConstFuture, Promise} /** * Construct a [[Reader]] from a [[Buf]]. */ private[io] final class BufReader(buf: Buf, chunkSize: Int) extends Reader[Buf] { private[this] var state: Try[Buf] = Return(buf) - - def read(): Future[Option[Buf]] = synchronized { - state match { - case Return(Buf.Empty) => Future.None - case Return(b) => - state = Return(b.slice(chunkSize, Int.MaxValue)) - Future.value(Some(b.slice(0, chunkSize))) - - case Throw(exc) => Future.exception(exc) + private[this] val closep = Promise[StreamTermination]() + + def read(): Future[Option[Buf]] = { + val result: Future[Option[Buf]] = synchronized { + state match { + case Return(Buf.Empty) => Future.None + case Return(b) => + state = Return(b.slice(chunkSize, Int.MaxValue)) + Future.value(Some(b.slice(0, chunkSize))) + + case t: Throw[_] => + new ConstFuture[Option[Buf]](t.cast[Option[Buf]]) + } + } + if (result eq Future.None) { + closep.updateIfEmpty(StreamTermination.FullyRead.Return) } + result } - def discard(): Unit = synchronized { - state = Throw(new ReaderDiscardedException) + def discard(): Unit = { + synchronized { + state = Throw(new ReaderDiscardedException) + } + closep.updateIfEmpty(StreamTermination.Discarded.Return) } + + def onClose: Future[StreamTermination] = closep } object BufReader { @@ -38,5 +51,5 @@ object BufReader { * `chunkSize`. */ def apply(buf: Buf, chunkSize: Int): Reader[Buf] = - if (buf.isEmpty) Reader.Null else new BufReader(buf, chunkSize) + if (buf.isEmpty) Reader.empty[Buf] else new BufReader(buf, chunkSize) } diff --git a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala index 500f2cc35c..28e1aa683a 100644 --- a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala +++ b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala @@ -1,7 +1,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncMutex -import com.twitter.util.{Closable, CloseAwaitably, Future, FuturePool, Time} +import com.twitter.util.{Closable, CloseAwaitably, Future, FuturePool, Time, Promise, Throw} import java.io.InputStream /** @@ -19,6 +19,7 @@ class InputStreamReader private[io] ( with CloseAwaitably { private[this] val mutex = new AsyncMutex() @volatile private[this] var discarded = false + private[this] val closep = Promise[StreamTermination]() /** * Constructs an [[InputStreamReader]] out of a given `inputStream`. The resulting [[Reader]] @@ -48,13 +49,18 @@ class InputStreamReader private[io] ( val c = inputStream.read(buffer, 0, chunkSize) if (c == -1) { pool { inputStream.close() } + closep.updateIfEmpty(StreamTermination.FullyRead.Return) None } else { Some(Buf.ByteArray.Owned(buffer, 0, c)) } } catch { case exc: InterruptedException => - discard() + // we use updateIfEmpty because this is potentially racy, if someone + // called close and then we were interrupted. + if (closep.updateIfEmpty(Throw(exc))) { + discard() + } throw exc } }.ensure { @@ -75,8 +81,12 @@ class InputStreamReader private[io] ( */ def close(deadline: Time): Future[Unit] = closeAwaitably { discarded = true - pool { inputStream.close() } + pool { inputStream.close() }.ensure { + closep.updateIfEmpty(StreamTermination.Discarded.Return) + } } + + def onClose: Future[StreamTermination] = closep } object InputStreamReader { diff --git a/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala b/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala index 90d5e95024..dc70ccdb65 100644 --- a/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala +++ b/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala @@ -1,6 +1,6 @@ package com.twitter.io -import com.twitter.util.{Future, FuturePool, Promise, Return, Throw, Time} +import com.twitter.util.{Future, FuturePool, Promise, Return, Throw, Time, ConstFuture} import java.io.OutputStream import java.util.concurrent.atomic.AtomicReference import scala.annotation.tailrec @@ -11,7 +11,7 @@ import scala.annotation.tailrec private[io] class OutputStreamWriter(out: OutputStream, bufsize: Int) extends Writer[Buf] { import com.twitter.io.OutputStreamWriter._ - private[this] val done = new Promise[Unit] + private[this] val done = new Promise[StreamTermination] private[this] val writeOp = new AtomicReference[Buf => Future[Unit]](doWrite) // Byte array reused on each write to avoid multiple allocations. @@ -36,8 +36,12 @@ private[io] class OutputStreamWriter(out: OutputStream, bufsize: Int) extends Wr buf => FuturePool.interruptibleUnboundedPool { drain(buf) } def write(buf: Buf): Future[Unit] = - if (done.isDefined) done - else + if (done.isDefined) { + done.transform { + case Return(StreamTermination.FullyRead) => Future.exception(CloseExc) + case t => (new ConstFuture(t)).unit + } + } else ( done or writeOp.getAndSet(_ => Future.exception(WriteExc))(buf) ) transform { @@ -54,12 +58,15 @@ private[io] class OutputStreamWriter(out: OutputStream, bufsize: Int) extends Wr def fail(cause: Throwable): Unit = done.updateIfEmpty(Throw(cause)) - def close(deadline: Time): Future[Unit] = - if (done.updateIfEmpty(Throw(CloseExc))) FuturePool.unboundedPool { + def close(deadline: Time): Future[Unit] = { + FuturePool.unboundedPool { out.close() - } else Future.Done + done.updateIfEmpty(StreamTermination.FullyRead.Return) + } + done.unit + } - def onClose: Future[Unit] = done + def onClose: Future[StreamTermination] = done } private object OutputStreamWriter { diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 3c438f2f7d..6d43e5892f 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -1,6 +1,6 @@ package com.twitter.io -import com.twitter.util.{Future, Promise, Return, Time} +import com.twitter.util.{Future, Promise, Return, Time, Throw} /** * A synchronous in-memory pipe that connects [[Reader]] and [[Writer]] in the sense @@ -91,7 +91,7 @@ final class Pipe[A] extends Reader[A] with Writer[A] { private[this] var state: State[A] = State.Idle // satisfied when a `read` observes the EOF (after calling close()) - private[this] val closep: Promise[Unit] = new Promise[Unit]() + private[this] val closep: Promise[StreamTermination] = new Promise[StreamTermination]() def read(): Future[Option[A]] = { val (waiter, result) = synchronized { @@ -101,7 +101,7 @@ final class Pipe[A] extends Reader[A] with Writer[A] { case State.Closing => state = State.Closed - (closep, Future.None) + (null, Future.None) case State.Closed => (null, Future.None) @@ -122,10 +122,13 @@ final class Pipe[A] extends Reader[A] with Writer[A] { } if (waiter != null) waiter.setDone() + if (result eq Future.None) closep.updateIfEmpty(StreamTermination.FullyRead.Return) result } - def discard(): Unit = fail(new ReaderDiscardedException()) + def discard(): Unit = { + fail(new ReaderDiscardedException(), discard = true) + } def write(buf: A): Future[Unit] = { val (waiter, value, result) = synchronized { @@ -160,7 +163,9 @@ final class Pipe[A] extends Reader[A] with Writer[A] { result } - def fail(cause: Throwable): Unit = { + def fail(cause: Throwable): Unit = fail(cause, discard = false) + + private[this] def fail(cause: Throwable, discard: Boolean): Unit = { val (closer, reader, writer) = synchronized { state match { case State.Closed | State.Failed(_) => @@ -181,7 +186,10 @@ final class Pipe[A] extends Reader[A] with Writer[A] { if (reader != null) reader.setException(cause) else if (writer != null) writer.setException(cause) - if (closer != null) closer.setException(cause) + if (closer != null) { + if (discard) closer.updateIfEmpty(StreamTermination.Discarded.Return) + else closer.updateIfEmpty(Throw(cause)) + } } def close(deadline: Time): Future[Unit] = { @@ -204,17 +212,19 @@ final class Pipe[A] extends Reader[A] with Writer[A] { } } - if (reader != null) + if (reader != null) { reader.update(Return.None) - else if (writer != null) - writer.setException(new IllegalStateException("close() while write is pending")) - - if (closer != null) closer.setDone() + closep.update(StreamTermination.FullyRead.Return) + } else if (writer != null) { + val exn = new IllegalStateException("close() while write is pending") + writer.setException(exn) + if (closer != null) closer.updateIfEmpty(Throw(exn)) + } - onClose + onClose.unit } - def onClose: Future[Unit] = closep + def onClose: Future[StreamTermination] = closep override def toString: String = synchronized(s"Pipe(state=$state)") } diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 55d33d6fc9..c8991ac778 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -93,6 +93,15 @@ trait Reader[+A] { * @note Although unnecessary, it's always safe to discard a fully-consumed stream. */ def discard(): Unit + + /** + * A [[Future]] that resolves once this reader is closed. + * + * If the result is a failed future, this indicates that it was not closed either by reading + * until the end of the stream nor by discarding. This is useful for any extra resource cleanup + * that you must do once the stream is no longer being used. + */ + def onClose: Future[StreamTermination] } /** @@ -102,13 +111,19 @@ class ReaderDiscardedException extends Exception("Reader's consumer has discarde object Reader { - val Null: Reader[Nothing] = new Reader[Nothing] { - def read(): Future[Option[Nothing]] = Future.None - def discard(): Unit = () + def empty[A]: Reader[A] = new Reader[A] { + private[this] val closep = Promise[StreamTermination]() + def read(): Future[Option[Nothing]] = { + closep.updateIfEmpty(StreamTermination.FullyRead.Return) + Future.None + } + def discard(): Unit = { + closep.updateIfEmpty(StreamTermination.Discarded.Return) + () + } + def onClose: Future[StreamTermination] = closep } - def empty[A]: Reader[A] = Null.asInstanceOf[Reader[A]] - // see Reader.chunked private final class ChunkedFramer(chunkSize: Int) extends (Buf => Seq[Buf]) { require(chunkSize > 0, s"chunkSize should be > 0 but was $chunkSize") @@ -162,6 +177,8 @@ object Reader { frames = Seq.empty r.discard() } + + def onClose: Future[StreamTermination] = r.onClose } /** @@ -287,6 +304,8 @@ object Reader { f.raise(new ReaderDiscardedException()) target.discard() } + + def onClose: Future[StreamTermination] = target.onClose } } diff --git a/util-core/src/main/scala/com/twitter/io/StreamTermination.scala b/util-core/src/main/scala/com/twitter/io/StreamTermination.scala new file mode 100644 index 0000000000..fddc326245 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/StreamTermination.scala @@ -0,0 +1,30 @@ +package com.twitter.io + +/** + * Trait that indicates what kind of stream termination a stream saw. + */ +sealed abstract class StreamTermination { + def isFullyRead: Boolean +} + +object StreamTermination { + + /** + * Indicates a stream terminated when it was read all the way until the end. + */ + case object FullyRead extends StreamTermination { + val Return: com.twitter.util.Return[FullyRead.type] = com.twitter.util.Return(this) + + def isFullyRead: Boolean = true + } + + /** + * Indicates a stream terminated when it was discarded by the reader before it + * could be read all the way until the end. + */ + case object Discarded extends StreamTermination { + val Return: com.twitter.util.Return[Discarded.type] = com.twitter.util.Return(this) + + def isFullyRead: Boolean = false + } +} diff --git a/util-core/src/main/scala/com/twitter/io/Writer.scala b/util-core/src/main/scala/com/twitter/io/Writer.scala index e64224273d..14c704d139 100644 --- a/util-core/src/main/scala/com/twitter/io/Writer.scala +++ b/util-core/src/main/scala/com/twitter/io/Writer.scala @@ -72,10 +72,14 @@ trait Writer[-A] extends Closable { self => def fail(cause: Throwable): Unit /** - * Expose a `Future` that is satisfied when the stream is closed. It may contain an error - * if the Writer is failed. + * A [[Future]] that resolves once this writer is closed and flushed. + * + * It may contain an error if the writer is failed. + * + * This is useful for any extra resource cleanup that you must do once the stream is no longer + * being used. */ - def onClose: Future[Unit] + def onClose: Future[StreamTermination] /** * Given f, a function from B into A, creates an Writer[B] whose `fail` and `close` functions @@ -88,7 +92,7 @@ trait Writer[-A] extends Closable { self => def write(element: B): Future[Unit] = self.write(f(element)) def fail(cause: Throwable): Unit = self.fail(cause) def close(deadline: Time): Future[Unit] = self.close(deadline) - def onClose: Future[Unit] = self.onClose + def onClose: Future[StreamTermination] = self.onClose } } @@ -113,7 +117,7 @@ object Writer { def write(element: A): Future[Unit] = Future.exception(new IllegalStateException("NullWriter")) def fail(cause: Throwable): Unit = () def close(deadline: Time): Future[Unit] = Future.Done - def onClose: Future[Unit] = Future.never + def onClose: Future[StreamTermination] = Future.never } val BufferSize: Int = 4096 diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD index bbb0aceff9..1e8340fa9d 100644 --- a/util-core/src/test/java/com/twitter/io/BUILD +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -5,6 +5,6 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", "util/util-core/src/main/java/com/twitter/io", - "util/util-core/src/main/scala/com/twitter/io", + "util/util-core/src/main/scala", ], ) diff --git a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java b/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java index 2a86332ffb..4d1ec8db43 100644 --- a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java +++ b/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java @@ -11,11 +11,6 @@ public class ReaderCompilationTest { - @Test - public void testNull() { - assertNotNull(Readers.NULL); - } - @Test public void testNewBufReader() { Readers.newBufReader(Bufs.EMPTY, 1024); diff --git a/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala new file mode 100644 index 0000000000..40b8d230a2 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala @@ -0,0 +1,86 @@ +package com.twitter.io + +import com.twitter.concurrent.AsyncStream +import com.twitter.conversions.storage._ +import com.twitter.conversions.time._ +import com.twitter.util.{Future, Return, Await, Throw} +import java.io.ByteArrayInputStream +import org.scalatest.FunSuite +import scala.util.Random + +abstract class ReaderSemanticsTest[A] extends FunSuite { + def createReader(): Reader[A] + + def loopAndCheck(reader: Reader[A], onClose: Future[StreamTermination]): Future[Unit] = { + reader.read().flatMap { + case t @ Some(_) => + assert(!onClose.isDefined) + assert(!reader.onClose.isDefined) + loopAndCheck(reader, onClose) + case t @ None => + assert(onClose.poll == Some(StreamTermination.FullyRead.Return)) + assert(onClose.poll == Some(StreamTermination.FullyRead.Return)) + Future.Done + } + } + + def await(f: Future[_]): Unit = { + Await.result(f, 5.seconds) + () + } + + test("When you read a reader until the end, onClose will be satisfied") { + val reader = createReader() + val f = loopAndCheck(reader, reader.onClose) + await(f) + assert(f.poll == Some(Return.Unit)) + } + + test("When you discard a reader, onClose will be satisfied") { + val reader = createReader() + assert(!reader.onClose.isDefined) + reader.discard() + assert(reader.onClose.poll == Some(StreamTermination.Discarded.Return)) + } +} + +class BufReaderSemanticsTest extends ReaderSemanticsTest[Buf] { + def createReader(): Reader[Buf] = { + val buf = Buf.Utf8(Random.nextString(2.kilobytes.inBytes.toInt)) + BufReader(buf, 1.kilobyte.inBytes.toInt) + } +} + +class ConcatReaderSemanticsTest extends ReaderSemanticsTest[Buf] { + def createReader(): Reader[Buf] = { + val buf = Buf.Utf8(Random.nextString(2.kilobytes.inBytes.toInt)) + Reader.concat( + AsyncStream( + BufReader(buf, 1.kilobyte.inBytes.toInt), + BufReader(buf, 1.kilobyte.inBytes.toInt) + ) + ) + } + + test("When a piece is failed, the entire thing is failed") { + val left = new Pipe[Buf]() + val exn = new Exception("boom!") + left.fail(exn) + val buf = Buf.Utf8(Random.nextString(2.kilobytes.inBytes.toInt)) + val right = BufReader(buf, 1.kilobyte.inBytes.toInt) + val reader = Reader.concat(AsyncStream(left, right)) + val actual = intercept[Exception] { + await(reader.read()) + } + assert(actual == exn) + assert(reader.onClose.poll == Some(Throw(exn))) + } +} + +class InputStreamReaderSemanticsTest extends ReaderSemanticsTest[Buf] { + def createReader(): Reader[Buf] = { + val bytes = Random.nextString(2.kilobytes.inBytes.toInt).getBytes + val inputStream = new ByteArrayInputStream(bytes) + InputStreamReader(inputStream, 1.kilobyte.inBytes.toInt) + } +} diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index d1d9f071a5..45e243a68c 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -113,6 +113,7 @@ class ReaderTest val head = new Reader[Buf] { def read(): Future[Option[Buf]] = p def discard(): Unit = p.setException(new ReaderDiscardedException) + def onClose: Future[StreamTermination] = Future.never } val reader = Reader.concat(head +:: undefined) reader.discard() @@ -124,6 +125,7 @@ class ReaderTest val head = new Reader[Buf] { def read(): Future[Option[Buf]] = p def discard(): Unit = p.setException(new ReaderDiscardedException) + def onClose: Future[StreamTermination] = Future.never } val reader = Reader.concat(head +:: undefined) assertReadWhileReading(reader) @@ -134,6 +136,7 @@ class ReaderTest val head = new Reader[Buf] { def read(): Future[Option[Buf]] = p def discard(): Unit = p.setException(new ReaderDiscardedException) + def onClose: Future[StreamTermination] = Future.never } val reader = Reader.concat(head +:: undefined) assertFailed(reader, p) @@ -144,6 +147,7 @@ class ReaderTest def read(): Future[Option[Buf]] = Future.exception(new Exception) def discard(): Unit = () + def onClose: Future[StreamTermination] = Future.never } val p = new Promise[Unit] From 3909d1ad17debc65b44a232ef92ca21f924d25e7 Mon Sep 17 00:00:00 2001 From: kenchan0130 Date: Mon, 26 Nov 2018 20:56:36 +0000 Subject: [PATCH 071/796] util-core: Change `Future.???` method from val to def Problem Currently, Future.??? always returns the same instance and therefore we cannot get valid stacktrace (information on where it is defined). Solution Change Future.??? from a val to def. Result The exception has a relevant stacktrace. Signed-off-by: Bryce Anderson JIRA Issues: CSL-7301 Differential Revision: https://phabricator.twitter.biz/D245326 --- .../src/main/scala/com/twitter/util/Future.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index b92eb75dfd..be40e356ba 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -40,12 +40,6 @@ object Future { /** A successfully satisfied constant `Future` of `false` */ val False: Future[Boolean] = new ConstFuture(Return.False) - /** - * A failed `Future` analogous to `Predef.???`. - */ - val ??? : Future[Nothing] = - Future.exception(new NotImplementedError("an implementation is missing")) - private val SomeReturnUnit = Some(Return.Unit) private val NotApplied: Future[Nothing] = new NoFuture private val AlwaysNotApplied: Any => Future[Nothing] = scala.Function.const(NotApplied) @@ -90,6 +84,12 @@ object Future { private[this] val RaiseException = new Exception with NoStackTrace @inline private final def raiseException = RaiseException + /** + * A failed `Future` analogous to [[Predef.???]]. + */ + def ??? : Future[Nothing] = + Future.exception(new NotImplementedError("an implementation is missing")) + /** * Creates a satisfied `Future` from a [[Try]]. * From 822d68161c6f1ef9e85e893e9cb2c1a249f7156b Mon Sep 17 00:00:00 2001 From: Automated go/ktf Flagger Date: Mon, 26 Nov 2018 21:26:59 +0000 Subject: [PATCH 072/796] The test target "util/util-core/src/test/scala/com/twitter/io:io" is failing too often in master in CI. Summary: Problem: The test target "util/util-core/src/test/scala/com/twitter/io:io" is failing too often in master in CI. Logs available here: Known to fail ticket: [CSL-7306](https://jira.twitter.biz/browse/CSL-7306) Solution: It has been marked "known_to_fail" in the BUILD file Result: This will cause it to be skipped in future builds, and its failures ignored in submit queue. This implies that code changes that further break this test will not be prevented from being submitted to the master branch, until the problem is fixed and the test is no longer marked known-to-fail. TBR=true Reviewers: O5331 source:/util/util-core/! Subscribers: #rb_csl Differential Revision: https://phabricator.twitter.biz/D245499 --- util-core/src/test/scala/com/twitter/io/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 88d823c101..d569a5a66d 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -9,6 +9,7 @@ junit_tests( # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` # for this target, we could try enabling this again. compiler_option_sets = {}, + known_to_fail = "CSL-7306", dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", From 8034e5577c953c2b99836865cd9cf6da652aea71 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 26 Nov 2018 22:29:32 +0000 Subject: [PATCH 073/796] source: Replace c.t.util.U64 with c.t.conversions.u64 # Problem / Solution Replacing an old and deprecated Util's API with simple converters, based on what's already available in JDK 8. JIRA Issues: CSL-6030 TBR=true Differential Revision: https://phabricator.twitter.biz/D244723 --- CHANGELOG.rst | 2 + .../scala/com/twitter/conversions/u64.scala | 21 +++ .../src/main/scala/com/twitter/util/U64.scala | 122 ------------- .../com/twitter/conversions/U64Test.scala | 22 +++ .../test/scala/com/twitter/util/U64Test.scala | 168 ------------------ 5 files changed, 45 insertions(+), 290 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/conversions/u64.scala delete mode 100644 util-core/src/main/scala/com/twitter/util/U64.scala create mode 100644 util-core/src/test/scala/com/twitter/conversions/U64Test.scala delete mode 100644 util-core/src/test/scala/com/twitter/util/U64Test.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6fb207679a..a58854b50a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,6 +27,8 @@ Breaking API Changes * util-core: Deleted `c.t.io.Reader.Null`, which was incompatible with `Reader#onClose` semantics. `c.t.io.Reader#empty[Nothing]` is a drop-in replacement. ``PHAB_ID=D236311`` +* util-core: Removed `c.t.util.U64` bits. Use `c.t.converters.u64._` instead. ``PHAB_ID=D244723`` + 18.11.0 ------- diff --git a/util-core/src/main/scala/com/twitter/conversions/u64.scala b/util-core/src/main/scala/com/twitter/conversions/u64.scala new file mode 100644 index 0000000000..2e04428296 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/u64.scala @@ -0,0 +1,21 @@ +package com.twitter.conversions + +object u64 { + + /** + * Parses this HEX string as an unsigned 64-bit long value. Be careful, this can throw + * [[NumberFormatException]]. + * + * @see [[java.lang.Long.parseUnsignedLong()]] + */ + implicit class StringOps(val self: String) extends AnyVal { + def toU64Long: Long = java.lang.Long.parseUnsignedLong(self, 16) + } + + /** + * Converts this unsigned 64-bit long value into a 16-character HEX string. + */ + implicit class LongOps(val self: Long) extends AnyVal { + def toU64HexString: String = "%016x".format(self) + } +} diff --git a/util-core/src/main/scala/com/twitter/util/U64.scala b/util-core/src/main/scala/com/twitter/util/U64.scala deleted file mode 100644 index 19c91f8c44..0000000000 --- a/util-core/src/main/scala/com/twitter/util/U64.scala +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2010 Twitter, Inc. -// -// Unsigned math on (64-bit) longs. - -package com.twitter.util - -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, DataOutputStream} -import scala.language.implicitConversions - -@deprecated("Use Java 8 unsigned Long APIs instead", "2018-02-13") -class RichU64Long(l64: Long) { - import U64._ - - def u64_<(y: Long): Boolean = u64_lt(l64, y) - - /** exclusive: (a, b) */ - def u64_within(a: Long, b: Long): Boolean = u64_lt(a, l64) && u64_lt(l64, b) - - /** inclusive: [a, b] */ - def u64_contained(a: Long, b: Long): Boolean = !u64_lt(l64, a) && !u64_lt(b, l64) - - def u64ToBigInt: BigInt = u64ToBigint(l64) - - def u64_/(y: Long): Long = - (u64ToBigInt / u64ToBigint(y)).longValue - - def u64_%(y: Long): Long = - (u64ToBigInt % u64ToBigint(y)).longValue - - // Other arithmetic operations don't need unsigned equivalents - // with 2s complement. - - def toU64ByteArray: Array[Byte] = { - val bytes = new ByteArrayOutputStream(8) - new DataOutputStream(bytes).writeLong(l64) - bytes.toByteArray - } - - def toU64HexString = (new RichU64ByteArray(toU64ByteArray)).toU64HexString -} - -@deprecated("Use Java 8 unsigned Long APIs instead", "2018-02-13") -class RichU64ByteArray(bytes: Array[Byte]) { - private def deSign(b: Byte): Int = { - if (b < 0) { - b + 256 - } else { - b - } - } - - // Note: this is padded, so we have lexical ordering. - def toU64HexString = bytes.map(deSign).map("%02x".format(_)).reduceLeft(_ + _) - def toU64Long = new DataInputStream(new ByteArrayInputStream(bytes)).readLong -} - -/** - * RichU64String parses string as a 64-bit unsigned hexadecimal long, - * outputting a signed Long or a byte array. Valid input is 1-16 hexadecimal - * digits (0-9, a-z, A-Z). - * - * @throws NumberFormatException if string is not a non-negative hexadecimal - * string - */ -@deprecated("Use Java 8 unsigned Long APIs instead", "2018-02-13") -class RichU64String(string: String) { - private[this] def validateHexDigit(c: Char): Unit = { - if (!(('0' <= c && c <= '9') || - ('a' <= c && c <= 'f') || - ('A' <= c && c <= 'F'))) { - throw new NumberFormatException("For input string: \"" + string + "\"") - } - } - - if (string.length > 16) { - throw new NumberFormatException("Number longer than 16 hex characters") - } else if (string.isEmpty) { - throw new NumberFormatException("Empty string") - } - string.foreach(validateHexDigit) - - def toU64ByteArray: Array[Byte] = { - val padded = "0" * (16 - string.length()) + string - (0 until 16 by 2) - .map(i => { - val parsed = Integer.parseInt(padded.slice(i, i + 2), 16) - assert(parsed >= 0) - parsed.toByte - }) - .toArray - } - - def toU64Long: Long = (new RichU64ByteArray(toU64ByteArray)).toU64Long -} - -@deprecated("Use Java 8 unsigned Long APIs instead", "2018-02-13") -object U64 { - private val bigInt0x8000000000000000L = (0x7FFFFFFFFFFFFFFFL: BigInt) + 1 - - val U64MAX = 0xFFFFFFFFFFFFFFFFL - val U64MIN = 0L - - def u64ToBigint(x: Long): BigInt = - if ((x & 0x8000000000000000L) != 0L) - ((x & 0x7FFFFFFFFFFFFFFFL): BigInt) + bigInt0x8000000000000000L - else - x: BigInt - - // compares x < y - def u64_lt(x: Long, y: Long): Boolean = - if (x < 0 == y < 0) - x < y // signed comparison, straightforward! - else - x > y // x is less if it doesn't have its high bit set (<0) - - implicit def longToRichU64Long(x: Long): RichU64Long = new RichU64Long(x) - - implicit def byteArrayToRichU64ByteArray(bytes: Array[Byte]): RichU64ByteArray = - new RichU64ByteArray(bytes) - - implicit def stringToRichU64String(string: String): RichU64String = new RichU64String(string) -} diff --git a/util-core/src/test/scala/com/twitter/conversions/U64Test.scala b/util-core/src/test/scala/com/twitter/conversions/U64Test.scala new file mode 100644 index 0000000000..aa98441cf4 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/conversions/U64Test.scala @@ -0,0 +1,22 @@ +package com.twitter.conversions + +import org.scalatest.FunSuite +import org.scalatest.prop.GeneratorDrivenPropertyChecks + +class U64Test extends FunSuite with GeneratorDrivenPropertyChecks { + + import u64._ + + test("toU64HextString") { + forAll { l: Long => + assert(l.toU64HexString == "%016x".format(l)) + } + } + + test("toU64Long") { + forAll { l: Long => + val s = "%016x".format(l) + assert(s.toU64Long == java.lang.Long.parseUnsignedLong(s, 16)) + } + } +} diff --git a/util-core/src/test/scala/com/twitter/util/U64Test.scala b/util-core/src/test/scala/com/twitter/util/U64Test.scala deleted file mode 100644 index 6273473424..0000000000 --- a/util-core/src/test/scala/com/twitter/util/U64Test.scala +++ /dev/null @@ -1,168 +0,0 @@ -package com.twitter.util - -import scala.util.Random -import org.junit.runner.RunWith -import org.scalacheck.Gen -import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -import org.scalatest.prop.GeneratorDrivenPropertyChecks - -@RunWith(classOf[JUnitRunner]) -class U64Test extends WordSpec with GeneratorDrivenPropertyChecks { - import U64._ - - "comparable" in { - { - val a = 0x0000000000000001L - assert(a == 1) - val b = 0x0000000000000002L - assert(b == 2) - - assert(a.u64_<(b) == true) - assert(b.u64_<(a) == false) - } - - { - val a = 0xFFFFFFFFFFFFFFFFL - assert(a == -1) - val b = 0xFFFFFFFFFFFFFFFEL - assert(b == -2) - - assert(a.u64_<(b) == false) - assert(b.u64_<(a) == true) - } - - { - val a = 0xFFFFFFFFFFFFFFFFL - assert(a == -1) - val b = 0x0000000000000001L - assert(b == 1) - - assert(a.u64_<(b) == false) - assert(b.u64_<(a) == true) - } - } - - "comparable in range" in { - assert(0L.u64_within(0, 1) == false) - assert(0L.u64_contained(0, 1) == true) - - // (inverted range) - assert(0L.u64_within(-1, 1) == false) - assert(1L.u64_within(-1, 1) == false) - assert(2L.u64_within(-1, 1) == false) - - assert(0xFFFFFFFFFFFFFFFEL.u64_within(0xFFFFFFFFFFFFFFFDL, 0xFFFFFFFFFFFFFFFFL) == true) - assert(0xFFFFFFFFFFFFFFFDL.u64_within(0xFFFFFFFFFFFFFFFDL, 0xFFFFFFFFFFFFFFFFL) == false) - assert(0xFFFFFFFFFFFFFFFFL.u64_within(0xFFFFFFFFFFFFFFFDL, 0xFFFFFFFFFFFFFFFFL) == false) - - assert(0xFFFFFFFFFFFFFFFEL.u64_contained(0xFFFFFFFFFFFFFFFDL, 0xFFFFFFFFFFFFFFFFL) == true) - assert(0xFFFFFFFFFFFFFFFDL.u64_contained(0xFFFFFFFFFFFFFFFDL, 0xFFFFFFFFFFFFFFFFL) == true) - assert(0xFFFFFFFFFFFFFFFFL.u64_contained(0xFFFFFFFFFFFFFFFDL, 0xFFFFFFFFFFFFFFFFL) == true) - - // Bit flip area! - assert(0x7FFFFFFFFFFFFFFFL.u64_within(0x7FFFFFFFFFFFFFFFL, 0x8000000000000000L) == false) - assert(0x8000000000000000L.u64_within(0x7FFFFFFFFFFFFFFFL, 0x8000000000000000L) == false) - - assert(0x7FFFFFFFFFFFFFFFL.u64_contained(0x7FFFFFFFFFFFFFFFL, 0x8000000000000000L) == true) - assert(0x8000000000000000L.u64_contained(0x7FFFFFFFFFFFFFFFL, 0x8000000000000000L) == true) - - assert(0x7FFFFFFFFFFFFFFAL.u64_within(0x7FFFFFFFFFFFFFFAL, 0x800000000000000AL) == false) - assert(0x7FFFFFFFFFFFFFFBL.u64_within(0x7FFFFFFFFFFFFFFAL, 0x800000000000000AL) == true) - assert(0x7FFFFFFFFFFFFFFFL.u64_within(0x7FFFFFFFFFFFFFFAL, 0x800000000000000AL) == true) - assert(0x8000000000000000L.u64_within(0x7FFFFFFFFFFFFFFAL, 0x800000000000000AL) == true) - assert(0x8000000000000001L.u64_within(0x7FFFFFFFFFFFFFFAL, 0x800000000000000AL) == true) - assert(0x8000000000000009L.u64_within(0x7FFFFFFFFFFFFFFAL, 0x800000000000000AL) == true) - assert(0x800000000000000AL.u64_within(0x7FFFFFFFFFFFFFFAL, 0x800000000000000AL) == false) - } - - "divisible" in { - assert(10L.u64_/(5L) == 2L) - assert(0xFFFFFFFFFFFFFFFFL / 0x0FFFFFFFFFFFFFFFL == 0L) - assert(0xFFFFFFFFFFFFFFFFL.u64_/(0x0FFFFFFFFFFFFFFFL) == 16L) - - assert(0x7FFFFFFFFFFFFFFFL.u64_/(2) == 0x3FFFFFFFFFFFFFFFL) - - assert(0x8000000000000000L / 2 == 0xc000000000000000L) - assert(0x8000000000000000L.u64_/(2) == 0x4000000000000000L) - - assert(0x8000000000000000L.u64_/(0x8000000000000000L) == 1) - assert(0x8000000000000000L.u64_/(0x8000000000000001L) == 0) - - assert(0xFF00000000000000L.u64_/(0x0F00000000000000L) == 0x11L) - assert(0x8F00000000000000L.u64_/(0x100) == 0x008F000000000000L) - assert(0x8000000000000000L.u64_/(3) == 0x2AAAAAAAAAAAAAAAL) - } - - "ids" should { - "survive conversions" in { - val rng = new Random - - (0 until 10000).foreach { _ => - val id = rng.nextLong - assert(id == (id.toU64ByteArray.toU64Long)) - assert(id == (id.toU64ByteArray.toU64HexString.toU64ByteArray.toU64Long)) - } - } - - "be serializable" in { - assert(0L.toU64HexString == "0000000000000000") - assert(0x0102030405060700L.toU64HexString == "0102030405060700") - assert(0xFFF1F2F3F4F5F6F7L.toU64HexString == "fff1f2f3f4f5f6f7") - } - - "convert from short hex string" in { - assert(new RichU64String("7b").toU64Long == 123L) - } - - "don't silently truncate" in { - intercept[NumberFormatException] { - new RichU64String("318528893302738945") - } - } - - "not parse with +" in { - intercept[NumberFormatException] { "+0".toU64Long } - intercept[NumberFormatException] { "0+".toU64Long } - intercept[NumberFormatException] { "00+0".toU64Long } - intercept[NumberFormatException] { "0+00".toU64Long } - intercept[NumberFormatException] { "+ffffffffffffffff".toU64Long } - } - - "not parse non-latin unicode digits" in { - // \u09e6 = BENGALI DIGIT ZERO (accepted by parseLong) - intercept[NumberFormatException] { "\u09e6".toU64Long } - } - - "parse mixed case" in { - assert(new RichU64String("aBcDeF").toU64Long == 0xabcdef) - } - - "not parse if negative" in { - val actual = intercept[NumberFormatException] { "-1".toU64Long } - assert(actual.getMessage == "For input string: \"-1\"") - - intercept[NumberFormatException] { "".toU64Long } - intercept[NumberFormatException] { "-f".toU64Long } - intercept[NumberFormatException] { "-af".toU64Long } - intercept[NumberFormatException] { "10-aff".toU64Long } - intercept[NumberFormatException] { "1-0aff".toU64Long } - } - - "not parse empty string" in { - // this is what Long.parseLong("") does - intercept[NumberFormatException] { "".toU64Long } - } - - "Java8 migrations" in { - forAll { l: Long => - assert(new RichU64Long(l).toU64HexString == "%016x".format(l)) - } - - forAll { l: Long => - val s = "%016x".format(l) - assert(new RichU64String(s).toU64Long == java.lang.Long.parseUnsignedLong(s, 16)) - } - } - } -} From 2eee66e6550423d0498bbcc67b5eb645957ba7b1 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Mon, 26 Nov 2018 22:52:35 +0000 Subject: [PATCH 074/796] util-core: Fixed flaky WriterTest Problem We changed OutputStreamWriter to only close after it had really closed the output stream, but didn't update the tests. This made the WriterTest flaky. Solution Update the test to match the behavior. JIRA Issues: CSL-7306 Differential Revision: https://phabricator.twitter.biz/D245581 --- util-core/src/test/scala/com/twitter/io/BUILD | 1 - util-core/src/test/scala/com/twitter/io/WriterTest.scala | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index d569a5a66d..88d823c101 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -9,7 +9,6 @@ junit_tests( # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` # for this target, we could try enabling this again. compiler_option_sets = {}, - known_to_fail = "CSL-7306", dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-core/src/test/scala/com/twitter/io/WriterTest.scala b/util-core/src/test/scala/com/twitter/io/WriterTest.scala index 29a2043e9a..6f90b696c1 100644 --- a/util-core/src/test/scala/com/twitter/io/WriterTest.scala +++ b/util-core/src/test/scala/com/twitter/io/WriterTest.scala @@ -11,7 +11,7 @@ class WriterTest extends FunSuite with Matchers { test("fromOutputStream - close") { val w = Writer.fromOutputStream(new ByteArrayOutputStream) - w.close() + await(w.close()) intercept[IllegalStateException] { await(w.write(Buf.Empty)) } } From 9f01785ae335d84652473d9174063d47dd912c8e Mon Sep 17 00:00:00 2001 From: teresy Date: Tue, 27 Nov 2018 01:20:23 +0000 Subject: [PATCH 075/796] util-zk: replace filter.size with more concise count Problem / Solution We can be more concise and efficient in ZkAsyncSemaphore by using `.count` instead of `.filter(..).size`. Signed-off-by: Bryce Anderson JIRA Issues: CSL-7307 Differential Revision: https://phabricator.twitter.biz/D245681 --- .../scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala index 82d79b3162..3445a85bb0 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala @@ -267,7 +267,7 @@ class ZkAsyncSemaphore( val (numPermitsInMax, numBelieversOfMax) = permitsToBelievers.maxBy { case (_, believers) => believers } - val cardinalityOfMax = permitsToBelievers.values.filter({ _ == numBelieversOfMax }).size + val cardinalityOfMax = permitsToBelievers.values.count(_ == numBelieversOfMax) if (cardinalityOfMax == 1) { // Consensus From d62627f83a1f878c753120a4f10d936a299f938e Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 28 Nov 2018 22:25:33 +0000 Subject: [PATCH 076/796] util-stats: Clean up some concurrency of CumulativeGauge Problem / Solution We use a combination of a StampedLock and an AtomicInteger to manage the reference count of a CumultiveGauge implementation which could be done with a single AtomicInteger, so just use the AtomicInteger. Differential Revision: https://phabricator.twitter.biz/D246495 --- .../finagle/stats/CumulativeGauge.scala | 65 ++++++++----------- 1 file changed, 27 insertions(+), 38 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index 8185e63cb8..f4dec5bada 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -5,8 +5,8 @@ import com.twitter.util.lint._ import java.lang.{Boolean => JBoolean} import java.util.concurrent.{ConcurrentHashMap, Executor, ForkJoinPool} import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.locks.StampedLock import java.util.function.{Function => JFunction} +import scala.annotation.tailrec import scala.collection.JavaConverters._ /** @@ -127,7 +127,7 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => } /** - * The StatsReceiver implements these. They provide the cumulated + * The StatsReceiver implements these. They provide the cumulative * gauges. */ protected[this] def registerGauge(verbosity: Verbosity, name: Seq[String], f: => Float): Unit @@ -149,46 +149,36 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => def apply(key: Seq[String]): CumulativeGauge = new CumulativeGauge { self.registerGauge(verbosity, key, getValue) - // we grab the read lock on increment and decrement, which can race each - // other. we only grab the write lock when we tear down the cumulative - // gauge, so that we can ensure no more increments happen after death. - private[this] val lock = new StampedLock() + // The number of registers starts at `0` because every new gauge will cause a + // registration, even the one that created the `CumulativeGauge`. Because 0 is + // a valid value to increment from, we use `-1` to signal the closed state where + // no more registration activity is possible. private[this] val registers = new AtomicInteger(0) - // protected by `lock` - private[this] var dead = false - - def register(): Boolean = { - val stamp = lock.readLock() - try { - if (dead) return false - registers.incrementAndGet() - } finally { - lock.unlockRead(stamp) - } - true + @tailrec def register(): Boolean = { + val c = registers.get + if (c == -1) false + else if (!registers.compareAndSet(c, c + 1)) register() + else true } - def deregister(): Unit = { - val readStamp = lock.readLock() - if (registers.decrementAndGet() == 0) { - // we could skip a cas by using tryConvertToWriteLock, but it makes - // the code much uglier. - lock.unlockRead(readStamp) - val writeStamp = lock.writeLock() - - try { - if (!dead && registers.get == 0) { - dead = true - gauges.remove(key) - self.deregisterGauge(key) - } - } finally { - lock.unlockWrite(writeStamp) + @tailrec def deregister(): Unit = registers.get match { + case -1 => () // Already closed + case 1 => + // Attempt to transition to the closed state + if (!registers.compareAndSet(1, -1)) { + deregister() // lost a race so try again + } else { + // Do the cleanup. + gauges.remove(key) + self.deregisterGauge(key) + } + + case c => + // Just a normal decrement + if (!registers.compareAndSet(c, c - 1)) { + deregister() // lost a race so try again } - } else { - lock.unlockRead(readStamp) - } } } } @@ -215,5 +205,4 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => if (cumulativeGauge == null) 0 else cumulativeGauge.size } - } From 757e9c691bf89aca73ec11e933a04124abd84edc Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 29 Nov 2018 17:47:53 +0000 Subject: [PATCH 077/796] util-core: Simplify Reader.concat via Reader.onClose # Problem / Solution `Reader.onClose` introduces a way to simplify how `Reader.concat` is implemented. The new approach doesn't require a bonus `Reader` wrapper. JIRA Issues: CSL-6849 Differential Revision: https://phabricator.twitter.biz/D245651 --- .../main/scala/com/twitter/io/Reader.scala | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index c8991ac778..ec8ec3d10a 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -287,26 +287,25 @@ object Reader { */ def concat[A](readers: AsyncStream[Reader[A]]): Reader[A] = { val target = new Pipe[A] - val f = copyMany(readers, target).respond { + val copied = copyMany(readers, target).respond { case Throw(exc) => target.fail(exc) case _ => target.close() } - new Reader[A] { - def read(): Future[Option[A]] = target.read() - def discard(): Unit = { + + target.onClose.respond { + case Return(StreamTermination.Discarded) => // We have to do this so that when the the target is discarded we can // interrupt the read operation. Consider the following: // // r.read(..) { case Some(b) => target.write(b) } // - // The computation r.read(..) will be interupted because we set an + // The computation r.read(..) will be interrupted because we set an // interrupt handler in Reader.copy to discard `r`. - f.raise(new ReaderDiscardedException()) - target.discard() - } - - def onClose: Future[StreamTermination] = target.onClose + copied.raise(new ReaderDiscardedException()) + case _ => () } + + target } /** From 1c52cd428463155c26abfaf6f3e0187ed399a16e Mon Sep 17 00:00:00 2001 From: Daniel Schobel Date: Tue, 4 Dec 2018 01:20:37 +0000 Subject: [PATCH 078/796] finagle/util/scrooge/twitter-server: update owners Problem / Solution Update OWNERS files. Differential Revision: https://phabricator.twitter.biz/D248571 --- OWNERS | 1 - util-app/OWNERS | 1 - util-benchmark/OWNERS | 1 - util-codec/OWNERS | 1 - util-core/OWNERS | 1 - util-intellij/OWNERS | 2 -- util-logging/OWNERS | 1 - util-security/OWNERS | 1 - util-stats/OWNERS | 1 - util-zk/OWNERS | 1 - 10 files changed, 11 deletions(-) diff --git a/OWNERS b/OWNERS index c313f6c1d5..a044bdad0f 100644 --- a/OWNERS +++ b/OWNERS @@ -1,5 +1,4 @@ banderson -dschobel koliver mnakamura roanta diff --git a/util-app/OWNERS b/util-app/OWNERS index 463d61c17c..1c2f0835fc 100644 --- a/util-app/OWNERS +++ b/util-app/OWNERS @@ -1,5 +1,4 @@ banderson -dschobel koliver mnakamura roanta diff --git a/util-benchmark/OWNERS b/util-benchmark/OWNERS index c313f6c1d5..a044bdad0f 100644 --- a/util-benchmark/OWNERS +++ b/util-benchmark/OWNERS @@ -1,5 +1,4 @@ banderson -dschobel koliver mnakamura roanta diff --git a/util-codec/OWNERS b/util-codec/OWNERS index c313f6c1d5..a044bdad0f 100644 --- a/util-codec/OWNERS +++ b/util-codec/OWNERS @@ -1,5 +1,4 @@ banderson -dschobel koliver mnakamura roanta diff --git a/util-core/OWNERS b/util-core/OWNERS index 97007381d5..b749f10796 100644 --- a/util-core/OWNERS +++ b/util-core/OWNERS @@ -1,5 +1,4 @@ banderson -dschobel koliver mnakamura roanta diff --git a/util-intellij/OWNERS b/util-intellij/OWNERS index 80c8af4f73..a044bdad0f 100644 --- a/util-intellij/OWNERS +++ b/util-intellij/OWNERS @@ -1,6 +1,4 @@ banderson -dschobel koliver mnakamura roanta -slance diff --git a/util-logging/OWNERS b/util-logging/OWNERS index bb5d54aad9..983c0ab37e 100644 --- a/util-logging/OWNERS +++ b/util-logging/OWNERS @@ -1,6 +1,5 @@ # hzhang is owner for ScribeHandler only banderson -dschobel koliver mnakamura roanta diff --git a/util-security/OWNERS b/util-security/OWNERS index cbb42ab042..fa6d114e20 100644 --- a/util-security/OWNERS +++ b/util-security/OWNERS @@ -1,6 +1,5 @@ banderson bdimmick -dschobel koliver mnakamura roanta diff --git a/util-stats/OWNERS b/util-stats/OWNERS index 37c64cba94..ad4779b7a0 100644 --- a/util-stats/OWNERS +++ b/util-stats/OWNERS @@ -1,5 +1,4 @@ banderson -dschobel koliver mnakamura roanta diff --git a/util-zk/OWNERS b/util-zk/OWNERS index a3a086672a..d893509652 100644 --- a/util-zk/OWNERS +++ b/util-zk/OWNERS @@ -1,5 +1,4 @@ banderson -dschobel koliver mnakamura roanta From 238506a2e28d9a1e60bea902a13e8780280432dc Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 4 Dec 2018 16:54:31 +0000 Subject: [PATCH 079/796] util-core: Fix Base64Long test case Problem / Solution The unit test "Base64LongTest.fromBase64 throws an IllegalArgumentException exception for characters out of range" failed on some inputs. This was a mistake in how the test was written as it should have checked against the inverted alphabet. JIRA Issues: CSL-7340 Differential Revision: https://phabricator.twitter.biz/D248564 --- .../test/scala/com/twitter/util/Base64LongTest.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala index b231b3e454..2f78779773 100644 --- a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala +++ b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala @@ -27,7 +27,7 @@ object Base64LongTest { go(Nil, 64, 0) } - private implicit val arbAlphabet = + private implicit val arbAlphabet: Arbitrary[Alphabet] = Arbitrary[Alphabet]( Gen.oneOf( Gen.const(StandardBase64Alphabet), @@ -65,7 +65,7 @@ class Base64LongTest extends FunSuite with GeneratorDrivenPropertyChecks { assert(toBase64(0) == "A") val b = new StringBuilder - forAll { (a: Alphabet) => + forAll { a: Alphabet => b.setLength(0) toBase64(b, 0, a) assert(b.result == a(0).toString) @@ -77,7 +77,7 @@ class Base64LongTest extends FunSuite with GeneratorDrivenPropertyChecks { } test("toBase64 uses the expected number of digits") { - BoundaryValues.foreach { (n: Long) => + BoundaryValues.foreach { n: Long => assert(toBase64(n).length == expectedLength(n)) } forAll((n: Long) => assert(toBase64(n).length == expectedLength(n))) @@ -112,9 +112,10 @@ class Base64LongTest extends FunSuite with GeneratorDrivenPropertyChecks { test("fromBase64 throws an IllegalArgumentException exception for characters out of range") { forAll { (s: String, a: Alphabet) => - if (s.exists(!a.isDefinedAt(_))) { + val inverted = Base64Long.invertAlphabet(a) + if (s.exists(!inverted.isDefinedAt(_))) { assertThrows[IllegalArgumentException]( - fromBase64(s, 0, s.length, Base64Long.invertAlphabet(a)) + fromBase64(s, 0, s.length, inverted) ) } } From dc1fecdcf75e3b2fb19897c7c4b6d7fdc9c4158d Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 4 Dec 2018 22:15:36 +0000 Subject: [PATCH 080/796] util-core: Fix InputStreamReaderTest Problem / Solution The test "InputStreamReaderSemanticsTest. When you discard a reader, onClose will be satisfied" was flaky as `discard` does not always satisfy `onClose` synchronously. JIRA Issues: CSL-7345 Differential Revision: https://phabricator.twitter.biz/D248877 --- .../main/scala/com/twitter/io/InputStreamReader.scala | 4 ++-- .../scala/com/twitter/io/ReaderSemanticsTest.scala | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala index 28e1aa683a..acaa4ddd8f 100644 --- a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala +++ b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala @@ -1,7 +1,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncMutex -import com.twitter.util.{Closable, CloseAwaitably, Future, FuturePool, Time, Promise, Throw} +import com.twitter.util._ import java.io.InputStream /** @@ -72,7 +72,7 @@ class InputStreamReader private[io] ( /** * Discard this reader: its output is no longer required. * - * This closes the underlying `InputStream`. + * This asynchronously closes the underlying `InputStream`. */ def discard(): Unit = close() diff --git a/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala index 40b8d230a2..b52f8069ce 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala @@ -13,20 +13,18 @@ abstract class ReaderSemanticsTest[A] extends FunSuite { def loopAndCheck(reader: Reader[A], onClose: Future[StreamTermination]): Future[Unit] = { reader.read().flatMap { - case t @ Some(_) => + case Some(_) => assert(!onClose.isDefined) assert(!reader.onClose.isDefined) loopAndCheck(reader, onClose) - case t @ None => - assert(onClose.poll == Some(StreamTermination.FullyRead.Return)) + case None => assert(onClose.poll == Some(StreamTermination.FullyRead.Return)) Future.Done } } - def await(f: Future[_]): Unit = { + protected def await[T](f: Future[T]): T = { Await.result(f, 5.seconds) - () } test("When you read a reader until the end, onClose will be satisfied") { @@ -40,7 +38,8 @@ abstract class ReaderSemanticsTest[A] extends FunSuite { val reader = createReader() assert(!reader.onClose.isDefined) reader.discard() - assert(reader.onClose.poll == Some(StreamTermination.Discarded.Return)) + val onClose = reader.onClose + assert(await(onClose) == StreamTermination.Discarded) } } From ddabd458a60189bddfb0a57fb80d468df6dc6844 Mon Sep 17 00:00:00 2001 From: Owen Parry Date: Wed, 5 Dec 2018 22:08:05 +0000 Subject: [PATCH 081/796] Update scalafmt config for util. Problem: The scalafmt config doesn't match the Twitter Style Guide. This causes angst and makes onboarding new folks onto scalafmt a difficult sell. Solution: Update the scalafmt config, and everyone who has opted into automatic application of scalafmt. Result: The code automatically matches the style guide, and people are happier to adopt scalafmt. --- .../scala/com/twitter/app/ClassPath.scala | 12 +- .../src/main/scala/com/twitter/app/Flag.scala | 3 +- .../main/scala/com/twitter/app/Flags.scala | 150 +++++++++--------- .../scala/com/twitter/app/GlobalFlag.scala | 9 +- .../com/twitter/app/JavaGlobalFlag.scala | 4 +- .../scala/com/twitter/app/LoadService.scala | 5 +- .../com/twitter/app/ColonizeTestApp.scala | 4 +- .../com/twitter/cache/KeyEncodingCache.scala | 6 +- .../com/twitter/collection/RecordSchema.scala | 3 +- .../scala/com/twitter/util/ImmutableLRU.scala | 3 +- .../com/twitter/concurrent/Scheduler.scala | 10 +- .../com/twitter/concurrent/AsyncMeter.scala | 3 +- .../com/twitter/concurrent/AsyncStream.scala | 30 ++-- .../com/twitter/concurrent/SpoolSource.scala | 3 +- .../src/main/scala/com/twitter/io/Buf.scala | 9 +- .../com/twitter/io/InputStreamReader.scala | 12 +- .../com/twitter/io/exp/ActivitySource.scala | 11 +- .../com/twitter/util/BatchExecutor.scala | 4 +- .../main/scala/com/twitter/util/Batcher.scala | 7 +- .../main/scala/com/twitter/util/Future.scala | 7 +- .../scala/com/twitter/util/FuturePool.scala | 4 +- .../main/scala/com/twitter/util/Local.scala | 76 ++++----- .../main/scala/com/twitter/util/Promise.scala | 15 +- .../twitter/util/SlowProbeProxyTimer.scala | 3 +- .../main/scala/com/twitter/util/Time.scala | 3 +- .../main/scala/com/twitter/util/Timer.scala | 4 +- .../src/main/scala/com/twitter/util/Var.scala | 4 +- .../com/twitter/util/WindowedAdder.scala | 3 +- .../concurrent/AsyncSemaphoreTest.scala | 3 +- .../test/scala/com/twitter/io/BufTest.scala | 7 +- .../scala/com/twitter/util/TimerTest.scala | 5 +- .../twitter/hashing/KetamaDistributor.scala | 4 +- .../scala/com/twitter/jvm/CpuProfile.scala | 3 +- .../main/scala/com/twitter/jvm/Hotspot.scala | 9 +- .../src/main/scala/com/twitter/jvm/Jvm.scala | 21 +-- .../scala/com/twitter/util/lint/Rule.scala | 8 +- .../com/twitter/logging/FileHandler.scala | 4 +- .../scala/com/twitter/logging/Formatter.scala | 4 +- .../scala/com/twitter/logging/Handler.scala | 12 +- .../com/twitter/logging/LazyLogRecord.scala | 6 +- .../com/twitter/logging/LoggerFactory.scala | 4 +- .../com/twitter/logging/ScribeHandler.scala | 4 +- .../com/twitter/logging/ScribeHandlers.scala | 5 +- .../com/twitter/logging/SyslogHandler.scala | 4 +- .../twitter/logging/ThrottledHandler.scala | 7 +- .../com/twitter/util/reflect/Proxy.scala | 8 +- .../util/security/PemFileTestUtils.scala | 24 ++- .../finagle/stats/ExceptionStatsHandler.scala | 8 +- .../stats/NameTranslatingStatsReceiver.scala | 4 +- .../VerbosityAdjustingStatsReceiver.scala | 9 +- .../util/testing/ArgumentCapture.scala | 99 ++++++------ .../util/tunable/JsonTunableMapper.scala | 3 +- .../tunable/ServiceLoadedTunableMap.scala | 17 +- .../com/twitter/zk/NativeConnector.scala | 22 ++- .../scala/com/twitter/zk/RetryPolicy.scala | 3 +- .../src/main/scala/com/twitter/zk/ZNode.scala | 3 +- .../main/scala/com/twitter/zk/ZkClient.scala | 12 +- .../zk/coordination/ZkAsyncSemaphore.scala | 3 +- .../scala/com/twitter/zk/ZkClientTest.scala | 11 +- 59 files changed, 326 insertions(+), 417 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/ClassPath.scala b/util-app/src/main/scala/com/twitter/app/ClassPath.scala index 308778158c..cd48546d9a 100644 --- a/util-app/src/main/scala/com/twitter/app/ClassPath.scala +++ b/util-app/src/main/scala/com/twitter/app/ClassPath.scala @@ -85,11 +85,7 @@ private[app] sealed abstract class ClassPath[CpInfo <: ClassPath.Info] { } // package protected for testing - private[app] def browseUri( - uri: URI, - loader: ClassLoader, - buf: mutable.Buffer[CpInfo] - ): Unit = + private[app] def browseUri(uri: URI, loader: ClassLoader, buf: mutable.Buffer[CpInfo]): Unit = browseUri0(uri, loader, buf, mutable.Set[URI]()) private[this] def browseUri0( @@ -127,11 +123,7 @@ private[app] sealed abstract class ClassPath[CpInfo <: ClassPath.Info] { processFile(prefix, f, buf) } - protected def processFile( - prefix: String, - file: File, - buf: mutable.Buffer[CpInfo] - ): Unit + protected def processFile(prefix: String, file: File, buf: mutable.Buffer[CpInfo]): Unit private def browseJar( file: File, diff --git a/util-app/src/main/scala/com/twitter/app/Flag.scala b/util-app/src/main/scala/com/twitter/app/Flag.scala index cde12f016d..d497cff166 100644 --- a/util-app/src/main/scala/com/twitter/app/Flag.scala +++ b/util-app/src/main/scala/com/twitter/app/Flag.scala @@ -56,8 +56,7 @@ class Flag[T: Flaggable] private[app] ( val name: String, val help: String, defaultOrUsage: Either[() => T, String], - failFastUntilParsed: Boolean -) { + failFastUntilParsed: Boolean) { import com.twitter.app.Flag._ import java.util.logging._ diff --git a/util-app/src/main/scala/com/twitter/app/Flags.scala b/util-app/src/main/scala/com/twitter/app/Flags.scala index 87022b8fe3..c1db75327d 100644 --- a/util-app/src/main/scala/com/twitter/app/Flags.scala +++ b/util-app/src/main/scala/com/twitter/app/Flags.scala @@ -120,83 +120,81 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) * @return A [[com.twitter.app.Flags.FlagParseResult]] representing the * result of parsing `args`. */ - def parseArgs( - args: Array[String], - allowUndefinedFlags: Boolean = false - ): FlagParseResult = synchronized { - reset() - val remaining = new ArrayBuffer[String] - var i = 0 - while (i < args.length) { - val a = args(i) - i += 1 - if (a == "--") { - remaining ++= args.slice(i, args.length) - i = args.length - } else if (a startsWith "-") { - a drop 1 split ("=", 2) match { - // There seems to be a bug Scala's pattern matching - // optimizer that leaves `v' dangling in the last case if - // we make this a wildcard (Array(k, _@_*)) - case Array(k) if !hasFlag(k) => - if (allowUndefinedFlags) - remaining += a - else - return Error( - "Error parsing flag \"%s\": %s".format(k, FlagUndefinedMessage) - ) - - // Flag isn't defined - case Array(k, _) if !hasFlag(k) => - if (allowUndefinedFlags) - remaining += a - else - return Error( - "Error parsing flag \"%s\": %s".format(k, FlagUndefinedMessage) - ) - - // Optional argument without a value - case Array(k) if flag(k).noArgumentOk => - flag(k).parse() - - // Mandatory argument without a value and with no more arguments. - case Array(k) if i == args.length => - return Error( - "Error parsing flag \"%s\": %s".format(k, FlagValueRequiredMessage) - ) - - // Mandatory argument with another argument - case Array(k) => - i += 1 - try flag(k).parse(args(i - 1)) - catch { - case NonFatal(e) => + def parseArgs(args: Array[String], allowUndefinedFlags: Boolean = false): FlagParseResult = + synchronized { + reset() + val remaining = new ArrayBuffer[String] + var i = 0 + while (i < args.length) { + val a = args(i) + i += 1 + if (a == "--") { + remaining ++= args.slice(i, args.length) + i = args.length + } else if (a startsWith "-") { + a drop 1 split ("=", 2) match { + // There seems to be a bug Scala's pattern matching + // optimizer that leaves `v' dangling in the last case if + // we make this a wildcard (Array(k, _@_*)) + case Array(k) if !hasFlag(k) => + if (allowUndefinedFlags) + remaining += a + else return Error( - "Error parsing flag \"%s\": %s".format(k, e.getMessage) + "Error parsing flag \"%s\": %s".format(k, FlagUndefinedMessage) ) - } - // Mandatory k=v - case Array(k, v) => - try flag(k).parse(v) - catch { - case e: Throwable => + // Flag isn't defined + case Array(k, _) if !hasFlag(k) => + if (allowUndefinedFlags) + remaining += a + else return Error( - "Error parsing flag \"%s\": %s".format(k, e.getMessage) + "Error parsing flag \"%s\": %s".format(k, FlagUndefinedMessage) ) - } + + // Optional argument without a value + case Array(k) if flag(k).noArgumentOk => + flag(k).parse() + + // Mandatory argument without a value and with no more arguments. + case Array(k) if i == args.length => + return Error( + "Error parsing flag \"%s\": %s".format(k, FlagValueRequiredMessage) + ) + + // Mandatory argument with another argument + case Array(k) => + i += 1 + try flag(k).parse(args(i - 1)) + catch { + case NonFatal(e) => + return Error( + "Error parsing flag \"%s\": %s".format(k, e.getMessage) + ) + } + + // Mandatory k=v + case Array(k, v) => + try flag(k).parse(v) + catch { + case e: Throwable => + return Error( + "Error parsing flag \"%s\": %s".format(k, e.getMessage) + ) + } + } + } else { + remaining += a } - } else { - remaining += a } - } - finishParsing() + finishParsing() - if (helpFlag()) - Help(usage) - else - Ok(remaining) - } + if (helpFlag()) + Help(usage) + else + Ok(remaining) + } /** * Parse an array of flag strings. @@ -212,14 +210,12 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) * @throws FlagParseException if an error occurs during flag-parsing. */ @deprecated("Prefer result-value based `Flags.parseArgs` method", "6.17.1") - def parse( - args: Array[String], - undefOk: Boolean = false - ): Seq[String] = parseArgs(args, undefOk) match { - case Ok(remainder) => remainder - case Help(usage) => throw FlagUsageError(usage) - case Error(reason) => throw FlagParseException(reason) - } + def parse(args: Array[String], undefOk: Boolean = false): Seq[String] = + parseArgs(args, undefOk) match { + case Ok(remainder) => remainder + case Help(usage) => throw FlagUsageError(usage) + case Error(reason) => throw FlagParseException(reason) + } /** * Parse an array of flag strings or exit the application (with exit code 1) diff --git a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala index 8fb6605d01..382aba2ebc 100644 --- a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala +++ b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala @@ -44,9 +44,12 @@ import scala.util.control.NonFatal * If you'd like to declare a new [[GlobalFlag]] in Java, see [[JavaGlobalFlag]]. */ @GlobalFlagVisible -abstract class GlobalFlag[T] private[app] (defaultOrUsage: Either[() => T, String], help: String)( - implicit _f: Flaggable[T] -) extends Flag[T](null, help, defaultOrUsage, false) { +abstract class GlobalFlag[T] private[app] ( + defaultOrUsage: Either[() => T, String], + help: String +)( + implicit _f: Flaggable[T]) + extends Flag[T](null, help, defaultOrUsage, false) { override protected[this] def parsingDone: Boolean = true diff --git a/util-app/src/main/scala/com/twitter/app/JavaGlobalFlag.scala b/util-app/src/main/scala/com/twitter/app/JavaGlobalFlag.scala index b4df2c38fb..97670c2d42 100644 --- a/util-app/src/main/scala/com/twitter/app/JavaGlobalFlag.scala +++ b/util-app/src/main/scala/com/twitter/app/JavaGlobalFlag.scala @@ -37,8 +37,8 @@ package com.twitter.app abstract class JavaGlobalFlag[T] private ( defaultOrUsage: Either[() => T, String], help: String, - flaggable: Flaggable[T] -) extends GlobalFlag[T](defaultOrUsage, help)(flaggable) { + flaggable: Flaggable[T]) + extends GlobalFlag[T](defaultOrUsage, help)(flaggable) { /** * A flag with a default value. diff --git a/util-app/src/main/scala/com/twitter/app/LoadService.scala b/util-app/src/main/scala/com/twitter/app/LoadService.scala index 44f35dbf89..958395eb1b 100644 --- a/util-app/src/main/scala/com/twitter/app/LoadService.scala +++ b/util-app/src/main/scala/com/twitter/app/LoadService.scala @@ -28,10 +28,7 @@ object LoadService { * @param implementations cannot be empty. * @see [[App.loadServiceBindings]] */ - class Binding[T]( - val iface: Class[T], - val implementations: Seq[T] - ) { + class Binding[T](val iface: Class[T], val implementations: Seq[T]) { /** * Convenience constructor for binding a single implementation. diff --git a/util-app/src/test/scala/com/twitter/app/ColonizeTestApp.scala b/util-app/src/test/scala/com/twitter/app/ColonizeTestApp.scala index efb2c270bc..0efaa06162 100644 --- a/util-app/src/test/scala/com/twitter/app/ColonizeTestApp.scala +++ b/util-app/src/test/scala/com/twitter/app/ColonizeTestApp.scala @@ -7,8 +7,8 @@ object SolarSystemPlanets { val orderFromSun: Int, val name: String, val mass: Kilogram, - val radius: Meter - ) extends Ordered[Planet] { + val radius: Meter) + extends Ordered[Planet] { def compare(that: Planet): Int = this.orderFromSun - that.orderFromSun diff --git a/util-cache/src/main/scala/com/twitter/cache/KeyEncodingCache.scala b/util-cache/src/main/scala/com/twitter/cache/KeyEncodingCache.scala index 05a2c693e9..e25ee05c14 100644 --- a/util-cache/src/main/scala/com/twitter/cache/KeyEncodingCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/KeyEncodingCache.scala @@ -20,10 +20,8 @@ import com.twitter.util.Future * { case (target: Int, _, _, _) => target } * )) */ -private[cache] class KeyEncodingCache[K, V, U]( - encode: K => V, - underlying: FutureCache[V, U] -) extends FutureCache[K, U] { +private[cache] class KeyEncodingCache[K, V, U](encode: K => V, underlying: FutureCache[V, U]) + extends FutureCache[K, U] { override def get(key: K): Option[Future[U]] = underlying.get(encode(key)) def set(key: K, value: Future[U]): Unit = underlying.set(encode(key), value) diff --git a/util-collection/src/main/scala/com/twitter/collection/RecordSchema.scala b/util-collection/src/main/scala/com/twitter/collection/RecordSchema.scala index 6f958b2d8d..3e5cc51eb8 100644 --- a/util-collection/src/main/scala/com/twitter/collection/RecordSchema.scala +++ b/util-collection/src/main/scala/com/twitter/collection/RecordSchema.scala @@ -34,8 +34,7 @@ final class RecordSchema { * externally. */ final class Record private[RecordSchema] ( - fields: IdentityHashMap[Field[_], Entry] = new IdentityHashMap[Field[_], Entry] - ) { + fields: IdentityHashMap[Field[_], Entry] = new IdentityHashMap[Field[_], Entry]) { private[this] def getOrInitializeEntry(field: Field[_]): Entry = { var entry = fields.get(field) diff --git a/util-collection/src/main/scala/com/twitter/util/ImmutableLRU.scala b/util-collection/src/main/scala/com/twitter/util/ImmutableLRU.scala index 17c04442b7..17581d53d4 100644 --- a/util-collection/src/main/scala/com/twitter/util/ImmutableLRU.scala +++ b/util-collection/src/main/scala/com/twitter/util/ImmutableLRU.scala @@ -27,8 +27,7 @@ class ImmutableLRU[K, V] private ( maxSize: Int, idx: Long, map: Map[K, (Long, V)], - ord: SortedMap[Long, K] -) { + ord: SortedMap[Long, K]) { // Scala's SortedMap requires a key ordering; ImmutableLRU doesn't // care about pulling a minimum value out of the SortedMap, so the diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala index 1f21c3366a..2eecdb736f 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala @@ -335,10 +335,8 @@ trait ExecutorScheduler { self: Scheduler => * A scheduler that dispatches directly to an underlying Java * cached threadpool executor. */ -class ThreadPoolScheduler( - val name: String, - val executorFactory: ThreadFactory => ExecutorService -) extends Scheduler +class ThreadPoolScheduler(val name: String, val executorFactory: ThreadFactory => ExecutorService) + extends Scheduler with ExecutorScheduler { def this(name: String) = this(name, Executors.newCachedThreadPool(_)) } @@ -354,8 +352,8 @@ class ThreadPoolScheduler( */ class BridgedThreadPoolScheduler( val name: String, - val executorFactory: ThreadFactory => ExecutorService -) extends Scheduler + val executorFactory: ThreadFactory => ExecutorService) + extends Scheduler with ExecutorScheduler { private[this] val local = new LocalScheduler diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala index 807352c736..1d1d45db67 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala @@ -168,7 +168,8 @@ class AsyncMeter private ( private[concurrent] val burstSize: Int, burstDuration: Duration, q: BlockingQueue[(Promise[Unit], Int)] -)(implicit timer: Timer) { +)( + implicit timer: Timer) { require(burstSize > 0, s"burst size of $burstSize, which is <= 0 doesn't make sense") require( diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 564221df3e..fda1396e4d 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -501,7 +501,8 @@ sealed abstract class AsyncStream[+A] { def fillBuffer( sizeRemaining: Int - )(s: => AsyncStream[A]): Future[(Seq[A], () => AsyncStream[A])] = + )(s: => AsyncStream[A] + ): Future[(Seq[A], () => AsyncStream[A])] = if (sizeRemaining < 1) Future.value((buffer, () => s)) else s match { @@ -787,19 +788,20 @@ object AsyncStream { case Throw(e) => headp.setException(e) tailp.setException(e) - case Return(v) => v match { - case Empty => - headp.become(Future.None) - tailp.setValue(Oneshot.empty) - case FromFuture(fa) => - headp.become(fa.map(Some(_))) - tailp.setValue(Oneshot.empty) - case Cons(fa, more2) => - headp.become(fa.map(Some(_))) - tailp.setValue(more2) - case Embed(newFas) => - readEmbed(newFas, headp, tailp) - } + case Return(v) => + v match { + case Empty => + headp.become(Future.None) + tailp.setValue(Oneshot.empty) + case FromFuture(fa) => + headp.become(fa.map(Some(_))) + tailp.setValue(Oneshot.empty) + case Cons(fa, more2) => + headp.become(fa.map(Some(_))) + tailp.setValue(more2) + case Embed(newFas) => + readEmbed(newFas, headp, tailp) + } } // Recover the AsyncStream interface. We don't return `as` because we want diff --git a/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala b/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala index 2edce47a4f..de04927a08 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala @@ -108,7 +108,8 @@ class SpoolSource[A](interruptHandler: PartialFunction[Throwable, Unit]) { @tailrec private[this] def updatingTailCall( newPromise: Promise[Spool[A]] - )(f: Promise[Spool[A]] => Unit): Unit = { + )(f: Promise[Spool[A]] => Unit + ): Unit = { val currentPromise = promiseRef.get // if the current promise is emptyPromise, then this source has already been closed if (currentPromise ne emptyPromise) { diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 56a5586cd6..706782f9f4 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -261,10 +261,7 @@ abstract class Buf { outer => from == 0 && until >= length /** Helps implementations validate the arguments to [[write]]. */ - protected[this] def checkWriteArgs( - outputLen: Int, - outputOff: Int - ): Unit = { + protected[this] def checkWriteArgs(outputLen: Int, outputOff: Int): Unit = { if (outputOff < 0) throw new IllegalArgumentException(s"offset must be non-negative: $outputOff") val len = length @@ -642,8 +639,8 @@ object Buf { class ByteArray( private[Buf] val bytes: Array[Byte], private[Buf] val begin: Int, - private[Buf] val end: Int - ) extends Buf { + private[Buf] val end: Int) + extends Buf { def get(index: Int): Byte = { val off = begin + index diff --git a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala index acaa4ddd8f..8530f1e903 100644 --- a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala +++ b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala @@ -10,11 +10,8 @@ import java.io.InputStream * The given `InputStream` will be closed when [[Reader.read]] * reaches the EOF or a call to [[discard()]] or [[close()]]. */ -class InputStreamReader private[io] ( - inputStream: InputStream, - chunkSize: Int, - pool: FuturePool -) extends Reader[Buf] +class InputStreamReader private[io] (inputStream: InputStream, chunkSize: Int, pool: FuturePool) + extends Reader[Buf] with Closable with CloseAwaitably { private[this] val mutex = new AsyncMutex() @@ -97,10 +94,7 @@ object InputStreamReader { * using [[FuturePool.interruptibleUnboundedPool]] for executing * all I/O. */ - def apply( - inputStream: InputStream, - chunkSize: Int = DefaultMaxBufferSize - ): InputStreamReader = + def apply(inputStream: InputStream, chunkSize: Int = DefaultMaxBufferSize): InputStreamReader = new InputStreamReader(inputStream, chunkSize) } diff --git a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala b/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala index 7287fdabd5..4c24508126 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala +++ b/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala @@ -41,7 +41,9 @@ object ActivitySource { */ def forFiles( period: Duration = Duration.fromSeconds(60) - )(implicit timer: Timer): ActivitySource[Buf] = + )( + implicit timer: Timer + ): ActivitySource[Buf] = new CachingActivitySource(new FilePollingActivitySource(period)(timer)) /** @@ -54,8 +56,8 @@ object ActivitySource { private[ActivitySource] class OrElse[T, U >: T]( primary: ActivitySource[T], - failover: ActivitySource[U] - ) extends ActivitySource[U] { + failover: ActivitySource[U]) + extends ActivitySource[U] { def get(name: String): Activity[U] = { primary.get(name) transform { case Activity.Failed(_) => failover.get(name) @@ -115,7 +117,8 @@ class CachingActivitySource[T](underlying: ActivitySource[T]) extends ActivitySo class FilePollingActivitySource private[exp] ( period: Duration, pool: FuturePool -)(implicit timer: Timer) +)( + implicit timer: Timer) extends ActivitySource[Buf] { private[exp] def this(period: Duration)(implicit timer: Timer) = diff --git a/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala b/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala index c6e640e661..8e1e1f56e0 100644 --- a/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala +++ b/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala @@ -31,8 +31,8 @@ private[util] class BatchExecutor[In, Out]( sizePercentile: => Float = 1.0f, f: Seq[In] => Future[Seq[Out]] )( - implicit timer: Timer -) extends Function1[In, Future[Out]] { batcher => + implicit timer: Timer) + extends Function1[In, Future[Out]] { batcher => import java.util.logging.Level.WARNING class ScheduledFlush(after: Duration, timer: Timer) { diff --git a/util-core/src/main/scala/com/twitter/util/Batcher.scala b/util-core/src/main/scala/com/twitter/util/Batcher.scala index 1c1bf9f38e..91bf74dd04 100644 --- a/util-core/src/main/scala/com/twitter/util/Batcher.scala +++ b/util-core/src/main/scala/com/twitter/util/Batcher.scala @@ -1,11 +1,8 @@ package com.twitter.util /** Provides an interface for working with a batched [[com.twitter.util.Future]] */ -class Batcher[In, Out] private[util] ( - executor: BatchExecutor[In, Out] -)( - implicit timer: Timer -) extends (In => Future[Out]) { batcher => +class Batcher[In, Out] private[util] (executor: BatchExecutor[In, Out])(implicit timer: Timer) + extends (In => Future[Out]) { batcher => /** * Enqueues requests for a batched [[com.twitter.util.Future]] diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index be40e356ba..3cf0995995 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1394,8 +1394,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( sizeThreshold: Int, timeThreshold: Duration = Duration.Top, sizePercentile: => Float = 1.0f - )( - f: Seq[In] => Future[Seq[Out]] + )(f: Seq[In] => Future[Seq[Out]] )( implicit timer: Timer ): Batcher[In, Out] = { @@ -1835,9 +1834,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * * @see [[handle]] */ - def rescue[B >: A]( - rescueException: PartialFunction[Throwable, Future[B]] - ): Future[B] = + def rescue[B >: A](rescueException: PartialFunction[Throwable, Future[B]]): Future[B] = transform { case Throw(t) => val result = rescueException.applyOrElse(t, Future.AlwaysNotApplied) diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index acfa8c414f..d06a8717d3 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -118,8 +118,8 @@ class InterruptibleExecutorServiceFuturePool(executor: ExecutorService) */ class ExecutorServiceFuturePool protected[this] ( val executor: ExecutorService, - val interruptible: Boolean -) extends FuturePool { + val interruptible: Boolean) + extends FuturePool { def this(executor: ExecutorService) = this(executor, false) def apply[T](f: => T): Future[T] = { diff --git a/util-core/src/main/scala/com/twitter/util/Local.scala b/util-core/src/main/scala/com/twitter/util/Local.scala index 6674c1a28f..e11f7bad37 100644 --- a/util-core/src/main/scala/com/twitter/util/Local.scala +++ b/util-core/src/main/scala/com/twitter/util/Local.scala @@ -26,10 +26,7 @@ object Local { def set(k: Key, v: Some[_]): Context = new Context1(k, v) } - private final class Context1( - k1: Key, - v1: Some[_] - ) extends Context { + private final class Context1(k1: Key, v1: Some[_]) extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 else None @@ -42,12 +39,7 @@ object Local { else new Context2(k1, v1, k, v) } - private final class Context2( - k1: Key, - v1: Some[_], - k2: Key, - v2: Some[_] - ) extends Context { + private final class Context2(k1: Key, v1: Some[_], k2: Key, v2: Some[_]) extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -131,14 +123,8 @@ object Local { } } */ - private final class Context3( - k1: Key, - v1: Some[_], - k2: Key, - v2: Some[_], - k3: Key, - v3: Some[_] - ) extends Context { + private final class Context3(k1: Key, v1: Some[_], k2: Key, v2: Some[_], k3: Key, v3: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -167,8 +153,8 @@ object Local { k3: Key, v3: Some[_], k4: Key, - v4: Some[_] - ) extends Context { + v4: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -202,8 +188,8 @@ object Local { k4: Key, v4: Some[_], k5: Key, - v5: Some[_] - ) extends Context { + v5: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -242,8 +228,8 @@ object Local { k5: Key, v5: Some[_], k6: Key, - v6: Some[_] - ) extends Context { + v6: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -287,8 +273,8 @@ object Local { k6: Key, v6: Some[_], k7: Key, - v7: Some[_] - ) extends Context { + v7: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -337,8 +323,8 @@ object Local { k7: Key, v7: Some[_], k8: Key, - v8: Some[_] - ) extends Context { + v8: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -399,8 +385,8 @@ object Local { k8: Key, v8: Some[_], k9: Key, - v9: Some[_] - ) extends Context { + v9: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -498,8 +484,8 @@ object Local { k9: Key, v9: Some[_], k10: Key, - v10: Some[_] - ) extends Context { + v10: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -817,8 +803,8 @@ object Local { k10: Key, v10: Some[_], k11: Key, - v11: Some[_] - ) extends Context { + v11: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -1419,8 +1405,8 @@ object Local { k11: Key, v11: Some[_], k12: Key, - v12: Some[_] - ) extends Context { + v12: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -2122,8 +2108,8 @@ object Local { k12: Key, v12: Some[_], k13: Key, - v13: Some[_] - ) extends Context { + v13: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -2934,8 +2920,8 @@ object Local { k13: Key, v13: Some[_], k14: Key, - v14: Some[_] - ) extends Context { + v14: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -3863,8 +3849,8 @@ object Local { k14: Key, v14: Some[_], k15: Key, - v15: Some[_] - ) extends Context { + v15: Some[_]) + extends Context { def get(k: Key): Option[_] = if (k eq k1) v1 @@ -4851,11 +4837,7 @@ object Local { else new ContextN(k, v, this) } - private final class ContextN( - kN: Key, - vN: Some[_], - rest: Context - ) extends Context { + private final class ContextN(kN: Key, vN: Some[_], rest: Context) extends Context { def get(k: Key): Option[_] = if (k eq kN) vN diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index faf6bba1fd..3d6710bdc4 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -204,9 +204,7 @@ object Promise { } } - private abstract class Transformer[A, B]( - saved: Local.Context - ) extends K[A] { + private abstract class Transformer[A, B](saved: Local.Context) extends K[A] { protected[this] def k(r: Try[A]): Unit @@ -233,8 +231,8 @@ object Promise { private final class FutureTransformer[A, B]( saved: Local.Context, f: Try[A] => Future[B], - promise: Promise[B] - ) extends Transformer[A, B](saved) { + promise: Promise[B]) + extends Transformer[A, B](saved) { protected[this] def k(r: Try[A]): Unit = // The promise can be fulfilled only by the transformer, so it's safe to use `become` here @@ -250,8 +248,8 @@ object Promise { private final class TryTransformer[A, B]( saved: Local.Context, f: Try[A] => Try[B], - promise: Promise[B] - ) extends Transformer[A, B](saved) { + promise: Promise[B]) + extends Transformer[A, B](saved) { protected[this] def k(r: Try[A]): Unit = { // The promise can be fulfilled only by the transformer, so it's safe to use `update` here @@ -281,8 +279,7 @@ object Promise { */ private class Interruptible[A]( val waitq: WaitQueue[A], - val handler: PartialFunction[Throwable, Unit] - ) + val handler: PartialFunction[Throwable, Unit]) /** * An unsatisfied [[Promise]] which forwards interrupts to `other`. diff --git a/util-core/src/main/scala/com/twitter/util/SlowProbeProxyTimer.scala b/util-core/src/main/scala/com/twitter/util/SlowProbeProxyTimer.scala index 45d5302175..4a4fa448be 100644 --- a/util-core/src/main/scala/com/twitter/util/SlowProbeProxyTimer.scala +++ b/util-core/src/main/scala/com/twitter/util/SlowProbeProxyTimer.scala @@ -48,7 +48,8 @@ abstract class SlowProbeProxyTimer(maxRuntime: Duration) extends ProxyTimer { override protected def schedulePeriodically( when: Time, period: Duration - )(f: => Unit): TimerTask = { + )(f: => Unit + ): TimerTask = { checkSlowTask() self.schedule(when, period)(meterTask(f)) } diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index f812e42713..89cc7614f2 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -505,8 +505,7 @@ trait TimeControl { class TimeFormat( pattern: String, locale: Option[Locale], - timezone: TimeZone = TimeZone.getTimeZone("UTC") -) { + timezone: TimeZone = TimeZone.getTimeZone("UTC")) { // jdk6 and jdk7 pick up the default locale differently in SimpleDateFormat, // so we can't rely on Locale.getDefault here. diff --git a/util-core/src/main/scala/com/twitter/util/Timer.scala b/util-core/src/main/scala/com/twitter/util/Timer.scala index 325e619018..2e1d1d8bbf 100644 --- a/util-core/src/main/scala/com/twitter/util/Timer.scala +++ b/util-core/src/main/scala/com/twitter/util/Timer.scala @@ -272,8 +272,8 @@ class JavaTimer(isDaemon: Boolean, name: Option[String]) extends Timer { class ScheduledThreadPoolTimer( poolSize: Int, threadFactory: ThreadFactory, - rejectedExecutionHandler: Option[RejectedExecutionHandler] -) extends Timer { + rejectedExecutionHandler: Option[RejectedExecutionHandler]) + extends Timer { def this(poolSize: Int, threadFactory: ThreadFactory) = this(poolSize, threadFactory, None) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 6945c7c740..134d661f94 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -265,7 +265,9 @@ object Var { */ def collect[T: ClassTag, CC[X] <: Traversable[X]]( vars: CC[Var[T]] - )(implicit newBuilder: CanBuildFrom[CC[T], T, CC[T]]): Var[CC[T]] = { + )( + implicit newBuilder: CanBuildFrom[CC[T], T, CC[T]] + ): Var[CC[T]] = { val vs = vars.toArray def tree(begin: Int, end: Int): Var[Seq[T]] = diff --git a/util-core/src/main/scala/com/twitter/util/WindowedAdder.scala b/util-core/src/main/scala/com/twitter/util/WindowedAdder.scala index 70529c61ca..7088a3cc95 100644 --- a/util-core/src/main/scala/com/twitter/util/WindowedAdder.scala +++ b/util-core/src/main/scala/com/twitter/util/WindowedAdder.scala @@ -30,8 +30,7 @@ private[twitter] object WindowedAdder { private[twitter] class WindowedAdder private[WindowedAdder] ( window: Long, N: Int, - now: () => Long -) { + now: () => Long) { private[this] val writer = new LongAdder() @volatile private[this] var gen = 0 private[this] val expiredGen = new AtomicInteger(gen) diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index d6512a0f96..a09eb2eb7d 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -13,8 +13,7 @@ class AsyncSemaphoreTest extends FunSpec { class AsyncSemaphoreHelper( val sem: AsyncSemaphore, var count: Int, - val permits: ConcurrentLinkedQueue[Permit] - ) { + val permits: ConcurrentLinkedQueue[Permit]) { def copy( sem: AsyncSemaphore = this.sem, count: Int = this.count, diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index 3605252bb8..393dc499fc 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -235,12 +235,7 @@ class BufTest } test("Buf.process from and until returns -1 when fully processed") { - def assertAllProcessed( - expected: Array[Byte], - buf: Buf, - from: Int, - until: Int - ): Unit = { + def assertAllProcessed(expected: Array[Byte], buf: Buf, from: Int, until: Int): Unit = { var seen = new mutable.ListBuffer[Byte]() val processor = new Buf.Processor { def apply(byte: Byte): Boolean = { diff --git a/util-core/src/test/scala/com/twitter/util/TimerTest.scala b/util-core/src/test/scala/com/twitter/util/TimerTest.scala index c91f1ba048..d8bd170686 100644 --- a/util-core/src/test/scala/com/twitter/util/TimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimerTest.scala @@ -336,10 +336,7 @@ class TimerTest extends FunSuite with MockitoSugar with Eventually with Integrat } } - private def mockTimerLocalPropagation( - timer: MockTimer, - localValue: Int - ): Int = { + private def mockTimerLocalPropagation(timer: MockTimer, localValue: Int): Int = { Time.withCurrentTimeFrozen { tc => val timerLocal = new AtomicInteger(0) val local = new Local[Int] diff --git a/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala b/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala index ba12f5e73e..35ccf6a985 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala @@ -13,8 +13,8 @@ case class KetamaNode[A](identifier: String, weight: Int, handle: A) class KetamaDistributor[A]( ketamaNodes: Seq[KetamaNode[A]], numReps: Int, - oldLibMemcachedVersionComplianceMode: Boolean = false -) extends Distributor[A] { + oldLibMemcachedVersionComplianceMode: Boolean = false) + extends Distributor[A] { /** * Extract a little-endian integer from a given byte array starting at `offset`. diff --git a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala index 02b802094f..bbc680543e 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala @@ -20,8 +20,7 @@ case class CpuProfile( // The number of samples taken. count: Int, // The number of samples missed. - missed: Int -) { + missed: Int) { /** * Write a Google pprof-compatible profile to `out`. The format is diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index d502fcfd01..d0c7458861 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -133,11 +133,10 @@ class Hotspot extends Jvm { def asSafepointBean(x: AnyRef) = { x.asInstanceOf[{ - def getSafepointSyncTime: Long - def getTotalSafepointTime: Long - def getSafepointCount: Long - } - ] + def getSafepointSyncTime: Long + def getTotalSafepointTime: Long + def getSafepointCount: Long + }] } try { asSafepointBean(runtimeBean) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala index 5a8ba65a9a..23e5e2e63c 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala @@ -21,11 +21,7 @@ import scala.util.control.NonFatal * @param ageHisto Histogram of the number of bytes that * have been copied as many times. Note: 0-indexed. */ -case class Heap( - allocated: Long, - tenuringThreshold: Long, - ageHisto: Seq[Long] -) +case class Heap(allocated: Long, tenuringThreshold: Long, ageHisto: Seq[Long]) /** * Information about the JVM's safepoint @@ -79,18 +75,9 @@ trait Pool { } } -case class Gc( - count: Long, - name: String, - timestamp: Time, - duration: Duration -) - -case class Snapshot( - timestamp: Time, - heap: Heap, - lastGcs: Seq[Gc] -) +case class Gc(count: Long, name: String, timestamp: Time, duration: Duration) + +case class Snapshot(timestamp: Time, heap: Heap, lastGcs: Seq[Gc]) /** * Access JVM internal performance counters. We maintain a strict diff --git a/util-lint/src/main/scala/com/twitter/util/lint/Rule.scala b/util-lint/src/main/scala/com/twitter/util/lint/Rule.scala index 86ebf16ca3..85b059c6a8 100644 --- a/util-lint/src/main/scala/com/twitter/util/lint/Rule.scala +++ b/util-lint/src/main/scala/com/twitter/util/lint/Rule.scala @@ -41,13 +41,7 @@ object Rule { * * @param fn is evaluated to determine if there are any issues. */ - def apply( - category: Category, - shortName: String, - desc: String - )( - fn: => Seq[Issue] - ): Rule = { + def apply(category: Category, shortName: String, desc: String)(fn: => Seq[Issue]): Rule = { val _cat = category new Rule { def apply(): Seq[Issue] = fn diff --git a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala index b7186c7fd4..4ce7ce7f2e 100644 --- a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala @@ -97,8 +97,8 @@ class FileHandler( val append: Boolean, rotateCount: Int, formatter: Formatter, - level: Option[Level] -) extends Handler(formatter, level) { + level: Option[Level]) + extends Handler(formatter, level) { // This converts relative paths to absolute paths, as expected val (filename, name) = { diff --git a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala index c9cb895046..36009f07ae 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala @@ -90,8 +90,8 @@ class Formatter( val truncateAt: Int = 0, val truncateStackTracesAt: Int = Formatter.DefaultStackTraceSizeLimit, val useFullPackageNames: Boolean = false, - val prefix: String = Formatter.DefaultPrefix -) extends javalog.Formatter { + val prefix: String = Formatter.DefaultPrefix) + extends javalog.Formatter { private val matcher = Formatter.DateFormatRegex.matcher(prefix) diff --git a/util-logging/src/main/scala/com/twitter/logging/Handler.scala b/util-logging/src/main/scala/com/twitter/logging/Handler.scala index c894ca4ba8..e92f5f384f 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Handler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Handler.scala @@ -84,10 +84,8 @@ object StringHandler { /** * Generates a HandlerFactory that returns a StringHandler */ - def apply( - formatter: Formatter = new Formatter(), - level: Option[Level] = None - ) = () => new StringHandler(formatter, level) + def apply(formatter: Formatter = new Formatter(), level: Option[Level] = None) = + () => new StringHandler(formatter, level) /** * for java compatibility @@ -127,10 +125,8 @@ object ConsoleHandler { /** * Generates a HandlerFactory that returns a ConsoleHandler */ - def apply( - formatter: Formatter = new Formatter(), - level: Option[Level] = None - ) = () => new ConsoleHandler(formatter, level) + def apply(formatter: Formatter = new Formatter(), level: Option[Level] = None) = + () => new ConsoleHandler(formatter, level) /** * for java compatibility diff --git a/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala b/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala index 51cef970bc..7f55f2e1b1 100644 --- a/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala +++ b/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala @@ -18,10 +18,8 @@ package com.twitter.logging import java.util.{logging => javalog} -class LazyLogRecord( - level: javalog.Level, - messageGenerator: => AnyRef -) extends LogRecord(level, "") { +class LazyLogRecord(level: javalog.Level, messageGenerator: => AnyRef) + extends LogRecord(level, "") { override lazy val getMessage = messageGenerator.toString } diff --git a/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala b/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala index 1426552747..ca423a77e4 100644 --- a/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala +++ b/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala @@ -38,8 +38,8 @@ case class LoggerFactory( node: String = "", level: Option[Level] = None, handlers: List[HandlerFactory] = Nil, - useParents: Boolean = true -) extends (() => Logger) { + useParents: Boolean = true) + extends (() => Logger) { /** * Registers new handlers and setting with the logger at `node` diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala index 23951acc6e..d3f03b8cd7 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala @@ -132,8 +132,8 @@ class ScribeHandler( maxMessagesToBuffer: Int, formatter: Formatter, level: Option[Level], - statsReceiver: StatsReceiver -) extends Handler(formatter, level) { + statsReceiver: StatsReceiver) + extends Handler(formatter, level) { import ScribeHandler._ def this( diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala index 8617ea7c2a..06a1a2fe98 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala @@ -10,10 +10,7 @@ object ScribeHandlers { * the provided arguments, so Java users don't need to provide the * other default arguments in the call. */ - def apply( - category: String, - formatter: Formatter - ): () => ScribeHandler = + def apply(category: String, formatter: Formatter): () => ScribeHandler = ScribeHandler.apply( hostname = DefaultHostname, port = DefaultPort, diff --git a/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala b/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala index 016495d7cb..eeed3266fa 100644 --- a/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala @@ -138,8 +138,8 @@ class SyslogFormatter( val priority: Int = SyslogHandler.PRIORITY_USER, timezone: Option[String] = None, truncateAt: Int = 0, - truncateStackTracesAt: Int = Formatter.DefaultStackTraceSizeLimit -) extends Formatter( + truncateStackTracesAt: Int = Formatter.DefaultStackTraceSizeLimit) + extends Formatter( timezone, truncateAt, truncateStackTracesAt, diff --git a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala index a057a76a28..f2ce15ec5b 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala @@ -68,11 +68,8 @@ object ThrottledHandler { * @param maxToDisplay * Maximum duplicate log entries to pass before suppressing them. */ -class ThrottledHandler( - handler: Handler, - val duration: Duration, - val maxToDisplay: Int -) extends ProxyHandler(handler) { +class ThrottledHandler(handler: Handler, val duration: Duration, val maxToDisplay: Int) + extends ProxyHandler(handler) { // we accept some raciness here. it means that sometimes extra log lines will // get logged. this was deemed preferable to holding locks. diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala index 5b9301e099..37df35e0f0 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala @@ -70,8 +70,8 @@ class ProxyFactory[I <: AnyRef: Manifest](f: MethodCall[I] => AnyRef) private[reflect] class MethodInterceptor[I <: AnyRef]( target: Option[I], - callback: MethodCall[I] => AnyRef -) extends CGMethodInterceptor + callback: MethodCall[I] => AnyRef) + extends CGMethodInterceptor with Serializable { val targetRef = target.getOrElse(null).asInstanceOf[I] @@ -85,8 +85,8 @@ final class MethodCall[T <: AnyRef] private[reflect] ( targetRef: T, val method: Method, val args: Array[AnyRef], - methodProxy: MethodProxy -) extends (() => AnyRef) { + methodProxy: MethodProxy) + extends (() => AnyRef) { lazy val target = if (targetRef ne null) Some(targetRef) else None diff --git a/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala b/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala index ac6b55a41c..eada596bbd 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala @@ -42,8 +42,9 @@ object PemFileTestUtils { def assertIOException[A](tryValue: Try[A]): Unit = assertException[IOException, A](tryValue) - def assertLogMessage(name: String)( - logMessage: String, + def assertLogMessage( + name: String + )(logMessage: String, filename: String, expectedExMessage: String ): Unit = { @@ -51,10 +52,7 @@ object PemFileTestUtils { assert(logMessage.contains(expectedExMessage)) } - def testFileDoesntExist[A]( - name: String, - read: File => Try[A] - ): Unit = { + def testFileDoesntExist[A](name: String, read: File => Try[A]): Unit = { val handler = newHandler() val tempFile = File.createTempFile("test", "ext") tempFile.deleteOnExit() @@ -68,10 +66,7 @@ object PemFileTestUtils { assertIOException(tryReadPem) } - def testFilePathIsntFile[A]( - name: String, - read: File => Try[A] - ): Unit = { + def testFilePathIsntFile[A](name: String, read: File => Try[A]): Unit = { val handler = newHandler() val tempFile = File.createTempFile("test", "ext") tempFile.deleteOnExit() @@ -83,10 +78,7 @@ object PemFileTestUtils { assertIOException(tryReadPem) } - def testFilePathIsntReadable[A]( - name: String, - read: File => Try[A] - ): Unit = { + def testFilePathIsntReadable[A](name: String, read: File => Try[A]): Unit = { val handler = newHandler() val tempFile = File.createTempFile("test", "ext") tempFile.deleteOnExit() @@ -104,7 +96,9 @@ object PemFileTestUtils { def testEmptyFile[A <: AnyRef, B]( name: String, read: File => Try[B] - )(implicit classTag: ClassTag[A]): Unit = { + )( + implicit classTag: ClassTag[A] + ): Unit = { val handler = newHandler() val tempFile = File.createTempFile("test", "ext") tempFile.deleteOnExit() diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala index 144d975096..d640071418 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala @@ -67,8 +67,8 @@ trait ExceptionStatsHandler { class CategorizingExceptionStatsHandler( categorizer: Throwable => Option[String] = _ => None, sourceFunction: Throwable => Option[String] = _ => None, - rollup: Boolean = true -) extends ExceptionStatsHandler { + rollup: Boolean = true) + extends ExceptionStatsHandler { import ExceptionStatsHandler._ private[this] val underlying: ExceptionStatsHandler = { @@ -131,8 +131,8 @@ private[finagle] class MultiCategorizingExceptionStatsHandler( mkLabel: Throwable => String = _ => ExceptionStatsHandler.Failures, mkFlags: Throwable => Set[String] = _ => Set.empty, mkSource: Throwable => Option[String] = _ => None, - rollup: Boolean = true -) extends ExceptionStatsHandler { + rollup: Boolean = true) + extends ExceptionStatsHandler { import ExceptionStatsHandler._ def record(statsReceiver: StatsReceiver, t: Throwable): Unit = { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index 3309a279c4..0cb029010e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -10,8 +10,8 @@ package com.twitter.finagle.stats */ abstract class NameTranslatingStatsReceiver( protected val self: StatsReceiver, - namespacePrefix: String -) extends StatsReceiverProxy { + namespacePrefix: String) + extends StatsReceiverProxy { def this(self: StatsReceiver) = this(self, "") diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala index 878ec62dba..348472ab9a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala @@ -6,8 +6,8 @@ package com.twitter.finagle.stats */ class VerbosityAdjustingStatsReceiver( protected val self: StatsReceiver, - defaultVerbosity: Verbosity -) extends StatsReceiverProxy { + defaultVerbosity: Verbosity) + extends StatsReceiverProxy { override def counter(verbosity: Verbosity, names: String*): Counter = self.counter(defaultVerbosity, names: _*) @@ -15,10 +15,7 @@ class VerbosityAdjustingStatsReceiver( override def stat(verbosity: Verbosity, names: String*): Stat = self.stat(defaultVerbosity, names: _*) - override def addGauge( - verbosity: Verbosity, - names: String* - )(f: => Float): Gauge = { + override def addGauge(verbosity: Verbosity, names: String*)(f: => Float): Gauge = { // scalafix:off StoreGaugesAsMemberVariables self.addGauge(defaultVerbosity, names: _*)(f) // scalafix:on StoreGaugesAsMemberVariables diff --git a/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala b/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala index 21172d68c7..488f5b16dc 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala +++ b/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala @@ -257,7 +257,8 @@ trait ArgumentCapture { E: ClassTag, F: ClassTag, G: ClassTag - ](func: (A, B, C, D, E, F, G) => _): Seq[(A, B, C, D, E, F, G)] = { + ](func: (A, B, C, D, E, F, G) => _ + ): Seq[(A, B, C, D, E, F, G)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) val argCaptorC = ArgumentCaptor.forClass(classTag[C].runtimeClass.asInstanceOf[Class[C]]) @@ -293,7 +294,8 @@ trait ArgumentCapture { E: ClassTag, F: ClassTag, G: ClassTag - ](func: (A, B, C, D, E, F, G) => _): (A, B, C, D, E, F, G) = + ](func: (A, B, C, D, E, F, G) => _ + ): (A, B, C, D, E, F, G) = capturingAll[A, B, C, D, E, F, G](func).lastOption.getOrElse(noArgWasCaptured()) /** Zip 8 iterables together into a Seq of 8-tuples. */ @@ -329,7 +331,8 @@ trait ArgumentCapture { F: ClassTag, G: ClassTag, H: ClassTag - ](func: (A, B, C, D, E, F, G, H) => _): Seq[(A, B, C, D, E, F, G, H)] = { + ](func: (A, B, C, D, E, F, G, H) => _ + ): Seq[(A, B, C, D, E, F, G, H)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) val argCaptorC = ArgumentCaptor.forClass(classTag[C].runtimeClass.asInstanceOf[Class[C]]) @@ -369,7 +372,8 @@ trait ArgumentCapture { F: ClassTag, G: ClassTag, H: ClassTag - ](func: (A, B, C, D, E, F, G, H) => _): (A, B, C, D, E, F, G, H) = + ](func: (A, B, C, D, E, F, G, H) => _ + ): (A, B, C, D, E, F, G, H) = capturingAll[A, B, C, D, E, F, G, H](func).lastOption.getOrElse(noArgWasCaptured()) /** Zip 9 iterables together into a Seq of 9-tuples. */ @@ -408,7 +412,8 @@ trait ArgumentCapture { G: ClassTag, H: ClassTag, I: ClassTag - ](func: (A, B, C, D, E, F, G, H, I) => _): Seq[(A, B, C, D, E, F, G, H, I)] = { + ](func: (A, B, C, D, E, F, G, H, I) => _ + ): Seq[(A, B, C, D, E, F, G, H, I)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) val argCaptorC = ArgumentCaptor.forClass(classTag[C].runtimeClass.asInstanceOf[Class[C]]) @@ -452,7 +457,8 @@ trait ArgumentCapture { G: ClassTag, H: ClassTag, I: ClassTag - ](func: (A, B, C, D, E, F, G, H, I) => _): (A, B, C, D, E, F, G, H, I) = + ](func: (A, B, C, D, E, F, G, H, I) => _ + ): (A, B, C, D, E, F, G, H, I) = capturingAll[A, B, C, D, E, F, G, H, I](func).lastOption.getOrElse(noArgWasCaptured()) /** Zip 10 iterables together into a Seq of 10-tuples. */ @@ -478,8 +484,7 @@ trait ArgumentCapture { .zip(arg7) .zip(arg8) .zip(arg9) - .map({ case (((((((((a, b), c), d), e), f), g), h), i), j) => (a, b, c, d, e, f, g, h, i, j) } - ) + .map({ case (((((((((a, b), c), d), e), f), g), h), i), j) => (a, b, c, d, e, f, g, h, i, j) }) .toSeq } @@ -495,7 +500,8 @@ trait ArgumentCapture { H: ClassTag, I: ClassTag, J: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J) => _): Seq[(A, B, C, D, E, F, G, H, I, J)] = { + ](func: (A, B, C, D, E, F, G, H, I, J) => _ + ): Seq[(A, B, C, D, E, F, G, H, I, J)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) val argCaptorC = ArgumentCaptor.forClass(classTag[C].runtimeClass.asInstanceOf[Class[C]]) @@ -543,7 +549,8 @@ trait ArgumentCapture { H: ClassTag, I: ClassTag, J: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J) => _): (A, B, C, D, E, F, G, H, I, J) = + ](func: (A, B, C, D, E, F, G, H, I, J) => _ + ): (A, B, C, D, E, F, G, H, I, J) = capturingAll[A, B, C, D, E, F, G, H, I, J](func).lastOption.getOrElse(noArgWasCaptured()) /** Zip 11 iterables together into a Seq of 11-tuples. */ @@ -591,7 +598,8 @@ trait ArgumentCapture { I: ClassTag, J: ClassTag, K: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K) => _): Seq[(A, B, C, D, E, F, G, H, I, J, K)] = { + ](func: (A, B, C, D, E, F, G, H, I, J, K) => _ + ): Seq[(A, B, C, D, E, F, G, H, I, J, K)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) val argCaptorC = ArgumentCaptor.forClass(classTag[C].runtimeClass.asInstanceOf[Class[C]]) @@ -643,7 +651,8 @@ trait ArgumentCapture { I: ClassTag, J: ClassTag, K: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K) => _): (A, B, C, D, E, F, G, H, I, J, K) = + ](func: (A, B, C, D, E, F, G, H, I, J, K) => _ + ): (A, B, C, D, E, F, G, H, I, J, K) = capturingAll[A, B, C, D, E, F, G, H, I, J, K](func).lastOption.getOrElse(noArgWasCaptured()) /** Zip 12 iterables together into a Seq of 12-tuples. */ @@ -694,7 +703,8 @@ trait ArgumentCapture { J: ClassTag, K: ClassTag, L: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L) => _): Seq[(A, B, C, D, E, F, G, H, I, J, K, L)] = { + ](func: (A, B, C, D, E, F, G, H, I, J, K, L) => _ + ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) val argCaptorC = ArgumentCaptor.forClass(classTag[C].runtimeClass.asInstanceOf[Class[C]]) @@ -750,7 +760,8 @@ trait ArgumentCapture { J: ClassTag, K: ClassTag, L: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L) => _): (A, B, C, D, E, F, G, H, I, J, K, L) = + ](func: (A, B, C, D, E, F, G, H, I, J, K, L) => _ + ): (A, B, C, D, E, F, G, H, I, J, K, L) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L](func).lastOption.getOrElse(noArgWasCaptured()) /** Zip 13 iterables together into a Seq of 13-tuples. */ @@ -804,8 +815,7 @@ trait ArgumentCapture { K: ClassTag, L: ClassTag, M: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -866,7 +876,8 @@ trait ArgumentCapture { K: ClassTag, L: ClassTag, M: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M) => _): (A, B, C, D, E, F, G, H, I, J, K, L, M) = + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M) => _ + ): (A, B, C, D, E, F, G, H, I, J, K, L, M) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M](func).lastOption .getOrElse(noArgWasCaptured()) @@ -924,8 +935,7 @@ trait ArgumentCapture { L: ClassTag, M: ClassTag, N: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1005,8 +1015,7 @@ trait ArgumentCapture { L: ClassTag, M: ClassTag, N: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1068,8 +1077,7 @@ trait ArgumentCapture { M: ClassTag, N: ClassTag, O: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1154,8 +1162,7 @@ trait ArgumentCapture { M: ClassTag, N: ClassTag, O: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1220,8 +1227,7 @@ trait ArgumentCapture { N: ClassTag, O: ClassTag, P: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1311,8 +1317,7 @@ trait ArgumentCapture { N: ClassTag, O: ClassTag, P: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1380,8 +1385,7 @@ trait ArgumentCapture { O: ClassTag, P: ClassTag, Q: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1476,8 +1480,7 @@ trait ArgumentCapture { O: ClassTag, P: ClassTag, Q: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1551,8 +1554,7 @@ trait ArgumentCapture { P: ClassTag, Q: ClassTag, R: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1652,8 +1654,7 @@ trait ArgumentCapture { P: ClassTag, Q: ClassTag, R: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1730,8 +1731,7 @@ trait ArgumentCapture { Q: ClassTag, R: ClassTag, S: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1836,8 +1836,7 @@ trait ArgumentCapture { Q: ClassTag, R: ClassTag, S: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1923,8 +1922,7 @@ trait ArgumentCapture { R: ClassTag, S: ClassTag, T: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -2034,8 +2032,7 @@ trait ArgumentCapture { R: ClassTag, S: ClassTag, T: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T](func).lastOption .getOrElse(noArgWasCaptured()) @@ -2127,8 +2124,7 @@ trait ArgumentCapture { S: ClassTag, T: ClassTag, U: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -2243,8 +2239,7 @@ trait ArgumentCapture { S: ClassTag, T: ClassTag, U: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U](func).lastOption .getOrElse(noArgWasCaptured()) @@ -2345,8 +2340,7 @@ trait ArgumentCapture { T: ClassTag, U: ClassTag, V: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -2466,8 +2460,7 @@ trait ArgumentCapture { T: ClassTag, U: ClassTag, V: ClassTag - ]( - func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => _ + ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V](func).lastOption .getOrElse(noArgWasCaptured()) diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala index 746ccc53b4..c5bf1a361e 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala @@ -15,8 +15,7 @@ object JsonTunableMapper { private case class JsonTunable( @JsonProperty(required = true) id: String, @JsonProperty(value = "type", required = true) valueType: Class[Any], - @JsonProperty(required = true) value: Any - ) + @JsonProperty(required = true) value: Any) private case class JsonTunables(@JsonProperty(required = true) tunables: Seq[JsonTunable]) diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/ServiceLoadedTunableMap.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/ServiceLoadedTunableMap.scala index 3b5a75e174..81386a3c56 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/ServiceLoadedTunableMap.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/ServiceLoadedTunableMap.scala @@ -42,13 +42,11 @@ object ServiceLoadedTunableMap { loadedMaps.compute( serviceLoadedTunableMap.id, new BiFunction[String, TunableMap, TunableMap] { - def apply( - id: String, - curr: TunableMap - ): TunableMap = { - if (curr != null) throw new IllegalStateException( - s"Found multiple `ServiceLoadedTunableMap`s for $id: $curr, $serviceLoadedTunableMap" - ) + def apply(id: String, curr: TunableMap): TunableMap = { + if (curr != null) + throw new IllegalStateException( + s"Found multiple `ServiceLoadedTunableMap`s for $id: $curr, $serviceLoadedTunableMap" + ) else serviceLoadedTunableMap } } @@ -68,10 +66,7 @@ object ServiceLoadedTunableMap { loadedMaps.computeIfPresent( serviceLoadedTunableMap.id, new BiFunction[String, TunableMap, TunableMap] { - def apply( - id: String, - curr: TunableMap - ): TunableMap = serviceLoadedTunableMap + def apply(id: String, curr: TunableMap): TunableMap = serviceLoadedTunableMap } ) } diff --git a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala index 55535b4b5b..c80c2ee00b 100644 --- a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala +++ b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala @@ -13,8 +13,8 @@ case class NativeConnector( connectTimeout: Option[Duration], sessionTimeout: Duration, timer: Timer, - authenticate: Option[AuthInfo] = None -) extends Connector + authenticate: Option[AuthInfo] = None) + extends Connector with Serialized { override val name = "native-zk-connector" @@ -79,13 +79,20 @@ case class NativeConnector( } object NativeConnector { - def apply(connectString: String, sessionTimeout: Duration)( + def apply( + connectString: String, + sessionTimeout: Duration + )( implicit timer: Timer ): NativeConnector = { NativeConnector(connectString, None, sessionTimeout, timer) } - def apply(connectString: String, connectTimeout: Duration, sessionTimeout: Duration)( + def apply( + connectString: String, + connectTimeout: Duration, + sessionTimeout: Duration + )( implicit timer: Timer ): NativeConnector = { NativeConnector(connectString, Some(connectTimeout), sessionTimeout, timer) @@ -96,7 +103,9 @@ object NativeConnector { connectTimeout: Duration, sessionTimeout: Duration, authenticate: Option[AuthInfo] - )(implicit timer: Timer): NativeConnector = { + )( + implicit timer: Timer + ): NativeConnector = { NativeConnector(connectString, Some(connectTimeout), sessionTimeout, timer, authenticate) } @@ -115,8 +124,7 @@ object NativeConnector { sessionTimeout: Duration, authenticate: Option[AuthInfo], timer: Timer, - log: Logger - ) { + log: Logger) { @volatile protected[this] var zookeeper: Option[ZooKeeper] = None diff --git a/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala b/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala index ca8eba31ad..e07318210d 100644 --- a/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala +++ b/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala @@ -46,7 +46,8 @@ object RetryPolicy { base: Duration, factor: Double = 2.0, maximum: Duration = 30.seconds - )(implicit timer: Timer) + )( + implicit timer: Timer) extends RetryPolicy { require(base > 0.seconds) require(factor >= 1) diff --git a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala index 67d2351b16..d7f550f375 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala @@ -382,6 +382,5 @@ object ZNode { case class TreeUpdate( parent: ZNode, added: Set[ZNode] = Set.empty[ZNode], - removed: Set[ZNode] = Set.empty[ZNode] - ) + removed: Set[ZNode] = Set.empty[ZNode]) } diff --git a/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala b/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala index 7d86285ab1..12425fe3f3 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala @@ -87,14 +87,22 @@ object ZkClient { } /** Build a ZkClient with a NativeConnector */ - def apply(connectString: String, connectTimeout: Option[Duration], sessionTimeout: Duration)( + def apply( + connectString: String, + connectTimeout: Option[Duration], + sessionTimeout: Duration + )( implicit timer: Timer ): ZkClient = { apply(NativeConnector(connectString, connectTimeout, sessionTimeout, timer)) } /** Build a ZkClient with a NativeConnector */ - def apply(connectString: String, connectTimeout: Duration, sessionTimeout: Duration)( + def apply( + connectString: String, + connectTimeout: Duration, + sessionTimeout: Duration + )( implicit timer: Timer ): ZkClient = { apply(connectString, Some(connectTimeout), sessionTimeout)(timer) diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala index 3445a85bb0..99a52ff7f3 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala @@ -39,8 +39,7 @@ class ZkAsyncSemaphore( zk: ZkClient, path: String, numPermits: Int, - maxWaiters: Option[Int] = None -) { + maxWaiters: Option[Int] = None) { import ZkAsyncSemaphore._ require(numPermits > 0) require(maxWaiters.getOrElse(0) >= 0) diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index 91b46b5f4e..0c4b156c8b 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -58,7 +58,8 @@ class ZkClientTest extends WordSpec with MockitoSugar { data: Array[Byte] = "".getBytes, acls: Seq[ACL] = zkClient.acl, mode: CreateMode = zkClient.mode - )(wait: => Future[String]): Unit = { + )(wait: => Future[String] + ): Unit = { when( zk.create( meq(path), @@ -149,7 +150,9 @@ class ZkClientTest extends WordSpec with MockitoSugar { def watchChildren( path: String - )(children: => Future[ZNode.Children])(update: => Future[WatchedEvent]): Unit = { + )(children: => Future[ZNode.Children] + )(update: => Future[WatchedEvent] + ): Unit = { val w = ArgumentCaptor.forClass(classOf[Watcher]) val cb = ArgumentCaptor.forClass(classOf[AsyncCallback.Children2Callback]) when(zk.getChildren(meq(path), w.capture(), cb.capture(), meq(null))) thenAnswer answer[ @@ -183,7 +186,9 @@ class ZkClientTest extends WordSpec with MockitoSugar { def watchData( path: String - )(result: => Future[ZNode.Data])(update: => Future[WatchedEvent]): Unit = { + )(result: => Future[ZNode.Data] + )(update: => Future[WatchedEvent] + ): Unit = { val w = ArgumentCaptor.forClass(classOf[Watcher]) val cb = ArgumentCaptor.forClass(classOf[AsyncCallback.DataCallback]) when(zk.getData(meq(path), w.capture(), cb.capture(), meq(null))) thenAnswer answer[ From 8919bb23b9bea89ef1d796ba19367bc3f33959aa Mon Sep 17 00:00:00 2001 From: Owen Parry Date: Wed, 5 Dec 2018 22:08:05 +0000 Subject: [PATCH 082/796] Update scalafmt config for util. Problem: The scalafmt config doesn't match the Twitter Style Guide. This causes angst and makes onboarding new folks onto scalafmt a difficult sell. Solution: Update the scalafmt config, and everyone who has opted into automatic application of scalafmt. Result: The code automatically matches the style guide, and people are happier to adopt scalafmt. --- .../src/main/scala/com/twitter/concurrent/AsyncStream.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index fda1396e4d..04b68ddaa0 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -583,7 +583,7 @@ sealed abstract class AsyncStream[+A] { * values. */ def force: Future[Unit] = foreach { _ => - } + } } object AsyncStream { @@ -601,7 +601,7 @@ object AsyncStream { new Cons(fa, next) def unapply[A](as: Cons[A]): Option[(Future[A], () => AsyncStream[A])] = - // note: pattern match returns the memoized value + // note: pattern match returns the memoized value Some((as.fa, () => as.more())) } @@ -824,4 +824,4 @@ object AsyncStream { val reader = new Oneshot(() => as) 0.until(n).map(_ => reader.toAsyncStream) } -} \ No newline at end of file +} From ee56e5f2418d5d6540d37268436ff6f47520edaf Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 5 Dec 2018 23:45:42 +0000 Subject: [PATCH 083/796] util-core: New conversions Problem Some of the conversions in `c.t.conversions` could benefit from using `AnyVal`. However, due to ABI breakages this is, wait for it, a tough conversion to make. Solution Introduce new implementations that follow a naming scheme of `SomethingOps`. Where possible the implementations are `AnyVal` based avoiding allocations for the common usage pattern. Migrate using drop-in replacements by updating imports. Result Consistency and less allocations for users. JIRA Issues: CSL-7356 Differential Revision: https://phabricator.twitter.biz/D249403 --- CHANGELOG.rst | 26 ++++ .../com/twitter/conversions/DurationOps.scala | 40 +++++ .../com/twitter/conversions/PercentOps.scala | 33 ++++ .../twitter/conversions/StorageUnitOps.scala | 37 +++++ .../com/twitter/conversions/StringOps.scala | 144 ++++++++++++++++++ .../com/twitter/conversions/ThreadOps.scala | 19 +++ .../com/twitter/conversions/U64Ops.scala | 22 +++ .../com/twitter/conversions/percent.scala | 1 + .../com/twitter/conversions/storage.scala | 1 + .../com/twitter/conversions/string.scala | 1 + .../com/twitter/conversions/thread.scala | 1 + .../scala/com/twitter/conversions/time.scala | 1 + .../scala/com/twitter/conversions/u64.scala | 1 + .../scala/com/twitter/util/Duration.scala | 18 ++- .../twitter/conversions/DurationOpsTest.scala | 20 +++ .../twitter/conversions/PercentOpsTest.scala | 50 ++++++ .../conversions/StorageUnitOpsTest.scala | 29 ++++ .../{U64Test.scala => U64OpsTest.scala} | 5 +- 18 files changed, 438 insertions(+), 11 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/conversions/DurationOps.scala create mode 100644 util-core/src/main/scala/com/twitter/conversions/PercentOps.scala create mode 100644 util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala create mode 100644 util-core/src/main/scala/com/twitter/conversions/StringOps.scala create mode 100644 util-core/src/main/scala/com/twitter/conversions/ThreadOps.scala create mode 100644 util-core/src/main/scala/com/twitter/conversions/U64Ops.scala create mode 100644 util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala create mode 100644 util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala create mode 100644 util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala rename util-core/src/test/scala/com/twitter/conversions/{U64Test.scala => U64OpsTest.scala} (77%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a58854b50a..2c90a47b64 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,12 +13,38 @@ New Features * util-core: Provide a way to listen for stream termination to `c.t.util.Reader`, `Reader#onClose` which is satisfied when the stream is discarded or read until the end. ``PHAB_ID=D236311`` +* util-core: Conversions in `c.t.conversions` have new implementations + that follow a naming scheme of `SomethingOps`. Where possible the implementations + are `AnyVal` based avoiding allocations for the common usage pattern. + ``PHAB_ID=D249403`` + + - `percent` is now `PercentOps` + - `storage` is now `StorageUnitOps` + - `string` is now `StringOps` + - `thread` is now `ThreadOps` + - `time` is now `DurationOps` + - `u64` is now `U64Ops` + Bug Fixes ~~~~~~~~~ * util-core: Fixed a bug where tail would sometimes return Some empty AsyncStream instead of None. ``PHAB_ID=D241513`` +Deprecations +~~~~~~~~~~~~ + +* util-core: Conversions in `c.t.conversions` have been deprecated in favor of `SomethingOps` + versions. Where possible the implementations are `AnyVal` based and use implicit classes + instead of implicit conversions. ``PHAB_ID=D249403`` + + - `percent` is now `PercentOps` + - `storage` is now `StorageUnitOps` + - `string` is now `StringOps` + - `thread` is now `ThreadOps` + - `time` is now `DurationOps` + - `u64` is now `U64Ops` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala b/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala new file mode 100644 index 0000000000..a9cc5bc904 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala @@ -0,0 +1,40 @@ +package com.twitter.conversions + +import com.twitter.util.Duration +import java.util.concurrent.TimeUnit + +/** + * Implicits for writing readable [[Duration]]s. + * + * @example + * {{{ + * import com.twitter.conversions.DurationOps._ + * + * 2000.nanoseconds + * 50.milliseconds + * 1.second + * 24.hours + * 40.days + * }}} + */ +object DurationOps { + + implicit class RichLong(val numNanos: Long) extends AnyVal { + def nanoseconds: Duration = Duration(numNanos, TimeUnit.NANOSECONDS) + def nanosecond: Duration = nanoseconds + def microseconds: Duration = Duration(numNanos, TimeUnit.MICROSECONDS) + def microsecond: Duration = microseconds + def milliseconds: Duration = Duration(numNanos, TimeUnit.MILLISECONDS) + def millisecond: Duration = milliseconds + def millis: Duration = milliseconds + def seconds: Duration = Duration(numNanos, TimeUnit.SECONDS) + def second: Duration = seconds + def minutes: Duration = Duration(numNanos, TimeUnit.MINUTES) + def minute: Duration = minutes + def hours: Duration = Duration(numNanos, TimeUnit.HOURS) + def hour: Duration = hours + def days: Duration = Duration(numNanos, TimeUnit.DAYS) + def day: Duration = days + } + +} diff --git a/util-core/src/main/scala/com/twitter/conversions/PercentOps.scala b/util-core/src/main/scala/com/twitter/conversions/PercentOps.scala new file mode 100644 index 0000000000..d916c958b8 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/PercentOps.scala @@ -0,0 +1,33 @@ +package com.twitter.conversions + +/** + * Implicits for turning `x.percent` (where `x` is an `Int` or `Double`) into a `Double` + * scaled to where 1.0 is 100 percent. + * + * @note Negative values, fractional values, and values greater than 100 are permitted. + * + * @example + * {{{ + * import com.twitter.conversions.PercentOps._ + * + * 1.percent == 0.01 + * 100.percent == 1.0 + * 99.9.percent == 0.999 + * 500.percent == 5.0 + * -10.percent == -0.1 + * }}} + */ +object PercentOps { + + private val BigDecimal100 = BigDecimal(100.0) + + implicit class RichDouble(val value: Double) extends AnyVal { + // convert wrapped to BigDecimal to preserve precision when dividing Doubles + def percent: Double = + if (value.equals(Double.NaN) + || value.equals(Double.PositiveInfinity) + || value.equals(Double.NegativeInfinity)) value + else (BigDecimal(value) / BigDecimal100).doubleValue + } + +} diff --git a/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala b/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala new file mode 100644 index 0000000000..cadf77e6ae --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala @@ -0,0 +1,37 @@ +package com.twitter.conversions + +import com.twitter.util.StorageUnit + +/** + * Implicits for writing readable [[StorageUnit]]s. + * + * @example + * {{{ + * import com.twitter.conversions.StorageUnitOps._ + * + * 5.bytes + * 1.kilobyte + * 256.gigabytes + * }}} + */ +object StorageUnitOps { + + implicit class RichLong(val numBytes: Long) extends AnyVal { + def byte: StorageUnit = bytes + def bytes: StorageUnit = StorageUnit.fromBytes(numBytes) + def kilobyte: StorageUnit = kilobytes + def kilobytes: StorageUnit = StorageUnit.fromKilobytes(numBytes) + def megabyte: StorageUnit = megabytes + def megabytes: StorageUnit = StorageUnit.fromMegabytes(numBytes) + def gigabyte: StorageUnit = gigabytes + def gigabytes: StorageUnit = StorageUnit.fromGigabytes(numBytes) + def terabyte: StorageUnit = terabytes + def terabytes: StorageUnit = StorageUnit.fromTerabytes(numBytes) + def petabyte: StorageUnit = petabytes + def petabytes: StorageUnit = StorageUnit.fromPetabytes(numBytes) + + def thousand: Long = numBytes * 1000 + def million: Long = numBytes * 1000 * 1000 + def billion: Long = numBytes * 1000 * 1000 * 1000 + } +} diff --git a/util-core/src/main/scala/com/twitter/conversions/StringOps.scala b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala new file mode 100644 index 0000000000..fd9e7dedc5 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala @@ -0,0 +1,144 @@ +package com.twitter.conversions + +import scala.util.matching.Regex + +object StringOps { + + // we intentionally don't unquote "\$" here, so it can be used to escape interpolation later. + private val UNQUOTE_RE = """\\(u[\dA-Fa-f]{4}|x[\dA-Fa-f]{2}|[/rnt\"\\])""".r + + private val QUOTE_RE = "[\u0000-\u001f\u007f-\uffff\\\\\"]".r + + implicit final class RichString(val string: String) extends AnyVal { + + /** + * For every section of a string that matches a regular expression, call + * a function to determine a replacement (as in python's + * `re.sub`). The function will be passed the Matcher object + * corresponding to the substring that matches the pattern, and that + * substring will be replaced by the function's result. + * + * For example, this call: + * {{{ + * "ohio".regexSub("""h.""".r) { m => "n" } + * }}} + * will return the string `"ono"`. + * + * The matches are found using `Matcher.find()` and so + * will obey all the normal java rules (the matches will not overlap, + * etc). + * + * @param re the regex pattern to replace + * @param replace a function that takes Regex.MatchData objects and + * returns a string to substitute + * @return the resulting string with replacements made + */ + def regexSub(re: Regex)(replace: Regex.MatchData => String): String = { + var offset = 0 + val out = new StringBuilder() + + for (m <- re.findAllIn(string).matchData) { + if (m.start > offset) { + out.append(string.substring(offset, m.start)) + } + + out.append(replace(m)) + offset = m.end + } + + if (offset < string.length) { + out.append(string.substring(offset)) + } + out.toString + } + + /** + * Quote a string so that unprintable chars (in ASCII) are represented by + * C-style backslash expressions. For example, a raw linefeed will be + * translated into `"\n"`. Control codes (anything below 0x20) + * and unprintables (anything above 0x7E) are turned into either + * `"\xHH"` or `"\\uHHHH"` expressions, depending on + * their range. Embedded backslashes and double-quotes are also quoted. + * + * @return a quoted string, suitable for ASCII display + */ + def quoteC(): String = { + regexSub(QUOTE_RE) { m => + m.matched.charAt(0) match { + case '\r' => "\\r" + case '\n' => "\\n" + case '\t' => "\\t" + case '"' => "\\\"" + case '\\' => "\\\\" + case c => + if (c <= 255) { + "\\x%02x".format(c.asInstanceOf[Int]) + } else { + "\\u%04x" format c.asInstanceOf[Int] + } + } + } + } + + /** + * Unquote an ASCII string that has been quoted in a style like + * [[quoteC()]] and convert it into a standard unicode string. + * `"\\uHHHH"` and `"\xHH"` expressions are unpacked + * into unicode characters, as well as `"\r"`, `"\n"`, + * `"\t"`, `"\\"`, and `'\"'`. + * + * @return an unquoted unicode string + */ + def unquoteC(): String = { + regexSub(UNQUOTE_RE) { m => + val ch = m.group(1).charAt(0) match { + // holy crap! this is terrible: + case 'u' => + Character.valueOf(Integer.valueOf(m.group(1).substring(1), 16).asInstanceOf[Int].toChar) + case 'x' => + Character.valueOf(Integer.valueOf(m.group(1).substring(1), 16).asInstanceOf[Int].toChar) + case 'r' => '\r' + case 'n' => '\n' + case 't' => '\t' + case x => x + } + ch.toString + } + } + + /** + * Turn a string of hex digits into a byte array. This does the exact + * opposite of `Array[Byte]#hexlify`. + */ + def unhexlify(): Array[Byte] = { + val buffer = new Array[Byte]((string.length + 1) / 2) + string.grouped(2).toSeq.zipWithIndex.foreach { + case (substr, i) => + buffer(i) = Integer.parseInt(substr, 16).toByte + } + buffer + } + } + + def hexlify(array: Array[Byte], from: Int, to: Int): String = { + val out = new StringBuffer + for (i <- from until to) { + val b = array(i) + val s = (b.toInt & 0xff).toHexString + if (s.length < 2) { + out.append('0') + } + out.append(s) + } + out.toString + } + + implicit final class RichByteArray(val bytes: Array[Byte]) extends AnyVal { + + /** + * Turn an `Array[Byte]` into a string of hex digits. + */ + def hexlify: String = StringOps.hexlify(bytes, 0, bytes.length) + } + +} diff --git a/util-core/src/main/scala/com/twitter/conversions/ThreadOps.scala b/util-core/src/main/scala/com/twitter/conversions/ThreadOps.scala new file mode 100644 index 0000000000..a6589c4c78 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/ThreadOps.scala @@ -0,0 +1,19 @@ +package com.twitter.conversions + +import java.util.concurrent.Callable +import scala.language.implicitConversions + +/** + * Implicits for turning a block of code into a Runnable or Callable. + */ +object ThreadOps { + + implicit def makeRunnable(f: => Unit): Runnable = new Runnable() { + def run(): Unit = f + } + + implicit def makeCallable[T](f: => T): Callable[T] = new Callable[T]() { + def call(): T = f + } + +} diff --git a/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala new file mode 100644 index 0000000000..07f6a3a1c6 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala @@ -0,0 +1,22 @@ +package com.twitter.conversions + +object U64Ops { + + /** + * Parses this HEX string as an unsigned 64-bit long value. Be careful, this can throw + * [[NumberFormatException]]. + * + * @see [[java.lang.Long.parseUnsignedLong()]] + */ + implicit class StringOps(val self: String) extends AnyVal { + def toU64Long: Long = java.lang.Long.parseUnsignedLong(self, 16) + } + + /** + * Converts this unsigned 64-bit long value into a 16-character HEX string. + */ + implicit class LongOps(val self: Long) extends AnyVal { + def toU64HexString: String = "%016x".format(self) + } + +} diff --git a/util-core/src/main/scala/com/twitter/conversions/percent.scala b/util-core/src/main/scala/com/twitter/conversions/percent.scala index c04ee79cb3..90a5fa942d 100644 --- a/util-core/src/main/scala/com/twitter/conversions/percent.scala +++ b/util-core/src/main/scala/com/twitter/conversions/percent.scala @@ -17,6 +17,7 @@ import scala.language.implicitConversions * -10.percent == -0.1 * }}} */ +@deprecated("Use the AnyVal version `com.twitter.conversions.PercentOps`", "2018-12-05") object percent { private val BigDecimal100 = BigDecimal(100.0) diff --git a/util-core/src/main/scala/com/twitter/conversions/storage.scala b/util-core/src/main/scala/com/twitter/conversions/storage.scala index a7dfef97e6..a34d5155b7 100644 --- a/util-core/src/main/scala/com/twitter/conversions/storage.scala +++ b/util-core/src/main/scala/com/twitter/conversions/storage.scala @@ -19,6 +19,7 @@ package com.twitter.conversions import com.twitter.util.StorageUnit import scala.language.implicitConversions +@deprecated("Use the AnyVal version `com.twitter.conversions.StorageUnitOps`", "2018-12-05") object storage { class RichWholeNumber(wrapped: Long) { def byte: StorageUnit = bytes diff --git a/util-core/src/main/scala/com/twitter/conversions/string.scala b/util-core/src/main/scala/com/twitter/conversions/string.scala index aa6aae5000..2e15ee3fb3 100644 --- a/util-core/src/main/scala/com/twitter/conversions/string.scala +++ b/util-core/src/main/scala/com/twitter/conversions/string.scala @@ -19,6 +19,7 @@ package com.twitter.conversions import scala.language.implicitConversions import scala.util.matching.Regex +@deprecated("Use the AnyVal version `com.twitter.conversions.StringOps`", "2018-12-05") object string { final class RichString(wrapped: String) { diff --git a/util-core/src/main/scala/com/twitter/conversions/thread.scala b/util-core/src/main/scala/com/twitter/conversions/thread.scala index ff4afab986..b03209c153 100644 --- a/util-core/src/main/scala/com/twitter/conversions/thread.scala +++ b/util-core/src/main/scala/com/twitter/conversions/thread.scala @@ -22,6 +22,7 @@ import scala.language.implicitConversions /** * Implicits for turning a block of code into a Runnable or Callable. */ +@deprecated("Use `com.twitter.conversions.ThreadOps`", "2018-12-05") object thread { implicit def makeRunnable(f: => Unit): Runnable = new Runnable() { def run(): Unit = f } diff --git a/util-core/src/main/scala/com/twitter/conversions/time.scala b/util-core/src/main/scala/com/twitter/conversions/time.scala index 993dbf4d6f..d8e4d9edf2 100644 --- a/util-core/src/main/scala/com/twitter/conversions/time.scala +++ b/util-core/src/main/scala/com/twitter/conversions/time.scala @@ -20,6 +20,7 @@ import com.twitter.util.Duration import java.util.concurrent.TimeUnit import scala.language.implicitConversions +@deprecated("Use the AnyVal version `com.twitter.conversions.DurationOps`", "2018-12-05") object time { class RichWholeNumber(wrapped: Long) { def nanoseconds: Duration = Duration(wrapped, TimeUnit.NANOSECONDS) diff --git a/util-core/src/main/scala/com/twitter/conversions/u64.scala b/util-core/src/main/scala/com/twitter/conversions/u64.scala index 2e04428296..6e898db129 100644 --- a/util-core/src/main/scala/com/twitter/conversions/u64.scala +++ b/util-core/src/main/scala/com/twitter/conversions/u64.scala @@ -1,5 +1,6 @@ package com.twitter.conversions +@deprecated("Use `com.twitter.conversions.U64Ops`", "2018-12-05") object u64 { /** diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index 188ddca04c..295b821911 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -5,7 +5,9 @@ import java.util.concurrent.TimeUnit object Duration extends TimeLikeOps[Duration] { - def fromNanoseconds(nanoseconds: Long): Duration = new Duration(nanoseconds) + def fromNanoseconds(nanoseconds: Long): Duration = + if (nanoseconds == 0L) Zero + else new Duration(nanoseconds) // This is needed for Java compatibility. override def fromFractionalSeconds(seconds: Double): Duration = @@ -14,12 +16,12 @@ object Duration extends TimeLikeOps[Duration] { override def fromMilliseconds(millis: Long): Duration = super.fromMilliseconds(millis) override def fromMicroseconds(micros: Long): Duration = super.fromMicroseconds(micros) - val NanosPerMicrosecond = 1000L - val NanosPerMillisecond = NanosPerMicrosecond * 1000L - val NanosPerSecond = NanosPerMillisecond * 1000L - val NanosPerMinute = NanosPerSecond * 60 - val NanosPerHour = NanosPerMinute * 60 - val NanosPerDay = NanosPerHour * 24 + val NanosPerMicrosecond: Long = 1000L + val NanosPerMillisecond: Long = NanosPerMicrosecond * 1000L + val NanosPerSecond: Long = NanosPerMillisecond * 1000L + val NanosPerMinute: Long = NanosPerSecond * 60 + val NanosPerHour: Long = NanosPerMinute * 60 + val NanosPerDay: Long = NanosPerHour * 24 /** * Create a duration from a [[java.util.concurrent.TimeUnit]]. @@ -36,7 +38,7 @@ object Duration extends TimeLikeOps[Duration] { } // This is needed for Java compatibility. - override val Zero: Duration = fromNanoseconds(0) + override val Zero: Duration = new Duration(0) /** * Duration `Top` is greater than any other duration, except for diff --git a/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala new file mode 100644 index 0000000000..4a196e4431 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala @@ -0,0 +1,20 @@ +package com.twitter.conversions + +import com.twitter.util.Duration +import org.scalatest.FunSuite +import com.twitter.conversions.DurationOps._ + +class DurationOpsTest extends FunSuite { + test("converts Duration.Zero") { + assert(0.seconds eq Duration.Zero) + assert(0.milliseconds eq Duration.Zero) + assert(0.seconds eq 0.seconds) + } + + test("converts nonzero durations") { + assert(1.seconds == Duration.fromSeconds(1)) + assert(123.milliseconds == Duration.fromMilliseconds(123)) + assert(100L.nanoseconds == Duration.fromNanoseconds(100L)) + } + +} diff --git a/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala new file mode 100644 index 0000000000..4bd407a97d --- /dev/null +++ b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala @@ -0,0 +1,50 @@ +package com.twitter.conversions + +import com.twitter.conversions.PercentOps._ +import org.scalacheck.Arbitrary.arbitrary +import org.scalacheck.Gen +import org.scalatest.FunSuite +import org.scalatest.prop.GeneratorDrivenPropertyChecks + +class PercentOpsTest extends FunSuite with GeneratorDrivenPropertyChecks { + + private[this] val Precision = 0.0000000001 + + private[this] def doubleEq(d1: Double, d2: Double): Boolean = { + Math.abs(d1 - d2) <= Precision + } + + test("percent can be fractional and precision is preserved") { + assert(99.9.percent == 0.999) + assert(99.99.percent == 0.9999) + assert(99.999.percent == 0.99999) + assert(12.3456.percent == 0.123456) + } + + test("percent can be > 100") { + assert(101.percent == 1.01) + assert(500.percent == 5.0) + } + + test("percent can be < 0") { + assert(-0.1.percent == -0.001) + assert(-500.percent == -5.0) + } + + test("assorted percentages") { + forAll(arbitrary[Int]) { i => + assert(new RichDouble(i).percent == i / 100.0) + } + + // We're not as accurate when we get into high double-digit exponents, but that's acceptable. + forAll(Gen.choose(-10000D, 10000D)) { d => + assert(doubleEq(new RichDouble(d).percent, d / 100.0)) + } + } + + test("doesn't blow up on edge cases") { + assert(Double.NaN.percent.equals(Double.NaN)) + assert(Double.NegativeInfinity.percent.equals(Double.NegativeInfinity)) + assert(Double.PositiveInfinity.percent.equals(Double.PositiveInfinity)) + } +} diff --git a/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala new file mode 100644 index 0000000000..2e46ba726b --- /dev/null +++ b/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala @@ -0,0 +1,29 @@ +package com.twitter.conversions + +import com.twitter.util.StorageUnit +import org.scalatest.FunSuite +import com.twitter.conversions.StorageUnitOps._ + +class StorageUnitOpsTest extends FunSuite { + + test("converts") { + assert(StorageUnit.fromBytes(1) == 1.byte) + assert(StorageUnit.fromBytes(2) == 2.bytes) + + assert(StorageUnit.fromKilobytes(1) == 1.kilobyte) + assert(StorageUnit.fromKilobytes(3) == 3.kilobytes) + + assert(StorageUnit.fromMegabytes(1) == 1.megabyte) + assert(StorageUnit.fromMegabytes(4) == 4.megabytes) + + assert(StorageUnit.fromGigabytes(1) == 1.gigabyte) + assert(StorageUnit.fromGigabytes(5) == 5.gigabytes) + + assert(StorageUnit.fromTerabytes(1) == 1.terabyte) + assert(StorageUnit.fromTerabytes(6) == 6.terabytes) + + assert(StorageUnit.fromPetabytes(1) == 1.petabyte) + assert(StorageUnit.fromPetabytes(7) == 7.petabytes) + } + +} diff --git a/util-core/src/test/scala/com/twitter/conversions/U64Test.scala b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala similarity index 77% rename from util-core/src/test/scala/com/twitter/conversions/U64Test.scala rename to util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala index aa98441cf4..32736a0b38 100644 --- a/util-core/src/test/scala/com/twitter/conversions/U64Test.scala +++ b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala @@ -3,9 +3,8 @@ package com.twitter.conversions import org.scalatest.FunSuite import org.scalatest.prop.GeneratorDrivenPropertyChecks -class U64Test extends FunSuite with GeneratorDrivenPropertyChecks { - - import u64._ +class U64OpsTest extends FunSuite with GeneratorDrivenPropertyChecks { + import com.twitter.conversions.U64Ops._ test("toU64HextString") { forAll { l: Long => From 1b46dba5403642e7efdea2ff8e033385ac6547a7 Mon Sep 17 00:00:00 2001 From: David Rusek Date: Tue, 11 Dec 2018 15:28:48 +0000 Subject: [PATCH 084/796] Twitter-oss: Prepare OSS libraries for release 18.12.0 Problem We want to release the next versions of our Twitter OSS libraries 18.12.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D251041 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2c90a47b64..d3033c8631 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +18.12.0 +------- + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 6b7d5d0b8f..0e105ab931 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "18.11.0" +val collUtils = "com.twitter" %% "util-collection" % "18.12.0" ``` # Units diff --git a/build.sbt b/build.sbt index af8905b0c4..0127045423 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.12.0-SNAPSHOT" +val releaseVersion = "18.12.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 487c2183708ae196274cbbbdb62d9b5664d258c3 Mon Sep 17 00:00:00 2001 From: David Rusek Date: Wed, 12 Dec 2018 19:20:41 +0000 Subject: [PATCH 085/796] Twitter-oss: update OSS libraries post-release to 19.1.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D252019 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0e105ab931..84807ea776 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 0127045423..1e172cecd1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "18.12.0" +val releaseVersion = "19.1.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 2d5d6da989ab83e8701b9f57ae0708a5851beb07 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Thu, 13 Dec 2018 01:08:28 +0000 Subject: [PATCH 086/796] util-core: Avoid conversions with same class name Problem As reported on Github in https://github.com/twitter/util/issues/239 the new conversion ops cannot be wildcard imported and used in the same class. Solution Give the classes unique names. Differential Revision: https://phabricator.twitter.biz/D252462 --- CHANGELOG.rst | 7 +++++++ .../com/twitter/conversions/DurationOps.scala | 2 +- .../com/twitter/conversions/PercentOps.scala | 2 +- .../com/twitter/conversions/StorageUnitOps.scala | 2 +- .../scala/com/twitter/conversions/U64Ops.scala | 4 ++-- .../twitter/conversions/OpsTogetherTest.scala | 16 ++++++++++++++++ .../com/twitter/conversions/PercentOpsTest.scala | 4 ++-- 7 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 util-core/src/test/scala/com/twitter/conversions/OpsTogetherTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d3033c8631..48c6585440 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + + * util-core: The implicit conversions classes in `c.t.conversions.SomethingOps` have been + renamed to have unique names. This allows them to be used together with wildcard imports. + See Github issue (https://github.com/twitter/util/issues/239). ``PHAB_ID=D252462`` + 18.12.0 ------- diff --git a/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala b/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala index a9cc5bc904..0e9e2da858 100644 --- a/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala @@ -19,7 +19,7 @@ import java.util.concurrent.TimeUnit */ object DurationOps { - implicit class RichLong(val numNanos: Long) extends AnyVal { + implicit class RichDuration(val numNanos: Long) extends AnyVal { def nanoseconds: Duration = Duration(numNanos, TimeUnit.NANOSECONDS) def nanosecond: Duration = nanoseconds def microseconds: Duration = Duration(numNanos, TimeUnit.MICROSECONDS) diff --git a/util-core/src/main/scala/com/twitter/conversions/PercentOps.scala b/util-core/src/main/scala/com/twitter/conversions/PercentOps.scala index d916c958b8..ea2484726d 100644 --- a/util-core/src/main/scala/com/twitter/conversions/PercentOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/PercentOps.scala @@ -21,7 +21,7 @@ object PercentOps { private val BigDecimal100 = BigDecimal(100.0) - implicit class RichDouble(val value: Double) extends AnyVal { + implicit class RichPercent(val value: Double) extends AnyVal { // convert wrapped to BigDecimal to preserve precision when dividing Doubles def percent: Double = if (value.equals(Double.NaN) diff --git a/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala b/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala index cadf77e6ae..72044afcfc 100644 --- a/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala @@ -16,7 +16,7 @@ import com.twitter.util.StorageUnit */ object StorageUnitOps { - implicit class RichLong(val numBytes: Long) extends AnyVal { + implicit class RichStorageUnit(val numBytes: Long) extends AnyVal { def byte: StorageUnit = bytes def bytes: StorageUnit = StorageUnit.fromBytes(numBytes) def kilobyte: StorageUnit = kilobytes diff --git a/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala index 07f6a3a1c6..d9e7b31cb7 100644 --- a/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala +++ b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala @@ -8,14 +8,14 @@ object U64Ops { * * @see [[java.lang.Long.parseUnsignedLong()]] */ - implicit class StringOps(val self: String) extends AnyVal { + implicit class RichU64String(val self: String) extends AnyVal { def toU64Long: Long = java.lang.Long.parseUnsignedLong(self, 16) } /** * Converts this unsigned 64-bit long value into a 16-character HEX string. */ - implicit class LongOps(val self: Long) extends AnyVal { + implicit class RichU64Long(val self: Long) extends AnyVal { def toU64HexString: String = "%016x".format(self) } diff --git a/util-core/src/test/scala/com/twitter/conversions/OpsTogetherTest.scala b/util-core/src/test/scala/com/twitter/conversions/OpsTogetherTest.scala new file mode 100644 index 0000000000..b0e24303c9 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/conversions/OpsTogetherTest.scala @@ -0,0 +1,16 @@ +package com.twitter.conversions + +import com.twitter.util.{Duration, StorageUnit} +import org.scalatest.FunSuite + +class OpsTogetherTest extends FunSuite { + test("multiple wildcard imports") { + import com.twitter.conversions.DurationOps._ + import com.twitter.conversions.PercentOps._ + import com.twitter.conversions.StorageUnitOps._ + + assert(5.percent == 0.05) + assert(1.seconds == Duration.fromSeconds(1)) + assert(1.byte == StorageUnit.fromBytes(1)) + } +} diff --git a/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala index 4bd407a97d..68f71b9163 100644 --- a/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala @@ -33,12 +33,12 @@ class PercentOpsTest extends FunSuite with GeneratorDrivenPropertyChecks { test("assorted percentages") { forAll(arbitrary[Int]) { i => - assert(new RichDouble(i).percent == i / 100.0) + assert(new RichPercent(i).percent == i / 100.0) } // We're not as accurate when we get into high double-digit exponents, but that's acceptable. forAll(Gen.choose(-10000D, 10000D)) { d => - assert(doubleEq(new RichDouble(d).percent, d / 100.0)) + assert(doubleEq(new RichPercent(d).percent, d / 100.0)) } } From 5bdbfd67c10725422a0f8e26e810b99ec885ae79 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 19 Dec 2018 03:12:45 +0000 Subject: [PATCH 087/796] Avoid deprecated com.twitter.conversions Problem In phab D249403 these conversions were deprecated. Solution Use the new versions. Note that `c.t.c.time` and non-wildcard imports still need conversions. JIRA Issues: CSL-7356 Differential Revision: https://phabricator.twitter.biz/D254011 --- README.md | 2 +- util-core/src/main/scala/com/twitter/util/StorageUnit.scala | 2 +- .../src/test/scala/com/twitter/io/ReaderSemanticsTest.scala | 2 +- util-core/src/test/scala/com/twitter/io/ReaderTest.scala | 2 +- util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala | 2 +- .../src/test/scala/com/twitter/util/StringConversionsTest.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala | 2 +- util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala | 2 +- .../src/main/scala/com/twitter/logging/ScribeHandler.scala | 2 +- .../src/test/scala/com/twitter/logging/FileHandlerTest.scala | 2 +- .../src/test/scala/com/twitter/logging/FormatterTest.scala | 2 +- .../src/test/scala/com/twitter/logging/ScribeHandlerTest.scala | 2 +- .../scala/com/twitter/util/tunable/JsonTunableMapperTest.scala | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 84807ea776..ec3e4d8adf 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ duration1.inMillis // => 1000L ## Space ```scala -import com.twitter.conversions.storage._ +import com.twitter.conversions.StorageUnitOps._ val amount = 8.megabytes amount.inBytes // => 8388608L amount.inKilobytes // => 8192L diff --git a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala index 01620d4111..1c608a23a8 100644 --- a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala +++ b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala @@ -73,7 +73,7 @@ object StorageUnit { * from `Long` and `Int` to construct instances. * * {{{ - * import com.twitter.conversions.storage._ + * import com.twitter.conversions.StorageUnitOps._ * import com.twitter.util.StorageUnit * * val size: StorageUnit = 10.kilobytes diff --git a/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala index b52f8069ce..92c414888a 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala @@ -1,7 +1,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream -import com.twitter.conversions.storage._ +import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.time._ import com.twitter.util.{Future, Return, Await, Throw} import java.io.ByteArrayInputStream diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 45e243a68c..df65c13ed5 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -2,7 +2,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.conversions.time._ -import com.twitter.conversions.storage._ +import com.twitter.conversions.StorageUnitOps._ import com.twitter.util.{Await, Awaitable, Future, Promise} import java.io.{ByteArrayInputStream, ByteArrayOutputStream} import java.util.concurrent.atomic.AtomicBoolean diff --git a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala index 3dbac51527..1770334a62 100644 --- a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala @@ -4,7 +4,7 @@ import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner -import com.twitter.conversions.storage._ +import com.twitter.conversions.StorageUnitOps._ @RunWith(classOf[JUnitRunner]) class StorageUnitTest extends FunSuite { diff --git a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala b/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala index b2e0baabeb..4ac9d3d6a5 100644 --- a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala @@ -20,7 +20,7 @@ import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner -import com.twitter.conversions.string._ +import com.twitter.conversions.StringOps._ @RunWith(classOf[JUnitRunner]) class StringConversionsTest extends WordSpec { diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index d0c7458861..b0238a4788 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -1,6 +1,6 @@ package com.twitter.jvm -import com.twitter.conversions.storage._ +import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.time._ import com.twitter.util.Time import java.lang.management.ManagementFactory diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index c8bad9d714..8d55fcebf2 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -1,6 +1,6 @@ package com.twitter.jvm -import com.twitter.conversions.string._ +import com.twitter.conversions.StringOps._ import com.twitter.finagle.stats.{BroadcastStatsReceiver, StatsReceiver} import java.lang.management.{ManagementFactory, BufferPoolMXBean} import scala.collection.JavaConverters._ diff --git a/util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala index f98e07f479..fa57bb40bb 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala @@ -1,6 +1,6 @@ package com.twitter.jvm -import com.twitter.conversions.storage._ +import com.twitter.conversions.StorageUnitOps._ import com.twitter.util.Time object NilJvm extends Jvm { diff --git a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala index 827b8264e2..f318994ce8 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala @@ -12,7 +12,7 @@ package com.twitter.jvm * in order: current time, next gc, estimated next GC. */ object EstimatorApp extends App { - import com.twitter.conversions.storage._ + import com.twitter.conversions.StorageUnitOps._ val estimator = args match { case Array("kalman", n, error) => diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala index d3f03b8cd7..b814870f9d 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala @@ -25,7 +25,7 @@ import java.util.concurrent.{ArrayBlockingQueue, LinkedBlockingQueue, TimeUnit, import java.util.{Arrays, logging => javalog} import com.twitter.concurrent.NamedPoolThreadFactory -import com.twitter.conversions.string._ +import com.twitter.conversions.StringOps._ import com.twitter.conversions.time._ import com.twitter.finagle.stats.{NullStatsReceiver, StatsReceiver} import com.twitter.util.{Duration, Time} diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index 1f6f5b0f5f..b3a53d03ee 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -21,7 +21,7 @@ import java.util.{Calendar, Date, logging => javalog} import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner -import com.twitter.conversions.storage._ +import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.time._ import com.twitter.io.TempFolder import com.twitter.util.Time diff --git a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala index 4798346276..617a4f055d 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala @@ -22,7 +22,7 @@ import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner -import com.twitter.conversions.string._ +import com.twitter.conversions.StringOps._ @RunWith(classOf[JUnitRunner]) class FormatterTest extends WordSpec { diff --git a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala index 94b6910641..1fefbff6fe 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala @@ -17,7 +17,7 @@ package com.twitter package logging -import com.twitter.conversions.string._ +import com.twitter.conversions.StringOps._ import com.twitter.conversions.time._ import com.twitter.finagle.stats.InMemoryStatsReceiver import com.twitter.util.{RandomSocket, Time} diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala index 5d51c858c7..5053135b35 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala @@ -3,7 +3,7 @@ package com.twitter.util.tunable import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.deser.std.StdDeserializer -import com.twitter.conversions.storage._ +import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.time._ import com.twitter.util.{Duration, Return, StorageUnit, Throw} import org.scalatest.FunSuite From 9998019a123ecd5b98c693a58b6c92d11ab444e5 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 19 Dec 2018 20:42:36 +0000 Subject: [PATCH 088/796] util-core: Reader.copy should handle the cancellation Problem When canceling the copy operation from a Reader to a Pipe, pipe translates the interrupt to ReaderDiscardedException so the interrupt handler cannot catch it and discard the Reader. Solution Handle the Failure first before `proxyTo` the new Promise. Differential Revision: https://phabricator.twitter.biz/D254531 --- .../main/scala/com/twitter/io/Reader.scala | 17 +++++++---- .../scala/com/twitter/io/ReaderTest.scala | 28 +++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index ec8ec3d10a..0a6cce4025 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -309,7 +309,8 @@ object Reader { } /** - * Copy bytes from many Readers to a Writer. The Writer is unmanaged, the + * Copy elements from many Readers to a Writer. Readers will be discarded if + * `copy` is cancelled (discarding the target). The Writer is unmanaged, the * caller is responsible for finalization and error handling, e.g.: * * {{{ @@ -320,9 +321,9 @@ object Reader { readers.foreachF(Reader.copy(_, target)) /** - * Copy the bytes from a Reader to a Writer in chunks of size `n`. The Writer - * is unmanaged, the caller is responsible for finalization and error - * handling, e.g.: + * Copy elements from a Reader to a Writer. The Reader will be discarded if + * `copy` is cancelled (discarding the writer). The Writer is unmanaged, the caller + * is responsible for finalization and error handling, e.g.: * * {{{ * Reader.copy(r, w, n) ensure w.close() @@ -337,7 +338,13 @@ object Reader { val p = new Promise[Unit] // We have to do this because discarding the writer doesn't interrupt read // operations, it only fails the next write operation. - loop().proxyTo(p) + loop().respond { + case t @ Throw(ex) if ex.isInstanceOf[ReaderDiscardedException] => + r.discard() + p.updateIfEmpty(t) + case a => + p.updateIfEmpty(a) + } p.setInterruptHandler { case exc => r.discard() } p } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index df65c13ed5..1c0e21e610 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -79,6 +79,34 @@ class ReaderTest } } + test("Reader.copy - discard the source if cancelled") { + var cancelled = false + val src = new Reader[Int] { + def read(): Future[Option[Int]] = Future.value(Some(1)) + def discard(): Unit = cancelled = true + def onClose: Future[StreamTermination] = Future.never + } + val dest = new Pipe[Int] + Reader.copy(src, dest) + assert(await(dest.read()) == Some(1)) + dest.discard() + assert(cancelled) + } + + test("Reader.copy - discard the source if interrupted") { + var cancelled = false + val src = new Reader[Int] { + def read(): Future[Option[Int]] = Future.value(Some(1)) + def discard(): Unit = cancelled = true + def onClose: Future[StreamTermination] = Future.never + } + val dest = new Pipe[Int] + val f = Reader.copy(src, dest) + assert(await(dest.read()) == Some(1)) + f.raise(new Exception("Freeze!")) + assert(cancelled) + } + test("Reader.chunked") { val stringAndChunk = for { s <- Gen.alphaStr From c4c33a3424c53cdad13b05536abfaa07eb78ff22 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 19 Dec 2018 22:28:22 +0000 Subject: [PATCH 089/796] Avoid deprecated com.twitter.conversions Problem In phab D249403 these conversions were deprecated. Solution Use the new versions. JIRA Issues: CSL-7356 Differential Revision: https://phabricator.twitter.biz/D254765 --- README.md | 4 ++-- doc/src/sphinx/util-cookbook/basics.rst | 4 ++-- util-app/src/main/scala/com/twitter/app/App.scala | 2 +- util-app/src/test/scala/com/twitter/app/AppTest.scala | 2 +- util-app/src/test/scala/com/twitter/app/FlagTest.scala | 2 +- util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala | 2 +- .../scala/com/twitter/concurrent/AsyncStreamBenchmark.scala | 2 +- .../src/main/scala/com/twitter/util/TimerBenchmark.scala | 2 +- util-cache/src/main/scala/com/twitter/cache/Refresh.scala | 2 +- .../com/twitter/cache/AbstractLoadingFutureCacheTest.scala | 2 +- util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala | 2 +- .../scala/com/twitter/collection/GenerationalQueueTest.scala | 2 +- util-core/src/main/scala/com/twitter/util/Duration.scala | 2 +- .../test/scala/com/twitter/concurrent/AsyncMeterTest.scala | 2 +- .../scala/com/twitter/concurrent/AsyncSemaphoreTest.scala | 2 +- .../test/scala/com/twitter/concurrent/AsyncStreamTest.scala | 2 +- .../src/test/scala/com/twitter/concurrent/OfferTest.scala | 2 +- .../src/test/scala/com/twitter/concurrent/OnceTest.scala | 2 +- .../src/test/scala/com/twitter/concurrent/PeriodTest.scala | 2 +- .../src/test/scala/com/twitter/concurrent/SchedulerTest.scala | 2 +- .../src/test/scala/com/twitter/concurrent/SpoolTest.scala | 2 +- util-core/src/test/scala/com/twitter/io/BufReaderTest.scala | 2 +- .../src/test/scala/com/twitter/io/InputStreamReaderTest.scala | 2 +- util-core/src/test/scala/com/twitter/io/PipeTest.scala | 2 +- .../src/test/scala/com/twitter/io/ReaderSemanticsTest.scala | 2 +- util-core/src/test/scala/com/twitter/io/ReaderTest.scala | 2 +- util-core/src/test/scala/com/twitter/io/WriterTest.scala | 2 +- .../test/scala/com/twitter/io/exp/ActivitySourceTest.scala | 2 +- .../src/test/scala/com/twitter/util/ClosableOnceTest.scala | 2 +- util-core/src/test/scala/com/twitter/util/ClosableTest.scala | 2 +- .../src/test/scala/com/twitter/util/CloseAwaitablyTest.scala | 2 +- util-core/src/test/scala/com/twitter/util/DurationTest.scala | 2 +- .../src/test/scala/com/twitter/util/FuturePoolTest.scala | 2 +- util-core/src/test/scala/com/twitter/util/FutureTest.scala | 2 +- util-core/src/test/scala/com/twitter/util/MemoizeTest.scala | 2 +- util-core/src/test/scala/com/twitter/util/PoolTest.scala | 2 +- util-core/src/test/scala/com/twitter/util/PromiseTest.scala | 2 +- .../test/scala/com/twitter/util/SlowProbeProxyTimerTest.scala | 2 +- util-core/src/test/scala/com/twitter/util/TimeTest.scala | 2 +- util-core/src/test/scala/com/twitter/util/TimerTest.scala | 2 +- .../src/test/scala/com/twitter/util/TokenBucketTest.scala | 2 +- .../src/test/scala/com/twitter/util/WindowedAdderTest.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/GcPredictor.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala | 2 +- util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala | 2 +- util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala | 2 +- .../src/main/scala/com/twitter/logging/ScribeHandler.scala | 2 +- .../src/main/scala/com/twitter/logging/ThrottledHandler.scala | 2 +- .../main/scala/com/twitter/logging/config/LoggerConfig.scala | 2 +- .../src/test/scala/com/twitter/logging/FileHandlerTest.scala | 2 +- .../src/test/scala/com/twitter/logging/LoggerTest.scala | 2 +- .../test/scala/com/twitter/logging/ScribeHandlerTest.scala | 2 +- .../test/scala/com/twitter/logging/ThrottledHandlerTest.scala | 2 +- .../scala/com/twitter/finagle/stats/StatsReceiverTest.scala | 2 +- .../com/twitter/util/tunable/JsonTunableMapperTest.scala | 2 +- util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala | 2 +- util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala | 2 +- .../com/twitter/zk/coordination/ShardCoordinatorTest.scala | 2 +- .../com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala | 2 +- 61 files changed, 63 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index ec3e4d8adf..c6aa1f3ba7 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ val collUtils = "com.twitter" %% "util-collection" % "18.12.0" ## Time ```scala -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ val duration1 = 1.second val duration2 = 2.minutes @@ -64,7 +64,7 @@ amount.inKilobytes // => 8192L A Non-actor re-implementation of Scala Futures. ```scala -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, Promise} val f = new Promise[Int] diff --git a/doc/src/sphinx/util-cookbook/basics.rst b/doc/src/sphinx/util-cookbook/basics.rst index 94c20e1558..0e1e144416 100644 --- a/doc/src/sphinx/util-cookbook/basics.rst +++ b/doc/src/sphinx/util-cookbook/basics.rst @@ -85,7 +85,7 @@ Scala: .. code-block:: scala - import com.twitter.conversions.time._ + import com.twitter.conversions.DurationOps._ import com.twitter.util.{FuturePool, Time} val time = Time.fromMilliseconds(123456L) @@ -135,7 +135,7 @@ Scala: .. code-block:: scala - import com.twitter.conversions.time._ + import com.twitter.conversions.DurationOps._ import com.twitter.util.{Future, MockTimer, Time} Time.withCurrentTimeFrozen { timeControl => diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index fbd03ba8e2..7b90e14063 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -1,6 +1,6 @@ package com.twitter.app -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.lang.reflect.InvocationTargetException import java.util.concurrent.ConcurrentLinkedQueue diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index dc81410b57..6ed8c5e3b3 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -1,7 +1,7 @@ package com.twitter.app import com.twitter.app.LoadService.Binding -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.util.concurrent.ConcurrentLinkedQueue import org.scalatest.FunSuite diff --git a/util-app/src/test/scala/com/twitter/app/FlagTest.scala b/util-app/src/test/scala/com/twitter/app/FlagTest.scala index 263aaeb898..d92dfb04d5 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagTest.scala @@ -1,7 +1,7 @@ package com.twitter.app import com.twitter.app.SolarSystemPlanets._ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Awaitable} import org.scalatest.FunSuite import scala.collection.mutable diff --git a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala index 5e83545800..1ea79207d1 100644 --- a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala +++ b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala @@ -1,7 +1,7 @@ package com.twitter.app import com.twitter.app.LoadService.Binding -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.util.loadServiceDenied import com.twitter.util.registry.{Entry, GlobalRegistry, SimpleRegistry} import com.twitter.util.{Await, Future, FuturePool} diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala index 89d1c1854d..c90d5a8f3a 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncStreamBenchmark.scala @@ -1,6 +1,6 @@ package com.twitter.concurrent -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, StdBenchAnnotations} import org.openjdk.jmh.annotations._ diff --git a/util-benchmark/src/main/scala/com/twitter/util/TimerBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/TimerBenchmark.scala index 2f3f22b4c6..76f0cf3e18 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/TimerBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/TimerBenchmark.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import java.util.concurrent.TimeUnit import org.openjdk.jmh.annotations._ diff --git a/util-cache/src/main/scala/com/twitter/cache/Refresh.scala b/util-cache/src/main/scala/com/twitter/cache/Refresh.scala index 3c12ffe8bc..e94b7b6766 100644 --- a/util-cache/src/main/scala/com/twitter/cache/Refresh.scala +++ b/util-cache/src/main/scala/com/twitter/cache/Refresh.scala @@ -17,7 +17,7 @@ import java.util.concurrent.atomic.AtomicReference * def getData(): Future[T] = { ... } * * can be memoized with a TTL of 1 hour as follows: - * import com.twitter.conversions.time._ + * import com.twitter.conversions.DurationOps._ * import com.twitter.cache.Refresh * val getData: () => Future[T] = Refresh.every(1.hour) { ... } */ diff --git a/util-cache/src/test/scala/com/twitter/cache/AbstractLoadingFutureCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/AbstractLoadingFutureCacheTest.scala index 03a4334ff7..5367d2bfb0 100644 --- a/util-cache/src/test/scala/com/twitter/cache/AbstractLoadingFutureCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/AbstractLoadingFutureCacheTest.scala @@ -1,6 +1,6 @@ package com.twitter.cache -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} import org.scalatest.FunSuite diff --git a/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala b/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala index 963a363b95..e092d119ef 100644 --- a/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala @@ -1,6 +1,6 @@ package com.twitter.cache -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, Promise, Time} import org.mockito.Mockito._ import org.scalatest._ diff --git a/util-collection/src/test/scala/com/twitter/collection/GenerationalQueueTest.scala b/util-collection/src/test/scala/com/twitter/collection/GenerationalQueueTest.scala index a698bfc506..28cea593e1 100644 --- a/util-collection/src/test/scala/com/twitter/collection/GenerationalQueueTest.scala +++ b/util-collection/src/test/scala/com/twitter/collection/GenerationalQueueTest.scala @@ -4,7 +4,7 @@ import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Duration, Time} @RunWith(classOf[JUnitRunner]) diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index 295b821911..caf4ca8023 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -289,7 +289,7 @@ private[util] object DurationBox { * using the time conversions: * * {{{ - * import com.twitter.conversions.time._ + * import com.twitter.conversions.DurationOps._ * * 3.days+4.nanoseconds * }}} diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala index e91e27c231..050ea9dbd2 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala @@ -1,7 +1,7 @@ package com.twitter.concurrent import com.twitter.util._ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import java.util.concurrent.{RejectedExecutionException, CancellationException} import org.junit.runner.RunWith import org.scalatest.FunSuite diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index a09eb2eb7d..3a44f6493e 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -1,6 +1,6 @@ package com.twitter.concurrent -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.util.concurrent.{ConcurrentLinkedQueue, RejectedExecutionException} import org.junit.runner.RunWith diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 1a4c648822..3c64ad627a 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -1,6 +1,6 @@ package com.twitter.concurrent -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util._ import org.junit.runner.RunWith import org.scalacheck.{Arbitrary, Gen} diff --git a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala index ee5472c59e..0ea3257558 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala @@ -8,7 +8,7 @@ import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, MockTimer, Promise, Return, Time} class SimpleOffer[T](var futures: Stream[Future[Tx[T]]]) extends Offer[T] { diff --git a/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala index 0371ade6e7..0d52547087 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala @@ -1,6 +1,6 @@ package com.twitter.concurrent -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Promise, Await} import org.junit.runner.RunWith import org.scalatest.FunSuite diff --git a/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala b/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala index 8161b9b749..06b13208c5 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala @@ -1,6 +1,6 @@ package com.twitter.concurrent -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner diff --git a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala index 778d5cae30..5c805508ab 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala @@ -1,6 +1,6 @@ package com.twitter.concurrent -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.util.concurrent.{Executors, TimeUnit} import java.util.logging.{Handler, Level, LogRecord} diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala index f2bc733c7b..35b51d7036 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala @@ -1,7 +1,7 @@ package com.twitter.concurrent import com.twitter.concurrent.Spool.{*::, seqToSpool} -import com.twitter.conversions.time.intToTimeableNumber +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, Promise, Return, Throw} import java.io.EOFException import org.junit.runner.RunWith diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index f098c2dc8d..5e5febd66d 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -1,6 +1,6 @@ package com.twitter.io -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} import org.junit.runner.RunWith import org.scalatest.FunSuite diff --git a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala index 7cb4a42df5..4ef85a0811 100644 --- a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala @@ -1,6 +1,6 @@ package com.twitter.io -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, FuturePool} import java.io.ByteArrayInputStream import org.junit.runner.RunWith diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index e13b7ccd2b..27aa91d192 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -1,6 +1,6 @@ package com.twitter.io -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.io.ReaderDiscardedException import com.twitter.util.{Await, Future, Return} import org.scalatest.{FunSuite, Matchers} diff --git a/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala index 92c414888a..80f72e2127 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala @@ -2,7 +2,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.conversions.StorageUnitOps._ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Future, Return, Await, Throw} import java.io.ByteArrayInputStream import org.scalatest.FunSuite diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 1c0e21e610..8fcda78b61 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -1,7 +1,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.conversions.StorageUnitOps._ import com.twitter.util.{Await, Awaitable, Future, Promise} import java.io.{ByteArrayInputStream, ByteArrayOutputStream} diff --git a/util-core/src/test/scala/com/twitter/io/WriterTest.scala b/util-core/src/test/scala/com/twitter/io/WriterTest.scala index 6f90b696c1..5ac95ae375 100644 --- a/util-core/src/test/scala/com/twitter/io/WriterTest.scala +++ b/util-core/src/test/scala/com/twitter/io/WriterTest.scala @@ -1,6 +1,6 @@ package com.twitter.io -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, Promise} import java.io.{ByteArrayOutputStream, OutputStream} import org.scalatest.{FunSuite, Matchers} diff --git a/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala index 440faafc6f..97441bd8d4 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala +++ b/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala @@ -62,7 +62,7 @@ class ActivitySourceTest extends FunSuite with BeforeAndAfter { test("FilePollingActivitySource") { Time.withCurrentTimeFrozen { timeControl => - import com.twitter.conversions.time._ + import com.twitter.conversions.DurationOps._ implicit val timer = new MockTimer val file = new FilePollingActivitySource(1.microsecond, FuturePool.immediatePool) val buf = file.get(tempFile) diff --git a/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala index 8801fb91f0..5ebc4a2462 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import org.scalatest.FunSuite class ClosableOnceTest extends FunSuite { diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index f6ae8fd72c..e006a98b6b 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time.intToTimeableNumber +import com.twitter.conversions.DurationOps._ import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} diff --git a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala index b5ea3ed1da..cdae36aba0 100644 --- a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 477bbfcb88..ecde962406 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -16,7 +16,7 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import java.util.concurrent.TimeUnit import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index a49f81fdd2..16437fd764 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import java.util.concurrent.{Future => JFuture, _} import org.junit.runner.RunWith import org.scalatest.FunSuite diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index 2afd1c32dd..bf39f45196 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicInteger import org.mockito.Matchers.any diff --git a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala index bc7ae18741..f2a613d27b 100644 --- a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.{TimeUnit, CountDownLatch => JavaCountDownLatch} import org.junit.runner.RunWith diff --git a/util-core/src/test/scala/com/twitter/util/PoolTest.scala b/util-core/src/test/scala/com/twitter/util/PoolTest.scala index fd11649556..bbfa4c476c 100644 --- a/util-core/src/test/scala/com/twitter/util/PoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PoolTest.scala @@ -6,7 +6,7 @@ import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ @RunWith(classOf[JUnitRunner]) class PoolTest extends WordSpec { diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index 9f28e5a554..72d97cb3ce 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner diff --git a/util-core/src/test/scala/com/twitter/util/SlowProbeProxyTimerTest.scala b/util-core/src/test/scala/com/twitter/util/SlowProbeProxyTimerTest.scala index 45e8f92f40..71808d0be9 100644 --- a/util-core/src/test/scala/com/twitter/util/SlowProbeProxyTimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/SlowProbeProxyTimerTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import org.scalatest.FunSuite import scala.collection.mutable diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 821324436d..57088b0d53 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -10,7 +10,7 @@ import org.scalatest.concurrent.{Eventually, IntegrationPatience} import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ trait TimeLikeSpec[T <: TimeLike[T]] extends WordSpec with GeneratorDrivenPropertyChecks { val ops: TimeLikeOps[T] diff --git a/util-core/src/test/scala/com/twitter/util/TimerTest.scala b/util-core/src/test/scala/com/twitter/util/TimerTest.scala index d8bd170686..b899e2b96f 100644 --- a/util-core/src/test/scala/com/twitter/util/TimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimerTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import java.util.concurrent.{CancellationException, ExecutorService} import java.util.concurrent.atomic.AtomicInteger import org.junit.runner.RunWith diff --git a/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala b/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala index 6b36aa46e2..830ac82ab1 100644 --- a/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala @@ -3,7 +3,7 @@ package com.twitter.util import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ @RunWith(classOf[JUnitRunner]) class TokenBucketTest extends FunSuite { diff --git a/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala b/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala index f057afba01..7a8f552774 100644 --- a/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner diff --git a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala index bbc680543e..142a86b36c 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala @@ -6,7 +6,7 @@ import java.nio.{ByteBuffer, ByteOrder} import scala.collection.mutable -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Duration, Future, Promise, Stopwatch, Time} /** diff --git a/util-jvm/src/main/scala/com/twitter/jvm/GcPredictor.scala b/util-jvm/src/main/scala/com/twitter/jvm/GcPredictor.scala index 477789003a..0b6d9b0c4f 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/GcPredictor.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/GcPredictor.scala @@ -1,6 +1,6 @@ package com.twitter.jvm -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Duration, Time, Timer} /** diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index b0238a4788..556a276294 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -1,7 +1,7 @@ package com.twitter.jvm import com.twitter.conversions.StorageUnitOps._ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.Time import java.lang.management.ManagementFactory import java.util.logging.{Level, Logger} diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala index 23e5e2e63c..b0142466ec 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala @@ -1,7 +1,7 @@ package com.twitter.jvm import com.twitter.concurrent.NamedPoolThreadFactory -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.lang.management.ManagementFactory import java.util.concurrent.{ConcurrentHashMap, Executors, ScheduledExecutorService, TimeUnit} diff --git a/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala index ae255d9eac..73fa5789dd 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala @@ -1,6 +1,6 @@ package com.twitter.jvm -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.Time import java.io.ByteArrayOutputStream import org.junit.runner.RunWith diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala index 0b7102ab8d..87c75e28f2 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala @@ -1,6 +1,6 @@ package com.twitter.jvm -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.logging.{Level, TestLogging} import com.twitter.util.Time import org.junit.runner.RunWith diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala index b814870f9d..26e8e204b1 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala @@ -26,7 +26,7 @@ import java.util.{Arrays, logging => javalog} import com.twitter.concurrent.NamedPoolThreadFactory import com.twitter.conversions.StringOps._ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.{NullStatsReceiver, StatsReceiver} import com.twitter.util.{Duration, Time} import scala.util.control.NonFatal diff --git a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala index f2ce15ec5b..f194ace493 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala @@ -16,7 +16,7 @@ package com.twitter.logging -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Duration, Time} import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} diff --git a/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala b/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala index 509d6bafbd..3f646e5343 100644 --- a/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala +++ b/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala @@ -17,7 +17,7 @@ package com.twitter.logging package config -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Config, Duration, NetUtil} @deprecated("use LoggerFactory", "6.12.1") diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index b3a53d03ee..c64322a85c 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -22,7 +22,7 @@ import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner import com.twitter.conversions.StorageUnitOps._ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import com.twitter.util.Time diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index 27f2a20668..95c582c17b 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -16,7 +16,7 @@ package com.twitter.logging -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import java.net.InetSocketAddress import java.util.concurrent.{Callable, CountDownLatch, Executors, Future, TimeUnit} diff --git a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala index 1fefbff6fe..92ac5957b6 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala @@ -18,7 +18,7 @@ package com.twitter package logging import com.twitter.conversions.StringOps._ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.InMemoryStatsReceiver import com.twitter.util.{RandomSocket, Time} import java.net.Socket diff --git a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala index 51922db563..c3573375af 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala @@ -16,7 +16,7 @@ package com.twitter.logging -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import com.twitter.util.Time import org.scalatest.{BeforeAndAfter, WordSpec} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 76d068d80f..3c600a0524 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -1,6 +1,6 @@ package com.twitter.finagle.stats -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} import java.util.concurrent.TimeUnit import org.junit.runner.RunWith diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala index 5053135b35..c818e2c1b6 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala @@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.twitter.conversions.StorageUnitOps._ -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Duration, Return, StorageUnit, Throw} import org.scalatest.FunSuite import scala.collection.JavaConverters._ diff --git a/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala b/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala index e07318210d..3246e77e91 100644 --- a/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala +++ b/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala @@ -2,7 +2,7 @@ package com.twitter.zk import org.apache.zookeeper.KeeperException -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Duration, Future, Timer} /** Pluggable retry strategy. */ diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index 0c4b156c8b..9d378fbbf1 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -15,7 +15,7 @@ import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.logging.{Level, Logger} import com.twitter.util._ diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala index 8e69781ee0..b929d1366f 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala @@ -8,7 +8,7 @@ import org.scalatest.WordSpec import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, JavaTimer} import com.twitter.zk.{NativeConnector, RetryPolicy, ZkClient} diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala index f80d2581eb..712d641c67 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala @@ -1,7 +1,7 @@ package com.twitter.zk.coordination import com.twitter.concurrent.Permit -import com.twitter.conversions.time._ +import com.twitter.conversions.DurationOps._ import com.twitter.util._ import com.twitter.zk.{NativeConnector, RetryPolicy, ZkClient, ZNode} import java.util.concurrent.ConcurrentLinkedQueue From ac15ad8bbd633aa5efd5e306d2dea2c40a50379e Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 20 Dec 2018 02:00:11 +0000 Subject: [PATCH 090/796] util-core: Add Reader map/flatMap and Reader.value. Problem/Solution Added Reader APIs `map` and `flatMap` to transform current `Reader` of type A to a `Reader` of type B. Added `value()` in the `Reader` object to construct a Reader from a given value. Differential Revision: https://phabricator.twitter.biz/D252165 --- CHANGELOG.rst | 6 + .../main/scala/com/twitter/io/Reader.scala | 76 ++++++++-- .../scala/com/twitter/io/ReaderTest.scala | 131 ++++++++++++++++++ 3 files changed, 205 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 48c6585440..a28d1ecafc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + + * util-core: Added Reader.map/flatMap to transform Reader[A] to Reader[B]. Added `fromFuture()` + and `value()` in the Reader object to construct a new Reader. ``PHAB_ID=D252165`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 0a6cce4025..3c6694b30d 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -75,7 +75,7 @@ import scala.collection.mutable.ListBuffer * } * }}} */ -trait Reader[+A] { +trait Reader[+A] { self => /** * Asynchronously read the next element of this stream. Returned [[Future]] will resolve into @@ -102,8 +102,36 @@ trait Reader[+A] { * that you must do once the stream is no longer being used. */ def onClose: Future[StreamTermination] + + /** + * Construct a new Reader by applying `f` to every item read from this Reader + * @param f the function constructs a new Reader[B] from the value of this Reader.read + */ + final def flatMap[B](f: A => Reader[B]): Reader[B] = { + val target = Reader.concat(Reader.toAsyncStream(map(f))) + target.onClose.respond { + case Return(StreamTermination.Discarded) => self.discard() + case _ => () + } + target + } + + /** + * Construct a new Reader by applying `f` to every item read from this Reader + * @param f the function transforms data of type A to B + */ + final def map[B](f: A => B): Reader[B] = new Reader[B] { + def read(): Future[Option[B]] = self.read().map(oa => oa.map(f)) + def discard(): Unit = self.discard() + def onClose: Future[StreamTermination] = self.onClose + } } +/** + * Abstract `Reader` class for Java compatibility. + */ +abstract class AbstractReader[+A] extends Reader[A] + /** * Indicates that a given stream was discarded by the Reader's consumer. */ @@ -181,6 +209,39 @@ object Reader { def onClose: Future[StreamTermination] = r.onClose } + /** + * Construct a `Reader` from a `Future` + */ + def fromFuture[A](fa: Future[A]): Reader[A] = new Reader[A] { + private[this] val closep = Promise[StreamTermination]() + + def read(): Future[Option[A]] = { + if (!closep.isDefined) { + closep.updateIfEmpty(StreamTermination.FullyRead.Return) + fa.map(a => Some(a)) + } else { + closep.flatMap { + case StreamTermination.FullyRead => Future.None + case StreamTermination.Discarded => Future.exception(new ReaderDiscardedException) + } + } + } + + def discard(): Unit = closep.updateIfEmpty(StreamTermination.Discarded.Return) + + def onClose: Future[StreamTermination] = closep + } + + /** + * Construct a `Reader` from a value `a` + */ + def value[A](a: A): Reader[A] = fromFuture[A](Future.value(a)) + + /** + * Construct a `Reader` from an exception `e` + */ + def exception[A](e: Throwable): Reader[A] = fromFuture[A](Future.exception(e)) + /** * Read the entire bytestream presented by `r`. */ @@ -335,16 +396,15 @@ object Reader { case None => Future.Done case Some(buf) => w.write(buf) before loop() } + + w.onClose.respond { + case Return(StreamTermination.Discarded) => r.discard() + case _ => () + } val p = new Promise[Unit] // We have to do this because discarding the writer doesn't interrupt read // operations, it only fails the next write operation. - loop().respond { - case t @ Throw(ex) if ex.isInstanceOf[ReaderDiscardedException] => - r.discard() - p.updateIfEmpty(t) - case a => - p.updateIfEmpty(a) - } + loop().proxyTo(p) p.setInterruptHandler { case exc => r.discard() } p } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 8fcda78b61..38b12582b6 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -5,6 +5,7 @@ import com.twitter.conversions.DurationOps._ import com.twitter.conversions.StorageUnitOps._ import com.twitter.util.{Await, Awaitable, Future, Promise} import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import java.nio.charset.{StandardCharsets => JChar} import java.util.concurrent.atomic.AtomicBoolean import org.mockito.Mockito._ import org.scalacheck.{Arbitrary, Gen} @@ -12,6 +13,7 @@ import org.scalatest.concurrent.{Eventually, IntegrationPatience} import org.scalatest.prop.GeneratorDrivenPropertyChecks import org.scalatest.{FunSuite, Matchers} import scala.annotation.tailrec +import scala.collection.mutable.ArrayBuffer class ReaderTest extends FunSuite @@ -51,6 +53,22 @@ class ReaderTest } } + private def readAllString(r: Reader[String]): Future[String] = { + def loop(left: StringBuilder): Future[String] = { + r.read.flatMap { + case Some(right) => loop(left.append(right)) + case _ => Future.value(left.toString()) + } + } + loop(StringBuilder.newBuilder) + } + + private def writeLoop[T](from: List[T], to: Writer[T]): Future[Unit] = + from match { + case h :: t => to.write(h).flatMap(_ => writeLoop(t, to)) + case _ => to.close() + } + test("Reader.copy - source and destination equality") { forAll { (p: Array[Byte], q: Array[Byte], r: Array[Byte]) => val rw = new Pipe[Buf] @@ -283,6 +301,119 @@ class ReaderTest assert(await(r.read()).contains(Buf.Empty)) assert(await(r.read()).isEmpty) } + + test("Reader.flatMap") { + forAll { s: String => + val reader1 = Reader.fromBuf(Buf.Utf8(s), 8) + val reader2 = reader1.flatMap { buf => + val pipe = new Pipe[Buf] + pipe.write(buf).flatMap(_ => pipe.write(Buf.Empty)).flatMap(_ => pipe.close()) + pipe + } + + assert(Buf.decodeString(await(Reader.readAll(reader2)), JChar.UTF_8) == s) + } + } + + test("Reader.flatMap: discard the intermediate will discard the output reader") { + val reader1 = Reader.fromBuf(Buf.Utf8("hi"), 8) + val reader2 = reader1.flatMap { buf => + val pipe = new Pipe[Buf] + pipe.write(buf).map(_ => pipe.discard()) + pipe + } + + await(reader2.read()) + intercept[ReaderDiscardedException] { + await(reader2.read()) + } + } + + test("Reader.flatMap: discard the origin reader will discard the output reader") { + val reader1 = Reader.fromBuf(Buf.Utf8("hi"), 1) + val samples = ArrayBuffer.empty[Reader[Buf]] + val reader2 = reader1.flatMap { buf => + val pipe = new Pipe[Buf] + pipe.write(buf).map(_ => samples += pipe).flatMap(_ => pipe.close()) + pipe + } + + await(reader2.read()) + reader1.discard() + await(reader2.read()) + + intercept[ReaderDiscardedException] { + println(await(reader2.read())) + } + + assert(samples.size == 2) + assert(await(samples(0).onClose) == StreamTermination.FullyRead) + assert(await(samples(1).onClose) == StreamTermination.FullyRead) + } + + test( + "Reader.flatMap: discard the output reader will discard " + + "the origin and intermediate") { + val reader1 = Reader.fromBuf(Buf.Utf8("hello"), 1) + val samples = ArrayBuffer.empty[Reader[Buf]] + val reader2 = reader1.flatMap { buf => + val pipe = new Pipe[Buf] + pipe.write(buf).map(_ => samples += pipe).flatMap(_ => pipe.close()) + pipe + } + + await(reader2.read()) + reader2.discard() + + assert(samples.size == 2) + assert(await(samples(0).onClose) == StreamTermination.FullyRead) + assert(await(samples(1).onClose) == StreamTermination.Discarded) + + intercept[ReaderDiscardedException] { + await(reader1.read()) + } + } + + test("Reader.value") { + forAll { a: AnyVal => + val r = Reader.value(a) + assert(await(r.read()) == Some(a)) + assert(await(r.read()) == None) + } + } + + test("Reader.exception") { + forAll { ex: Exception => + val r = Reader.exception(ex) + intercept[Exception] { + await(r.read()) + } + assert(await(r.read()) == None) + } + } + + test("Reader.fromFuture") { + // read regularly + val r1 = Reader.fromFuture(Future.value(1)) + assert(await(r1.read()) == Some(1)) + assert(await(r1.read()) == None) + + // discard + val r2 = Reader.fromFuture(Future.value(2)) + r2.discard() + intercept[ReaderDiscardedException] { + await(r2.read()) + } + } + + test("Reader.map") { + forAll { l: List[Int] => + val pipe = new Pipe[Int] + writeLoop(l, pipe) + val reader2 = pipe.map(_.toString) + assert(await(readAllString(reader2)) == l.mkString) + } + } } object ReaderTest { From 2bb19f2a14e72ffe19e23b395af19824a2e609f5 Mon Sep 17 00:00:00 2001 From: Chris Thalinger Date: Thu, 20 Dec 2018 22:34:00 +0000 Subject: [PATCH 091/796] VM-3100: add a is-Graal-running JVM metric JIRA Issues: VM-3100 Differential Revision: https://phabricator.twitter.biz/D244779 --- util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 8d55fcebf2..eeb2f35084 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -49,6 +49,14 @@ object JvmStats { case _ => } + val compilerStats = stats.scope("compiler"); + gauges.add(compilerStats.addGauge("graal") { + System.getProperty("jvmci.Compiler") match { + case "graal" => 1 + case _ => 0 + } + }) + ManagementFactory.getCompilationMXBean() match { case null => case compilation => From 655af888148d3428785f6b0a0cfd657be60c23d9 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Thu, 3 Jan 2019 21:44:51 +0000 Subject: [PATCH 092/796] [util-slf4j-api] `this` is not available for static vars Problem It is not legal to call `this.getClass` for a static variable. The documentation shows illegal syntax Solution While the question of should loggers be static or not is a good question, given most prefer static loggers, so change the syntax to match. Differential Revision: https://phabricator.twitter.biz/D256892 --- util-slf4j-api/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-slf4j-api/README.md b/util-slf4j-api/README.md index 69c986d73b..543c104995 100644 --- a/util-slf4j-api/README.md +++ b/util-slf4j-api/README.md @@ -77,7 +77,7 @@ Or simply instantiate a new logger: ``` class MyJavaClass extends MyBaseScalaClass { - private static final Logger logger = new Logger(this.getClass); + private static final Logger logger = new Logger(MyJavaClass.class); ... From 63815225ef894029bf107c1c2b4e17182e515994 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 3 Jan 2019 23:05:27 +0000 Subject: [PATCH 093/796] util-core: Move Writer.FailingWriter under finagle-http Problem There is no need for a "failing" `Writer[Buf]` instance defined under until-core. Solution Move `Writer.FailingWriter` under finagle-http, where it's used. JIRA Issues: CSL-7011 Differential Revision: https://phabricator.twitter.biz/D256615 --- CHANGELOG.rst | 3 +++ .../src/main/scala/com/twitter/io/Writer.scala | 14 -------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a28d1ecafc..a7f0f1c70d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,9 @@ Breaking API Changes renamed to have unique names. This allows them to be used together with wildcard imports. See Github issue (https://github.com/twitter/util/issues/239). ``PHAB_ID=D252462`` + * util-core: Both `c.t.io.Writer.FailingWriter` and `c.t.io.Writer.fail` were removed. Build your + own instance should you need to. ``PHAB_ID=D256615`` + 18.12.0 ------- diff --git a/util-core/src/main/scala/com/twitter/io/Writer.scala b/util-core/src/main/scala/com/twitter/io/Writer.scala index 14c704d139..389aece660 100644 --- a/util-core/src/main/scala/com/twitter/io/Writer.scala +++ b/util-core/src/main/scala/com/twitter/io/Writer.scala @@ -106,20 +106,6 @@ abstract class AbstractWriter[-A] extends Writer[A] */ object Writer { - /** - * A [[Writer]] instance that will always fail. Useful for situations - * where writing to the [[Writer]] is nonsensical such as the [[Writer]] on the - * `Response` returned by the Finagle HTTP client. - */ - val FailingWriter: Writer[Buf] = fail[Buf] - - def fail[A]: Writer[A] = new Writer[A] { - def write(element: A): Future[Unit] = Future.exception(new IllegalStateException("NullWriter")) - def fail(cause: Throwable): Unit = () - def close(deadline: Time): Future[Unit] = Future.Done - def onClose: Future[StreamTermination] = Future.never - } - val BufferSize: Int = 4096 /** From 1c748332580eb9b20f20a521429e1a6d79b1db82 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Tue, 8 Jan 2019 20:13:00 +0000 Subject: [PATCH 094/796] twitter-oss: Prepare OSS libraries for release 19.1.0 Problem We want to release the next versions of our Twitter OSS libraries 19.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-7429 Differential Revision: https://phabricator.twitter.biz/D258182 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a7f0f1c70d..ac4af85cd8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.1.0 +------- + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index c6aa1f3ba7..3445d76f01 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "18.12.0" +val collUtils = "com.twitter" %% "util-collection" % "19.1.0" ``` # Units diff --git a/build.sbt b/build.sbt index 1e172cecd1..95ae2e575e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.1.0-SNAPSHOT" +val releaseVersion = "19.1.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From a49bab4db50b09f2516805cb82a318d6b0557c8c Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 9 Jan 2019 20:02:54 +0000 Subject: [PATCH 095/796] util-core: Add Reader.flatten and Reader.fromSeq Problem/Solution Added some Reader APIs. `Reader.flatten` to allow Reader concatenate directly from a Reader[[Reader[_]], and also simplifier `Reader#flatMap`. `Reader.fromSeq` to create a new Reader from a Seq. Differential Revision: https://phabricator.twitter.biz/D255424 --- CHANGELOG.rst | 2 + .../main/scala/com/twitter/io/Reader.scala | 90 +++++++++++++++- .../scala/com/twitter/io/ReaderTest.scala | 102 +++++++++++++++++- 3 files changed, 185 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ac4af85cd8..60c2a847e2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ Unreleased New Features ~~~~~~~~~~~~ + * util-core: Added `Reader.flatten` to flatten a `Reader[Reader[_]]` to `Reader[_]`, + and `Reader.fromSeq` to create a new Reader from a Seq. ``PHAB_ID=D255424`` * util-core: Added Reader.map/flatMap to transform Reader[A] to Reader[B]. Added `fromFuture()` and `value()` in the Reader object to construct a new Reader. ``PHAB_ID=D252165`` diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 3c6694b30d..64f1e6d901 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -108,7 +108,7 @@ trait Reader[+A] { self => * @param f the function constructs a new Reader[B] from the value of this Reader.read */ final def flatMap[B](f: A => Reader[B]): Reader[B] = { - val target = Reader.concat(Reader.toAsyncStream(map(f))) + val target = Reader.flatten(map(f)) target.onClose.respond { case Return(StreamTermination.Discarded) => self.discard() case _ => () @@ -125,6 +125,12 @@ trait Reader[+A] { self => def discard(): Unit = self.discard() def onClose: Future[StreamTermination] = self.onClose } + + /** + * Converts a `Reader[Reader[B]]` into a `Reader[B]` + */ + def flatten[B](implicit ev: A <:< Reader[B]): Reader[B] = + Reader.flatten(this.asInstanceOf[Reader[Reader[B]]]) } /** @@ -319,6 +325,43 @@ object Reader { */ def fromStream(s: InputStream, chunkSize: Int): Reader[Buf] = InputStreamReader(s, chunkSize) + /** + * Create a new [[Reader]] from a given [[Seq]]. + * + * The resources held by the returned [[Reader]] are released on reading of EOF and + * [[Reader.discard()]]. + */ + def fromSeq[A](seq: Seq[A]): Reader[A] = new Reader[A] { self => + private[this] val closep = Promise[StreamTermination]() + private[this] var state: Try[Seq[A]] = Return(seq) + + def read(): Future[Option[A]] = { + val result: Future[Option[A]] = self.synchronized { + state match { + case Return(Nil) => Future.None + case Return(head +: tail) => + state = Return(tail) + Future.value(Some(head)) + case t: Throw[_] => + Future.const(t.cast[Option[A]]) + } + } + if (result eq Future.None) { + closep.updateIfEmpty(StreamTermination.FullyRead.Return) + } + result + } + + def discard(): Unit = { + self.synchronized { + state = Throw(new ReaderDiscardedException) + } + closep.updateIfEmpty(StreamTermination.Discarded.Return) + } + + def onClose: Future[StreamTermination] = closep + } + /** * Allow [[AsyncStream]] to be consumed as a [[Reader]] */ @@ -343,8 +386,9 @@ object Reader { } /** - * Convenient abstraction to read from a stream of Readers as if it were a - * single Reader. + * Convenient abstraction to read from a stream (AsyncStream) of Readers as if + * it were a single Reader. + * @param readers An AsyncStream holds a stream of Reader[A] */ def concat[A](readers: AsyncStream[Reader[A]]): Reader[A] = { val target = new Pipe[A] @@ -369,6 +413,43 @@ object Reader { target } + /** + * Convenient abstraction to read from a stream (Reader) of Readers as if + * it were a single Reader. The subsequent readers are unmanaged, the caller is + * responsible for discarding those when abandoned. + * @param readers A Reader holds a stream of Reader[A] + */ + def flatten[A](readers: Reader[Reader[A]]): Reader[A] = new Reader[A] { self => + + private[this] val closep = Promise[StreamTermination]() + // access this currentReader is synchronized on `self` + private[this] var currentReader: Reader[A] = Reader.empty + + // this method should be called inside a synchronized block + def updateCurrentAndRead(): Future[Option[A]] = + readers.read().flatMap { + case Some(reader) => + currentReader = reader + read() + case None => Future.None + } + + def read(): Future[Option[A]] = self.synchronized { + currentReader.read().flatMap { + case sa @ Some(_) => Future.value(sa) + case _ => updateCurrentAndRead + } + } + + def discard(): Unit = { + closep.updateIfEmpty(StreamTermination.Discarded.Return) + currentReader.discard() + readers.discard() + } + + def onClose: Future[StreamTermination] = closep + } + /** * Copy elements from many Readers to a Writer. Readers will be discarded if * `copy` is cancelled (discarding the target). The Writer is unmanaged, the @@ -377,6 +458,7 @@ object Reader { * {{{ * Reader.copyMany(readers, writer) ensure writer.close() * }}} + * @param readers An AsyncStream holds a stream of Reader[A] */ def copyMany[A](readers: AsyncStream[Reader[A]], target: Writer[A]): Future[Unit] = readers.foreachF(Reader.copy(_, target)) @@ -394,7 +476,7 @@ object Reader { def loop(): Future[Unit] = r.read().flatMap { case None => Future.Done - case Some(buf) => w.write(buf) before loop() + case Some(elem) => w.write(elem) before loop() } w.onClose.respond { diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 38b12582b6..ff775e700b 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -30,6 +30,8 @@ class ReaderTest def undefined: AsyncStream[Reader[Buf]] = throw new Exception + def undefinedReaders: List[Reader[Buf]] = List.empty + private def assertReadWhileReading(r: Reader[Buf]): Unit = { val f = r.read() intercept[IllegalStateException] { @@ -210,6 +212,98 @@ class ReaderTest assert(!p.isDefined) } + test("Reader.flatten") { + forAll { ss: List[String] => + val readers = ss.map(s => Reader.fromBuf(Buf.Utf8(s), 16)) + val buf = Reader.readAll(Reader.flatten(Reader.fromSeq(readers))) + assert(await(buf) == Buf.Utf8(ss.mkString)) + } + } + + test("Reader#flatten") { + forAll { ss: List[String] => + val readers = ss.map(s => Reader.fromBuf(Buf.Utf8(s), 16)) + val buf = Reader.readAll((Reader.fromSeq(readers).flatten)) + assert(await(buf) == Buf.Utf8(ss.mkString)) + } + } + + test("Reader.flatten empty") { + val readers = Reader.flatten(Reader.fromSeq(undefinedReaders)) + assert(await(readers.read()) == None) + } + + test("Reader.flatten - discard") { + val p = new Promise[Option[Buf]] + + val head = new Reader[Buf] { + def read(): Future[Option[Buf]] = p + def discard(): Unit = p.setException(new ReaderDiscardedException) + def onClose: Future[StreamTermination] = Future.never + } + + val sourceReader = Reader.fromSeq(head +: undefinedReaders) + val reader = Reader.flatten(sourceReader) + reader.read() + reader.discard() + assert(p.isDefined) + assert(await(sourceReader.onClose) == StreamTermination.Discarded) + } + + test("Reader.flatten - failed") { + val p = new Promise[Option[Buf]] + val head = new Reader[Buf] { + def read(): Future[Option[Buf]] = p + def discard(): Unit = p.setException(new ReaderDiscardedException) + def onClose: Future[StreamTermination] = Future.never + } + val reader = Reader.flatten(Reader.fromSeq(head +: undefinedReaders)) + assertFailed(reader, p) + } + + test("Reader.flatten - lazy tail") { + val head = new Reader[Buf] { + def read(): Future[Option[Buf]] = Future.exception(new Exception) + + def discard(): Unit = () + def onClose: Future[StreamTermination] = Future.never + } + val p = new Promise[Unit] + + def tail: Reader[Reader[Buf]] = { + p.setDone() + Reader.empty + } + + val combined = Reader.flatten(Reader.fromSeq(head +: undefinedReaders)) + val buf = Reader.readAll(combined) + intercept[Exception] { + await(buf) + } + assert(!p.isDefined) + } + + test("Reader.fromSeq works on infinite streams") { + def ones: Stream[Int] = 1 #:: ones + val reader = Reader.fromSeq(ones) + assert(await(reader.read()) == Some(1)) + assert(await(reader.read()) == Some(1)) + assert(await(reader.read()) == Some(1)) + } + + test("Reader.flatten - concurrent read on a Reader with empty Reader prefixed") { + val p = new Promise[Int] + val reader = Reader.flatten( + Reader.fromSeq(Seq(Reader.empty, Reader.fromFuture(p), Reader.fromSeq(Seq(2, 3))))) + + val x = reader.read() + val y = reader.read() + val z = reader.read() + + p.setValue(1) + assert(Seq(Some(1), Some(2), Some(3)) == Seq(await(x), await(y), await(z))) + } + test("Reader.fromStream closes resources on EOF read") { val in = spy(new ByteArrayInputStream(arr(0, 10))) val r = Reader.fromStream(in, 4) @@ -340,15 +434,12 @@ class ReaderTest await(reader2.read()) reader1.discard() - await(reader2.read()) intercept[ReaderDiscardedException] { - println(await(reader2.read())) + await(reader2.read()) } - - assert(samples.size == 2) + assert(samples.size == 1) assert(await(samples(0).onClose) == StreamTermination.FullyRead) - assert(await(samples(1).onClose) == StreamTermination.FullyRead) } test( @@ -362,6 +453,7 @@ class ReaderTest pipe } + await(reader2.read()) await(reader2.read()) reader2.discard() From 9002c7133994d106f12130a877a527e8ca901224 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 10 Jan 2019 22:23:53 +0000 Subject: [PATCH 096/796] twitter-oss: update OSS libraries post-release to 19.2.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-7429 Differential Revision: https://phabricator.twitter.biz/D259499 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3445d76f01..67427832fa 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 95ae2e575e..25076315ba 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.1.0" +val releaseVersion = "19.2.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 136fb5416deb21674256059024cf819c70665214 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 11 Jan 2019 16:49:23 +0000 Subject: [PATCH 097/796] util: Small CHANGELOG fixes Problem / Solution Reader.flatten wasn't in by the time we merged in the pre-release script, so let's kick it to the next release. Also, the indentation was wrong in the 19.1.0 section. Differential Revision: https://phabricator.twitter.biz/D259595 --- CHANGELOG.rst | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 60c2a847e2..84833afa5a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,26 +7,30 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-core: Added `Reader.flatten` to flatten a `Reader[Reader[_]]` to `Reader[_]`, + and `Reader.fromSeq` to create a new Reader from a Seq. ``PHAB_ID=D255424`` + 19.1.0 ------- New Features ~~~~~~~~~~~~ - * util-core: Added `Reader.flatten` to flatten a `Reader[Reader[_]]` to `Reader[_]`, - and `Reader.fromSeq` to create a new Reader from a Seq. ``PHAB_ID=D255424`` - * util-core: Added Reader.map/flatMap to transform Reader[A] to Reader[B]. Added `fromFuture()` - and `value()` in the Reader object to construct a new Reader. ``PHAB_ID=D252165`` +* util-core: Added Reader.map/flatMap to transform Reader[A] to Reader[B]. Added `fromFuture()` + and `value()` in the Reader object to construct a new Reader. ``PHAB_ID=D252165`` Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ - * util-core: The implicit conversions classes in `c.t.conversions.SomethingOps` have been - renamed to have unique names. This allows them to be used together with wildcard imports. - See Github issue (https://github.com/twitter/util/issues/239). ``PHAB_ID=D252462`` +* util-core: The implicit conversions classes in `c.t.conversions.SomethingOps` have been + renamed to have unique names. This allows them to be used together with wildcard imports. + See Github issue (https://github.com/twitter/util/issues/239). ``PHAB_ID=D252462`` - * util-core: Both `c.t.io.Writer.FailingWriter` and `c.t.io.Writer.fail` were removed. Build your - own instance should you need to. ``PHAB_ID=D256615`` +* util-core: Both `c.t.io.Writer.FailingWriter` and `c.t.io.Writer.fail` were removed. Build your + own instance should you need to. ``PHAB_ID=D256615`` 18.12.0 ------- From eda0b390f3ac61fc3dd01df6255c18ef807f6920 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Mon, 14 Jan 2019 19:41:37 +0000 Subject: [PATCH 098/796] util-core: Add Duration.fromMinutes Problem There is no fromMinutes method defined in the Duration object. This forces users to use fromSeconds or other more granular methods and do the conversion by themselves. Solution Added Duration.fromMinutes and tests. JIRA Issues: CSL-7095 Differential Revision: https://phabricator.twitter.biz/D259795 --- CHANGELOG.rst | 3 +++ .../src/main/scala/com/twitter/util/Duration.scala | 1 + .../src/main/scala/com/twitter/util/Time.scala | 3 +++ .../src/test/scala/com/twitter/util/TimeTest.scala | 14 ++++++++++++-- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 84833afa5a..caaec64d9e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,9 @@ New Features * util-core: Added `Reader.flatten` to flatten a `Reader[Reader[_]]` to `Reader[_]`, and `Reader.fromSeq` to create a new Reader from a Seq. ``PHAB_ID=D255424`` +* util-core: Added `Duration.fromMinutes` to return a `Duration` from a given number of minutes. + ``PHAB_ID=D259795`` + 19.1.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index caf4ca8023..c481e2289a 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -13,6 +13,7 @@ object Duration extends TimeLikeOps[Duration] { override def fromFractionalSeconds(seconds: Double): Duration = super.fromFractionalSeconds(seconds) override def fromSeconds(seconds: Int): Duration = super.fromSeconds(seconds) + override def fromMinutes(minutes: Int): Duration = super.fromMinutes(minutes) override def fromMilliseconds(millis: Long): Duration = super.fromMilliseconds(millis) override def fromMicroseconds(micros: Long): Duration = super.fromMicroseconds(micros) diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index 89cc7614f2..fef09cc077 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -96,6 +96,8 @@ trait TimeLikeOps[This <: TimeLike[This]] { def fromSeconds(seconds: Int): This = fromMilliseconds(1000L * seconds) + def fromMinutes(minutes: Int): This = fromMilliseconds(60L * 1000L * minutes) + /** * Make a new `This` from the given number of seconds. * Because this method takes a Double, it can represent values less than a second. @@ -289,6 +291,7 @@ object Time extends TimeLikeOps[Time] { // This is needed for Java compatibility. override def fromFractionalSeconds(seconds: Double): Time = super.fromFractionalSeconds(seconds) override def fromSeconds(seconds: Int): Time = super.fromSeconds(seconds) + override def fromMinutes(minutes: Int): Time = super.fromMinutes(minutes) override def fromMilliseconds(millis: Long): Time = super.fromMilliseconds(millis) override def fromMicroseconds(micros: Long): Time = super.fromMicroseconds(micros) diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 57088b0d53..74fac0d2a1 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -118,7 +118,8 @@ trait TimeLikeSpec[T <: TimeLike[T]] extends WordSpec with GeneratorDrivenProper fromMicroseconds(0), fromFractionalSeconds(0), fromMilliseconds(0), - fromSeconds(0) + fromSeconds(0), + fromMinutes(0) )) { assert(z.isZero == true) } @@ -602,7 +603,7 @@ with IntegrationPatience { } } - "fromSeconds(Double)" in { + "fromFractionalSeconds" in { val tolerance = 2.microseconds // we permit 1us slop forAll { i: Int => @@ -663,6 +664,15 @@ with IntegrationPatience { assert(Time.fromMilliseconds(currentTimeMs).inNanoseconds == currentTimeMs * 1000000L) } + "fromMinutes" in { + assert(Time.fromMinutes(0).inNanoseconds == 0L) + assert(Time.fromMinutes(-1).inNanoseconds == -60L * 1000000000L) + + assert(Time.fromMinutes(Int.MaxValue).inNanoseconds == Long.MaxValue) + assert(Time.fromMinutes(Int.MaxValue) == Time.Top) + assert(Time.fromMinutes(Int.MinValue) == Time.Bottom) + } + "until" in { val t0 = Time.now val t1 = t0 + 10.seconds From fbcaaf4a38360bf918be70d5cc2b745285f2a7d9 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 17 Jan 2019 01:17:10 +0000 Subject: [PATCH 099/796] util-core: Simpler Reader.fromFuture and Reader.flatMap # Problem / Solution Several things could be simplified in `Reader.scala`: - `Reader.flatMap` doesn't need to propagate `discard` from `Reader.flatten` to `self`, it's already being propagated via `map`. - `Reader.fromFuture` doesn't need to check if `closep` is defined before updating it. `Promise.updateIfEmpty` result could be used instead. JIRA Issues: CSL-7011 Differential Revision: https://phabricator.twitter.biz/D261499 --- .../main/scala/com/twitter/io/Reader.scala | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 64f1e6d901..55443cd08f 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -107,14 +107,7 @@ trait Reader[+A] { self => * Construct a new Reader by applying `f` to every item read from this Reader * @param f the function constructs a new Reader[B] from the value of this Reader.read */ - final def flatMap[B](f: A => Reader[B]): Reader[B] = { - val target = Reader.flatten(map(f)) - target.onClose.respond { - case Return(StreamTermination.Discarded) => self.discard() - case _ => () - } - target - } + final def flatMap[B](f: A => Reader[B]): Reader[B] = Reader.flatten(map(f)) /** * Construct a new Reader by applying `f` to every item read from this Reader @@ -219,23 +212,21 @@ object Reader { * Construct a `Reader` from a `Future` */ def fromFuture[A](fa: Future[A]): Reader[A] = new Reader[A] { + private[this] val closep = Promise[StreamTermination]() - def read(): Future[Option[A]] = { - if (!closep.isDefined) { - closep.updateIfEmpty(StreamTermination.FullyRead.Return) - fa.map(a => Some(a)) - } else { + final def read(): Future[Option[A]] = + if (closep.updateIfEmpty(StreamTermination.FullyRead.Return)) + fa.map(Some.apply) + else closep.flatMap { case StreamTermination.FullyRead => Future.None case StreamTermination.Discarded => Future.exception(new ReaderDiscardedException) } - } - } - def discard(): Unit = closep.updateIfEmpty(StreamTermination.Discarded.Return) + final def discard(): Unit = closep.updateIfEmpty(StreamTermination.Discarded.Return) - def onClose: Future[StreamTermination] = closep + final def onClose: Future[StreamTermination] = closep } /** @@ -437,7 +428,7 @@ object Reader { def read(): Future[Option[A]] = self.synchronized { currentReader.read().flatMap { case sa @ Some(_) => Future.value(sa) - case _ => updateCurrentAndRead + case _ => updateCurrentAndRead() } } From a760898ae4df31f518133febb58226128ab0683a Mon Sep 17 00:00:00 2001 From: Win Wang Date: Wed, 23 Jan 2019 01:31:57 +0000 Subject: [PATCH 100/796] util-core: Rewrite with RscCompat to support compilation with Rsc Problem In preparation of supporting compilation of util with Rsc, util-core needs to be Rsc compatible. Solution Ascribe inferred types to visible definitions, mostly with RscCompat rule: github.com/twitter/rsc/tree/master/scalafix/rules/src/main/scala/rsc/rules Result Rsc should compile util-core with no errors. JIRA Issues: LANG-978 Differential Revision: https://phabricator.twitter.biz/D263147 --- .../scala/com/twitter/concurrent/Offer.scala | 2 +- .../com/twitter/concurrent/Scheduler.scala | 4 +-- .../scala/com/twitter/concurrent/Tx.scala | 2 +- .../com/twitter/concurrent/AsyncMeter.scala | 2 +- .../com/twitter/concurrent/AsyncQueue.scala | 2 +- .../com/twitter/concurrent/Serialized.scala | 2 +- .../scala/com/twitter/concurrent/Spool.scala | 16 ++++----- .../com/twitter/conversions/string.scala | 2 +- .../scala/com/twitter/io/BufInputStream.scala | 4 +-- .../scala/com/twitter/io/ByteReader.scala | 2 +- .../com/twitter/io/OutputStreamWriter.scala | 4 +-- .../scala/com/twitter/io/TempFolder.scala | 2 +- .../scala/com/twitter/util/Awaitable.scala | 4 +-- .../com/twitter/util/BatchExecutor.scala | 16 +++++---- .../scala/com/twitter/util/BoundedStack.scala | 6 ++-- .../scala/com/twitter/util/Cancellable.scala | 2 +- .../scala/com/twitter/util/ClosableOnce.scala | 4 +-- .../main/scala/com/twitter/util/Codec.scala | 4 +-- .../com/twitter/util/CountDownLatch.scala | 15 ++++---- .../scala/com/twitter/util/Credentials.scala | 5 +-- .../scala/com/twitter/util/Disposable.scala | 10 +++--- .../scala/com/twitter/util/Duration.scala | 2 +- .../main/scala/com/twitter/util/Event.scala | 2 +- .../com/twitter/util/JavaSingleton.scala | 2 +- .../com/twitter/util/LastWriteWinsQueue.scala | 34 +++++++++---------- .../main/scala/com/twitter/util/Monitor.scala | 2 +- .../scala/com/twitter/util/RandomSocket.scala | 2 +- .../main/scala/com/twitter/util/Signal.scala | 4 +-- .../scala/com/twitter/util/StateMachine.scala | 2 +- .../scala/com/twitter/util/StorageUnit.scala | 4 +-- .../main/scala/com/twitter/util/Time.scala | 6 ++-- .../main/scala/com/twitter/util/Timer.scala | 8 ++--- .../src/main/scala/com/twitter/util/Try.scala | 32 ++++++++--------- .../src/main/scala/com/twitter/util/Var.scala | 14 ++++---- .../concurrent/AsyncSemaphoreTest.scala | 5 +-- .../twitter/concurrent/AsyncStreamTest.scala | 6 ++-- .../com/twitter/concurrent/OfferTest.scala | 2 +- .../twitter/concurrent/SchedulerTest.scala | 2 +- .../test/scala/com/twitter/io/BufTest.scala | 3 +- .../scala/com/twitter/io/ByteReaderTest.scala | 2 +- .../twitter/io/InputStreamReaderTest.scala | 4 +-- .../twitter/io/exp/ActivitySourceTest.scala | 6 ++-- .../com/twitter/util/CloseAwaitablyTest.scala | 4 +-- .../scala/com/twitter/util/DurationTest.scala | 2 +- .../com/twitter/util/FuturePoolTest.scala | 8 ++--- .../scala/com/twitter/util/PromiseTest.scala | 4 +-- .../scala/com/twitter/util/TimeTest.scala | 2 +- .../test/scala/com/twitter/util/TryTest.scala | 2 +- 48 files changed, 141 insertions(+), 135 deletions(-) diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala index 97d324559d..0ccc91fd73 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala @@ -312,6 +312,6 @@ object Offer { } object LostSynchronization extends Exception { - override def fillInStackTrace = this + override def fillInStackTrace: LostSynchronization.type = this } } diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala index 2eecdb736f..1b3f716d9f 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala @@ -138,8 +138,8 @@ private[concurrent] object LocalScheduler { private[this] val rs = new ArrayDeque[Runnable] private[this] var running = false - @volatile var numDispatches = 0L - @volatile var blockingNanos = 0L + @volatile var numDispatches: Long = 0L + @volatile var blockingNanos: Long = 0L override def toString: String = s"Activation(${System.identityHashCode(this)})" diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala index 9cc36fc74e..82ed9cd20d 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala @@ -65,7 +65,7 @@ object Tx { /** * A constant `Tx` with the value of `Unit`. */ - val Unit = const(()) + val Unit: Tx[Unit] = const(()) object AlreadyDone extends Exception("Tx is already done") object AlreadyAckd extends Exception("Tx was already ackd") diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala index 1d1d45db67..58e1e79799 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala @@ -24,7 +24,7 @@ private[concurrent] object Period { } object AsyncMeter { - private[concurrent] val MinimumInterval = Duration.fromMilliseconds(1) + private[concurrent] val MinimumInterval: Duration = Duration.fromMilliseconds(1) /** * Creates an [[AsyncMeter]] that allows smoothed out `permits` per diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala index 865848c154..03f37a2bba 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala @@ -203,5 +203,5 @@ class AsyncQueue[T](maxPendingOffers: Int) { } } - override def toString = s"AsyncQueue<${synchronized(state)}>" + override def toString: String = s"AsyncQueue<${synchronized(state)}>" } diff --git a/util-core/src/main/scala/com/twitter/concurrent/Serialized.scala b/util-core/src/main/scala/com/twitter/concurrent/Serialized.scala index a9f6ac22a4..4849e9b987 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Serialized.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Serialized.scala @@ -19,7 +19,7 @@ trait Serialized { } } - private[this] val nwaiters = new AtomicInteger(0) + private[this] val nwaiters: AtomicInteger = new AtomicInteger(0) protected val serializedQueue: java.util.Queue[Job[_]] = new ConcurrentLinkedQueue[Job[_]] protected def serialized[A](f: => A): Future[A] = { diff --git a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala index 6bc3a160d9..3f8ada588f 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala @@ -67,7 +67,7 @@ sealed trait Spool[+A] { * Apply {{f}} for each item in the spool, until the end. {{f}} is * applied as the items become available. */ - def foreach[B](f: A => B) = foreachElem(_ foreach f) + def foreach[B](f: A => B): Future[Unit] = foreachElem(_ foreach f) /** * A version of {{foreach}} that wraps each element in an @@ -291,8 +291,8 @@ abstract class AbstractSpool[A] extends Spool[A] */ object Spool { case class Cons[A](head: A, tail: Future[Spool[A]]) extends Spool[A] { - def isEmpty = false - override def toString = "Cons(%s, %c)".format(head, if (tail.isDefined) '*' else '?') + def isEmpty: Boolean = false + override def toString: String = "Cons(%s, %c)".format(head, if (tail.isDefined) '*' else '?') } private class LazyCons[A](val head: A, next: => Future[Spool[A]]) extends Spool[A] { @@ -303,10 +303,10 @@ object Spool { } object Empty extends Spool[Nothing] { - def isEmpty = true - def head = throw new NoSuchElementException("spool is empty") - def tail = Future.exception(new NoSuchElementException("spool is empty")) - override def toString = "Empty" + def isEmpty: Boolean = true + def head: Nothing = throw new NoSuchElementException("spool is empty") + def tail: Future[Nothing] = Future.exception(new NoSuchElementException("spool is empty")) + override def toString: String = "Empty" } /** @@ -352,7 +352,7 @@ object Spool { * @deprecated Deprecated in favor of {{*::}}. This will eventually be removed. */ @deprecated("Use *:: instead.", "6.14.1") - def **::(head: A) = cons(head, tail) + def **::(head: A): Spool[A] = cons(head, tail) } implicit def syntax1[A](s: Spool[A]): Syntax1[A] = new Syntax1[A](s) diff --git a/util-core/src/main/scala/com/twitter/conversions/string.scala b/util-core/src/main/scala/com/twitter/conversions/string.scala index 2e15ee3fb3..7496726d0f 100644 --- a/util-core/src/main/scala/com/twitter/conversions/string.scala +++ b/util-core/src/main/scala/com/twitter/conversions/string.scala @@ -106,7 +106,7 @@ object string { * * @return an unquoted unicode string */ - def unquoteC() = { + def unquoteC(): String = { regexSub(UNQUOTE_RE) { m => val ch = m.group(1).charAt(0) match { // holy crap! this is terrible: diff --git a/util-core/src/main/scala/com/twitter/io/BufInputStream.scala b/util-core/src/main/scala/com/twitter/io/BufInputStream.scala index 25f3af3c7e..86a6b8e241 100644 --- a/util-core/src/main/scala/com/twitter/io/BufInputStream.scala +++ b/util-core/src/main/scala/com/twitter/io/BufInputStream.scala @@ -25,7 +25,7 @@ class BufInputStream(val buf: Buf) extends InputStream { override def close(): Unit = {} // Marks the current position in this input stream. - override def mark(readlimit: Int) = synchronized { mrk = index } + override def mark(readlimit: Int): Unit = synchronized { mrk = index } // Tests if this input stream supports the mark and reset methods. override def markSupported(): Boolean = true @@ -68,7 +68,7 @@ class BufInputStream(val buf: Buf) extends InputStream { * Repositions this stream to the position at the time the mark * method was last called on this input stream. */ - override def reset() = synchronized { index = mrk } + override def reset(): Unit = synchronized { index = mrk } /** * Skips over and discards n bytes of data from this input stream. diff --git a/util-core/src/main/scala/com/twitter/io/ByteReader.scala b/util-core/src/main/scala/com/twitter/io/ByteReader.scala index 0ffe75177a..3d71044d35 100644 --- a/util-core/src/main/scala/com/twitter/io/ByteReader.scala +++ b/util-core/src/main/scala/com/twitter/io/ByteReader.scala @@ -290,7 +290,7 @@ object ByteReader { /** * The max value of a signed 24 bit "medium" integer */ - private[io] val SignedMediumMax = 0x800000 + private[io] val SignedMediumMax: Int = 0x800000 } private class ByteReaderImpl(buf: Buf) extends ByteReader { diff --git a/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala b/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala index dc70ccdb65..f41400e5f2 100644 --- a/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala +++ b/util-core/src/main/scala/com/twitter/io/OutputStreamWriter.scala @@ -70,6 +70,6 @@ private[io] class OutputStreamWriter(out: OutputStream, bufsize: Int) extends Wr } private object OutputStreamWriter { - val WriteExc = new IllegalStateException("write while writing") - val CloseExc = new IllegalStateException("write after close") + val WriteExc: IllegalStateException = new IllegalStateException("write while writing") + val CloseExc: IllegalStateException = new IllegalStateException("write after close") } diff --git a/util-core/src/main/scala/com/twitter/io/TempFolder.scala b/util-core/src/main/scala/com/twitter/io/TempFolder.scala index ceb977ca9b..56aecde891 100644 --- a/util-core/src/main/scala/com/twitter/io/TempFolder.scala +++ b/util-core/src/main/scala/com/twitter/io/TempFolder.scala @@ -26,7 +26,7 @@ import java.io.File * is effectively a thread-local global. */ trait TempFolder { - private val _folderName = new ThreadLocal[File] + private val _folderName: ThreadLocal[File] = new ThreadLocal[File] /** * Runs the given block of code with the presence of a temporary folder whose name can be diff --git a/util-core/src/main/scala/com/twitter/util/Awaitable.scala b/util-core/src/main/scala/com/twitter/util/Awaitable.scala index b0089e0854..c6deb99642 100644 --- a/util-core/src/main/scala/com/twitter/util/Awaitable.scala +++ b/util-core/src/main/scala/com/twitter/util/Awaitable.scala @@ -188,8 +188,8 @@ object Await { // See http://stackoverflow.com/questions/26643045/java-interoperability-woes-with-scala-generics-and-boxing private[util] trait CloseAwaitably0[U <: Unit] extends Awaitable[U] { - private[this] val onClose = new Promise[U] - private[this] val closed = new AtomicBoolean(false) + private[this] val onClose: Promise[U] = new Promise[U] + private[this] val closed: AtomicBoolean = new AtomicBoolean(false) /** * closeAwaitably is intended to be used as a wrapper for diff --git a/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala b/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala index 8e1e1f56e0..601781523a 100644 --- a/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala +++ b/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala @@ -4,6 +4,7 @@ import java.util.concurrent.CancellationException import java.util.logging.Logger import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer /** * A "batcher" that takes a function `Seq[In] => Future[Seq[Out]]` and @@ -36,8 +37,8 @@ private[util] class BatchExecutor[In, Out]( import java.util.logging.Level.WARNING class ScheduledFlush(after: Duration, timer: Timer) { - @volatile var cancelled = false - val task = timer.schedule(after.fromNow) { flush() } + @volatile var cancelled: Boolean = false + val task: TimerTask = timer.schedule(after.fromNow) { flush() } def cancel(): Unit = { cancelled = true @@ -55,14 +56,15 @@ private[util] class BatchExecutor[In, Out]( } } - val log = Logger.getLogger("Future.batched") + val log: Logger = Logger.getLogger("Future.batched") // operations on these are synchronized on `this`. - val buf = new mutable.ArrayBuffer[(In, Promise[Out])](sizeThreshold) + val buf: ArrayBuffer[Tuple2[In, Promise[Out]]] = + new mutable.ArrayBuffer[(In, Promise[Out])](sizeThreshold) var scheduled: Option[ScheduledFlush] = scala.None - var currentBufThreshold = newBufThreshold + var currentBufThreshold: Int = newBufThreshold - def currentBufPercentile = sizePercentile match { + def currentBufPercentile: Float = sizePercentile match { case tooHigh if tooHigh > 1.0f => log.log(WARNING, "value returned for sizePercentile (%f) was > 1.0f, using 1.0", tooHigh) 1.0f @@ -74,7 +76,7 @@ private[util] class BatchExecutor[In, Out]( case p => p } - def newBufThreshold = + def newBufThreshold: Int = math.round(currentBufPercentile * sizeThreshold) match { case tooLow if tooLow < 1 => 1 case size => math.min(size, sizeThreshold) diff --git a/util-core/src/main/scala/com/twitter/util/BoundedStack.scala b/util-core/src/main/scala/com/twitter/util/BoundedStack.scala index 38f8fe7ed6..ba309144a1 100644 --- a/util-core/src/main/scala/com/twitter/util/BoundedStack.scala +++ b/util-core/src/main/scala/com/twitter/util/BoundedStack.scala @@ -11,8 +11,8 @@ class BoundedStack[A: ClassTag](val maxSize: Int) extends Seq[A] { private var top = 0 private var count_ = 0 - def length = count_ - override def size = count_ + def length: Int = count_ + override def size: Int = count_ def clear(): Unit = { top = 0 @@ -84,7 +84,7 @@ class BoundedStack[A: ClassTag](val maxSize: Int) extends Seq[A] { } } - override def iterator = new Iterator[A] { + override def iterator: Iterator[A] = new Iterator[A] { var idx = 0 def hasNext = idx != count_ def next = { diff --git a/util-core/src/main/scala/com/twitter/util/Cancellable.scala b/util-core/src/main/scala/com/twitter/util/Cancellable.scala index a1fffb9f12..04d0114ec3 100644 --- a/util-core/src/main/scala/com/twitter/util/Cancellable.scala +++ b/util-core/src/main/scala/com/twitter/util/Cancellable.scala @@ -38,7 +38,7 @@ object Cancellable { class CancellableSink(f: => Unit) extends Cancellable { private[this] val wasCancelled = new AtomicBoolean(false) - def isCancelled = wasCancelled.get + def isCancelled: Boolean = wasCancelled.get def cancel(): Unit = { if (wasCancelled.compareAndSet(false, true)) f } def linkTo(other: Cancellable): Unit = { throw new Exception("linking not supported in CancellableSink") diff --git a/util-core/src/main/scala/com/twitter/util/ClosableOnce.scala b/util-core/src/main/scala/com/twitter/util/ClosableOnce.scala index 8ea5b0e216..8e5b9f6927 100644 --- a/util-core/src/main/scala/com/twitter/util/ClosableOnce.scala +++ b/util-core/src/main/scala/com/twitter/util/ClosableOnce.scala @@ -12,8 +12,8 @@ import scala.util.control.NonFatal trait ClosableOnce extends Closable { // Our intrinsic lock for mutating the `closed` field - private[this] val closePromise = Promise[Unit]() - private[this] var closed = false + private[this] val closePromise: Promise[Unit] = Promise[Unit]() + private[this] var closed: Boolean = false private[this] def once(): Boolean = closePromise.synchronized { if (closed) { diff --git a/util-core/src/main/scala/com/twitter/util/Codec.scala b/util-core/src/main/scala/com/twitter/util/Codec.scala index 48d5bac5ac..71ee2bc205 100644 --- a/util-core/src/main/scala/com/twitter/util/Codec.scala +++ b/util-core/src/main/scala/com/twitter/util/Codec.scala @@ -25,7 +25,7 @@ trait Encoder[T, S] extends (T => S) { */ def encode(t: T): S - def apply(t: T) = encode(t) + def apply(t: T): S = encode(t) } trait DecoderCompanion { @@ -51,7 +51,7 @@ trait Decoder[T, S] { */ def decode(s: S): T - def invert(s: S) = decode(s) + def invert(s: S): T = decode(s) } object Codec extends EncoderCompanion with DecoderCompanion { diff --git a/util-core/src/main/scala/com/twitter/util/CountDownLatch.scala b/util-core/src/main/scala/com/twitter/util/CountDownLatch.scala index 6e885ee5af..45f9da9725 100644 --- a/util-core/src/main/scala/com/twitter/util/CountDownLatch.scala +++ b/util-core/src/main/scala/com/twitter/util/CountDownLatch.scala @@ -3,13 +3,14 @@ package com.twitter.util import java.util.concurrent.TimeUnit class CountDownLatch(val initialCount: Int) { - val underlying = new java.util.concurrent.CountDownLatch(initialCount) - def count = underlying.getCount - def isZero = count == 0 - def countDown() = underlying.countDown() - def await() = underlying.await() - def await(timeout: Duration) = underlying.await(timeout.inMillis, TimeUnit.MILLISECONDS) - def within(timeout: Duration) = await(timeout) || { + val underlying: java.util.concurrent.CountDownLatch = + new java.util.concurrent.CountDownLatch(initialCount) + def count: Long = underlying.getCount + def isZero: Boolean = count == 0 + def countDown(): Unit = underlying.countDown() + def await(): Unit = underlying.await() + def await(timeout: Duration): Boolean = underlying.await(timeout.inMillis, TimeUnit.MILLISECONDS) + def within(timeout: Duration): Boolean = await(timeout) || { throw new TimeoutException(timeout.toString) } } diff --git a/util-core/src/main/scala/com/twitter/util/Credentials.scala b/util-core/src/main/scala/com/twitter/util/Credentials.scala index c535164516..437a5c7c85 100644 --- a/util-core/src/main/scala/com/twitter/util/Credentials.scala +++ b/util-core/src/main/scala/com/twitter/util/Credentials.scala @@ -21,6 +21,7 @@ import java.io.{File, IOException} import scala.collection.JavaConverters._ import scala.io.Source import scala.util.parsing.combinator._ +import scala.util.matching.Regex /** * Simple helper to read authentication credentials from a text file. @@ -31,12 +32,12 @@ import scala.util.parsing.combinator._ object Credentials { object parser extends RegexParsers { - override val whiteSpace = "(?:\\s+|#.*\\r?\\n)+".r + override val whiteSpace: Regex = "(?:\\s+|#.*\\r?\\n)+".r private[this] val key = "[\\w-]+".r private[this] val value = ".+".r - def auth = key ~ ":" ~ value ^^ { case k ~ ":" ~ v => (k, v) } + def auth: Parser[Tuple2[String, String]] = key ~ ":" ~ value ^^ { case k ~ ":" ~ v => (k, v) } def content: Parser[Map[String, String]] = rep(auth) ^^ { auths => Map(auths: _*) } diff --git a/util-core/src/main/scala/com/twitter/util/Disposable.scala b/util-core/src/main/scala/com/twitter/util/Disposable.scala index b4cc8ef7e3..9c3679657e 100644 --- a/util-core/src/main/scala/com/twitter/util/Disposable.scala +++ b/util-core/src/main/scala/com/twitter/util/Disposable.scala @@ -66,7 +66,7 @@ trait Disposable[+T] { } object Disposable { - def const[T](t: T) = new Disposable[T] { + def const[T](t: T): Disposable[T] = new Disposable[T] { def get = t def dispose(deadline: Time) = Future.value(()) } @@ -131,13 +131,13 @@ trait Managed[+T] { selfT => } object Managed { - def singleton[T](t: Disposable[T]) = new Managed[T] { def make() = t } - def const[T](t: T) = singleton(Disposable.const(t)) + def singleton[T](t: Disposable[T]): Managed[T] = new Managed[T] { def make() = t } + def const[T](t: T): Managed[T] = singleton(Disposable.const(t)) } class DoubleTrouble(cause1: Throwable, cause2: Throwable) extends Exception { - override def getStackTrace = cause1.getStackTrace - override def getMessage = + override def getStackTrace: Array[StackTraceElement] = cause1.getStackTrace + override def getMessage: String = "Double failure while disposing composite resource: %s \n %s".format( cause1.getMessage, cause2.getMessage diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index c481e2289a..64d27090d9 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -305,7 +305,7 @@ private[util] object DurationBox { * that are truly infinite; for example the absence of a timeout. */ sealed class Duration private[util] (protected val nanos: Long) extends { - protected val ops = Duration + protected val ops: Duration.type = Duration } with TimeLike[Duration] with Serializable { import ops._ diff --git a/util-core/src/main/scala/com/twitter/util/Event.scala b/util-core/src/main/scala/com/twitter/util/Event.scala index 5683afc558..ca89f2824a 100644 --- a/util-core/src/main/scala/com/twitter/util/Event.scala +++ b/util-core/src/main/scala/com/twitter/util/Event.scala @@ -263,7 +263,7 @@ trait Event[+T] { self => * builder. A value containing the current version of the collection * is notified for each incoming event. */ - def build[U >: T, That](implicit cbf: CanBuild[U, That]) = new Event[That] { + def build[U >: T, That](implicit cbf: CanBuild[U, That]): Event[That] = new Event[That] { def register(s: Witness[That]): Closable = { val b = cbf() self.respond { t => diff --git a/util-core/src/main/scala/com/twitter/util/JavaSingleton.scala b/util-core/src/main/scala/com/twitter/util/JavaSingleton.scala index 4b7abeef5c..af5c55b01b 100644 --- a/util-core/src/main/scala/com/twitter/util/JavaSingleton.scala +++ b/util-core/src/main/scala/com/twitter/util/JavaSingleton.scala @@ -4,5 +4,5 @@ package com.twitter.util * A mixin to allow scala objects to be used from java. */ trait JavaSingleton { - def get = this + def get: JavaSingleton = this } diff --git a/util-core/src/main/scala/com/twitter/util/LastWriteWinsQueue.scala b/util-core/src/main/scala/com/twitter/util/LastWriteWinsQueue.scala index 13f4d93a89..405f1bc8c8 100644 --- a/util-core/src/main/scala/com/twitter/util/LastWriteWinsQueue.scala +++ b/util-core/src/main/scala/com/twitter/util/LastWriteWinsQueue.scala @@ -7,22 +7,22 @@ import java.util.concurrent.atomic.AtomicReference * When the Queue is full, a push replaces the item. */ class LastWriteWinsQueue[A] extends java.util.Queue[A] { - val item = new AtomicReference[Option[A]](None) + val item: AtomicReference[Option[A]] = new AtomicReference[Option[A]](None) def clear(): Unit = { item.set(None) } - def retainAll(p1: Collection[_]) = throw new UnsupportedOperationException + def retainAll(p1: Collection[_]): Nothing = throw new UnsupportedOperationException - def removeAll(p1: Collection[_]) = throw new UnsupportedOperationException + def removeAll(p1: Collection[_]): Nothing = throw new UnsupportedOperationException - def addAll(p1: Collection[_ <: A]) = throw new UnsupportedOperationException + def addAll(p1: Collection[_ <: A]): Nothing = throw new UnsupportedOperationException - def containsAll(p1: Collection[_]) = + def containsAll(p1: Collection[_]): Boolean = p1.size == 1 && item.get == p1.iterator.next() - def remove(candidate: AnyRef) = { + def remove(candidate: AnyRef): Boolean = { val contained = item.get val containsCandidate = contained.map(_ == candidate).getOrElse(false) if (containsCandidate) { @@ -41,30 +41,30 @@ class LastWriteWinsQueue[A] extends java.util.Queue[A] { } else Array[Any]().asInstanceOf[Array[T with java.lang.Object]] } - def toArray = toArray(new Array[AnyRef](0)) + def toArray: Array[AnyRef with Object] = toArray(new Array[AnyRef](0)) - def iterator = null + def iterator: Null = null - def contains(p1: AnyRef) = false + def contains(p1: AnyRef): Boolean = false - def isEmpty = item.get.isDefined + def isEmpty: Boolean = item.get.isDefined - def size = if (item.get.isDefined) 1 else 0 + def size: Int = if (item.get.isDefined) 1 else 0 - def peek = item.get.getOrElse(null.asInstanceOf[A]) + def peek: A = item.get.getOrElse(null.asInstanceOf[A]) - def element = item.get.getOrElse(throw new NoSuchElementException) + def element: A = item.get.getOrElse(throw new NoSuchElementException) - def poll = item.getAndSet(None).getOrElse(null.asInstanceOf[A]) + def poll: A = item.getAndSet(None).getOrElse(null.asInstanceOf[A]) - def remove = item.getAndSet(None).getOrElse(throw new NoSuchElementException) + def remove: A = item.getAndSet(None).getOrElse(throw new NoSuchElementException) - def offer(p1: A) = { + def offer(p1: A): Boolean = { item.set(Some(p1)) true } - def add(p1: A) = { + def add(p1: A): Boolean = { item.set(Some(p1)) true } diff --git a/util-core/src/main/scala/com/twitter/util/Monitor.scala b/util-core/src/main/scala/com/twitter/util/Monitor.scala index ad8e02251f..be01c45175 100644 --- a/util-core/src/main/scala/com/twitter/util/Monitor.scala +++ b/util-core/src/main/scala/com/twitter/util/Monitor.scala @@ -31,7 +31,7 @@ trait Monitor { self => */ def handle(exc: Throwable): Boolean - private[this] val someSelf = Some(self) + private[this] val someSelf: Some[Monitor] = Some(self) /** * Run `f` inside of the monitor context. If `f` throws diff --git a/util-core/src/main/scala/com/twitter/util/RandomSocket.scala b/util-core/src/main/scala/com/twitter/util/RandomSocket.scala index 33ec22ff8f..2cb02c5355 100644 --- a/util-core/src/main/scala/com/twitter/util/RandomSocket.scala +++ b/util-core/src/main/scala/com/twitter/util/RandomSocket.scala @@ -14,7 +14,7 @@ object RandomSocket { private[this] val ephemeralSocketAddress = localSocketOnPort(0) @deprecated("RandomSocket cannot ensure that the address is not in use.", "2014-11-13") - def apply() = nextAddress() + def apply(): InetSocketAddress = nextAddress() @deprecated("RandomSocket cannot ensure that the address is not in use.", "2014-11-13") def nextAddress(): InetSocketAddress = diff --git a/util-core/src/main/scala/com/twitter/util/Signal.scala b/util-core/src/main/scala/com/twitter/util/Signal.scala index cfe25569f0..a0657df184 100644 --- a/util-core/src/main/scala/com/twitter/util/Signal.scala +++ b/util-core/src/main/scala/com/twitter/util/Signal.scala @@ -21,7 +21,7 @@ import java.lang.reflect.{InvocationHandler, Method, Proxy} import scala.collection.{Map, Set, mutable} object SignalHandlerFactory { - def apply() = { + def apply(): Option[SunSignalHandler] = { // only one actual implementation for now SunSignalHandler.instantiate } @@ -32,7 +32,7 @@ trait SignalHandler { } object SunSignalHandler { - def instantiate() = { + def instantiate(): Option[SunSignalHandler] = { try { Class.forName("sun.misc.Signal") Some(new SunSignalHandler()) diff --git a/util-core/src/main/scala/com/twitter/util/StateMachine.scala b/util-core/src/main/scala/com/twitter/util/StateMachine.scala index f138baa9e1..434f87fa95 100644 --- a/util-core/src/main/scala/com/twitter/util/StateMachine.scala +++ b/util-core/src/main/scala/com/twitter/util/StateMachine.scala @@ -11,7 +11,7 @@ trait StateMachine { protected abstract class State protected var state: State = _ - protected def transition[A](command: String)(f: PartialFunction[State, A]) = synchronized { + protected def transition[A](command: String)(f: PartialFunction[State, A]): A = synchronized { if (f.isDefinedAt(state)) { f(state) } else { diff --git a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala index 1c608a23a8..668643e0d9 100644 --- a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala +++ b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala @@ -31,8 +31,8 @@ object StorageUnit { def fromExabytes(exabytes: Long): StorageUnit = new StorageUnit(exabytes * 1024 * 1024 * 1024 * 1024 * 1024 * 1024) - val infinite = new StorageUnit(Long.MaxValue) - val zero = new StorageUnit(0) + val infinite: StorageUnit = new StorageUnit(Long.MaxValue) + val zero: StorageUnit = new StorageUnit(0) private def factor(s: String): Long = { var lower = s.toLowerCase diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index fef09cc077..bb2b54bb63 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -404,8 +404,8 @@ object Time extends TimeLikeOps[Time] { /** * Note, this should only ever be updated by methods used for testing. */ - private[util] val localGetTime = new Local[() => Time] - private[util] val localGetTimer = new Local[MockTimer] + private[util] val localGetTime: Local[scala.Function0[Time]] = new Local[() => Time] + private[util] val localGetTimer: Local[MockTimer] = new Local[MockTimer] /** * Creates a [[Time]] instance of the given [[Date]]. @@ -553,7 +553,7 @@ class TimeFormat( * @see [[TimeFormat]] for converting to and from `String` representations. */ sealed class Time private[util] (protected val nanos: Long) extends { - protected val ops = Time + protected val ops: Time.type = Time } with TimeLike[Time] with Serializable { import ops._ diff --git a/util-core/src/main/scala/com/twitter/util/Timer.scala b/util-core/src/main/scala/com/twitter/util/Timer.scala index 2e1d1d8bbf..124f1af3fc 100644 --- a/util-core/src/main/scala/com/twitter/util/Timer.scala +++ b/util-core/src/main/scala/com/twitter/util/Timer.scala @@ -347,7 +347,7 @@ class MockTimer extends Timer { // These are weird semantics admittedly, but there may // be a bunch of tests that rely on them already. case class Task(var when: Time, runner: () => Unit) extends TimerTask { - var isCancelled = false + var isCancelled: Boolean = false def cancel(): Unit = MockTimer.this.synchronized { isCancelled = true @@ -357,9 +357,9 @@ class MockTimer extends Timer { } } - var isStopped = false - var tasks = ArrayBuffer[Task]() - var nCancelled = 0 + var isStopped: Boolean = false + var tasks: ArrayBuffer[Task] = ArrayBuffer[Task]() + var nCancelled: Int = 0 def tick(): Unit = synchronized { if (isStopped) diff --git a/util-core/src/main/scala/com/twitter/util/Try.scala b/util-core/src/main/scala/com/twitter/util/Try.scala index 5c39fb471e..883e881a15 100644 --- a/util-core/src/main/scala/com/twitter/util/Try.scala +++ b/util-core/src/main/scala/com/twitter/util/Try.scala @@ -128,7 +128,7 @@ sealed abstract class Try[+R] { /** * Returns the value from this Return or the given argument if this is a Throw. */ - def getOrElse[R2 >: R](default: => R2) = if (isReturn) apply() else default + def getOrElse[R2 >: R](default: => R2): R2 = if (isReturn) apply() else default /** * Returns the value from this Return or throws the exception if this is a Throw @@ -139,7 +139,7 @@ sealed abstract class Try[+R] { * Returns the value from this Return or throws the exception if this is a Throw. * Alias for apply() */ - def get() = apply() + def get(): R = apply() /** * Applies the given function f if this is a Result. @@ -214,7 +214,7 @@ sealed abstract class Try[+R] { /** * Returns None if this is a Throw or a Some containing the value if this is a Return */ - def toOption = if (isReturn) Some(apply()) else None + def toOption: Option[R] = if (isReturn) Some(apply()) else None /** * Invokes the given closure when the value is available. Returns @@ -236,7 +236,7 @@ sealed abstract class Try[+R] { * Returns the given function applied to the value from this Return or returns this if this is a Throw. * Alias for flatMap */ - def andThen[R2](f: R => Try[R2]) = flatMap(f) + def andThen[R2](f: R => Try[R2]): Try[R2] = flatMap(f) def flatten[T](implicit ev: R <:< Try[T]): Try[T] @@ -249,10 +249,10 @@ object Throw { final case class Throw[+R](e: Throwable) extends Try[R] { def asScala: scala.util.Try[R] = Failure(e) - def isThrow = true - def isReturn = false + def isThrow: Boolean = true + def isReturn: Boolean = false def throwable: Throwable = e - def rescue[R2 >: R](rescueException: PartialFunction[Throwable, Try[R2]]) = { + def rescue[R2 >: R](rescueException: PartialFunction[Throwable, Try[R2]]): Try[R2] = { try { val result = rescueException.applyOrElse(e, Throw.AlwaysNotApplied) if (result eq Throw.NotApplied) this else result @@ -261,16 +261,16 @@ final case class Throw[+R](e: Throwable) extends Try[R] { } } def apply(): R = throw e - def flatMap[R2](f: R => Try[R2]) = this.asInstanceOf[Throw[R2]] + def flatMap[R2](f: R => Try[R2]): Throw[R2] = this.asInstanceOf[Throw[R2]] def flatten[T](implicit ev: R <:< Try[T]): Try[T] = this.asInstanceOf[Throw[T]] def map[X](f: R => X): Try[X] = this.asInstanceOf[Throw[X]] def cast[X]: Try[X] = this.asInstanceOf[Throw[X]] - def exists(p: R => Boolean) = false - def filter(p: R => Boolean) = this - def withFilter(p: R => Boolean) = this - def onFailure(rescueException: Throwable => Unit) = { rescueException(e); this } - def onSuccess(f: R => Unit) = this - def handle[R2 >: R](rescueException: PartialFunction[Throwable, R2]) = + def exists(p: R => Boolean): Boolean = false + def filter(p: R => Boolean): Throw[R] = this + def withFilter(p: R => Boolean): Throw[R] = this + def onFailure(rescueException: Throwable => Unit): Throw[R] = { rescueException(e); this } + def onSuccess(f: R => Unit): Throw[R] = this + def handle[R2 >: R](rescueException: PartialFunction[Throwable, R2]): Try[R2] = if (rescueException.isDefinedAt(e)) { Try(rescueException(e)) } else { @@ -279,8 +279,8 @@ final case class Throw[+R](e: Throwable) extends Try[R] { } object Return { - val Unit = Return(()) - val Void = Return[Void](null) + val Unit: Return[Unit] = Return(()) + val Void: Return[Void] = Return[Void](null) val None: Return[Option[Nothing]] = Return(Option.empty) val Nil: Return[Seq[Nothing]] = Return(Seq.empty) val True: Return[Boolean] = Return(true) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 134d661f94..5a6fae9024 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -159,7 +159,7 @@ object Var { } private[util] object Observer { - def apply[T](k: T => Unit) = new Observer(k) + def apply[T](k: T => Unit): Observer[T] = new Observer(k) } /** @@ -433,16 +433,16 @@ private object UpdatableVar { import Var.Observer case class Party[T](obs: Observer[T], depth: Int, n: Long) { - @volatile var active = true + @volatile var active: Boolean = true } case class State[T](value: T, version: Long, parties: immutable.SortedSet[Party[T]]) { - def -(p: Party[T]) = copy(parties = parties - p) - def +(p: Party[T]) = copy(parties = parties + p) - def :=(newv: T) = copy(value = newv, version = version + 1) + def -(p: Party[T]): State[T] = copy(parties = parties - p) + def +(p: Party[T]): State[T] = copy(parties = parties + p) + def :=(newv: T): State[T] = copy(value = newv, version = version + 1) } - implicit def order[T] = new Ordering[Party[T]] { + implicit def order[T]: Ordering[Party[T]] = new Ordering[Party[T]] { // This is safe because observers are compared // only from the same counter. def compare(a: Party[T], b: Party[T]): Int = { @@ -495,7 +495,7 @@ private[util] class UpdatableVar[T](init: T) extends Var[T] with Updatable[T] wi } } - override def toString = "Var(" + state.get.value + ")@" + hashCode + override def toString: String = "Var(" + state.get.value + ")@" + hashCode } /** diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index 3a44f6493e..6cf21bda0c 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -7,6 +7,7 @@ import org.junit.runner.RunWith import org.scalatest.fixture.FunSpec import org.scalatest.junit.JUnitRunner import scala.collection.mutable +import org.scalatest.Outcome @RunWith(classOf[JUnitRunner]) class AsyncSemaphoreTest extends FunSpec { @@ -18,7 +19,7 @@ class AsyncSemaphoreTest extends FunSpec { sem: AsyncSemaphore = this.sem, count: Int = this.count, permits: ConcurrentLinkedQueue[Permit] = this.permits - ) = + ): AsyncSemaphoreHelper = new AsyncSemaphoreHelper(sem, count, permits) } @@ -26,7 +27,7 @@ class AsyncSemaphoreTest extends FunSpec { private def await[T](t: Awaitable[T]): T = Await.result(t, 15.seconds) - override def withFixture(test: OneArgTest) = { + override def withFixture(test: OneArgTest): Outcome = { val sem = new AsyncSemaphore(2) val helper = new AsyncSemaphoreHelper(sem, 0, new ConcurrentLinkedQueue[Permit]) withFixture(test.toNoArgTest(helper)) diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 3c64ad627a..ac2ba71537 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -743,18 +743,18 @@ class AsyncStreamTest extends FunSuite with GeneratorDrivenPropertyChecks { } private object AsyncStreamTest { - val genListAndN = for { + val genListAndN: Gen[Tuple2[List[Int], Int]] = for { as <- Arbitrary.arbitrary[List[Int]] n <- Gen.choose(0, as.length) } yield (as, n) - val genListAndSentinel = for { + val genListAndSentinel: Gen[Tuple2[List[Int], Int]] = for { as <- Arbitrary.arbitrary[List[Int]] if as.nonEmpty n <- Gen.choose(0, as.length - 1) } yield (as, as(n)) - def await[T](fut: Future[T]) = Await.result(fut, 100.milliseconds) + def await[T](fut: Future[T]): T = Await.result(fut, 100.milliseconds) def undefined[A]: A = throw new Exception diff --git a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala index 0ea3257558..7127cc2842 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala @@ -15,7 +15,7 @@ class SimpleOffer[T](var futures: Stream[Future[Tx[T]]]) extends Offer[T] { def this(fut: Future[Tx[T]]) = this(Stream.continually(fut)) def this(tx: Tx[T]) = this(Stream.continually(Future.value(tx))) - def prepare() = { + def prepare(): Future[Tx[T]] = { val next #:: rest = futures futures = rest next diff --git a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala index 5c805508ab..31c343fd7f 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala @@ -16,7 +16,7 @@ abstract class LocalSchedulerTest(lifo: Boolean) extends FunSuite with Matchers def run(): Unit = f }) - val N = 100 + val N: Int = 100 test("run the first submitter immediately") { var ok = false diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index 393dc499fc..569d8184d9 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -10,6 +10,7 @@ import org.scalatest.junit.AssertionsForJUnit import org.scalatest.mockito.MockitoSugar import org.scalatest.prop.{Checkers, GeneratorDrivenPropertyChecks} import scala.collection.mutable +import java.nio.charset.Charset class BufTest extends FunSuite @@ -18,7 +19,7 @@ class BufTest with Checkers with AssertionsForJUnit { - val AllCharsets = Seq( + val AllCharsets: Seq[Charset] = Seq( JChar.ISO_8859_1, JChar.US_ASCII, JChar.UTF_8, diff --git a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala index 2105767c29..6e83eb6a10 100644 --- a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala @@ -14,7 +14,7 @@ class ByteReaderTest extends FunSuite with GeneratorDrivenPropertyChecks { def readerWith(bytes: Byte*): ByteReader = ByteReader(Buf.ByteArray.Owned(bytes.toArray)) - def maskMedium(i: Int) = i & 0x00ffffff + def maskMedium(i: Int): Int = i & 0x00ffffff test("readBytes increments position by the number of bytes returned") { val br = readerWith() diff --git a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala index 4ef85a0811..ac62566073 100644 --- a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala @@ -10,8 +10,8 @@ import org.scalatest.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) class InputStreamReaderTest extends FunSuite { - def arr(i: Int, j: Int) = Array.range(i, j).map(_.toByte) - def buf(i: Int, j: Int) = Buf.ByteArray.Owned(arr(i, j)) + def arr(i: Int, j: Int): Array[Byte] = Array.range(i, j).map(_.toByte) + def buf(i: Int, j: Int): Buf = Buf.ByteArray.Owned(arr(i, j)) test("InputStreamReader - read empty") { val a = Array.empty[Byte] diff --git a/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala index 97441bd8d4..9b042b093c 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala +++ b/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala @@ -11,15 +11,15 @@ import scala.util.Random @RunWith(classOf[JUnitRunner]) class ActivitySourceTest extends FunSuite with BeforeAndAfter { - val ok = new ActivitySource[String] { + val ok: ActivitySource[String] = new ActivitySource[String] { def get(varName: String) = Activity.value(varName) } - val failed = new ActivitySource[Nothing] { + val failed: ActivitySource[Nothing] = new ActivitySource[Nothing] { def get(varName: String) = Activity.exception(new Exception(varName)) } - val tempFile = Random.alphanumeric.take(5).mkString + val tempFile: String = Random.alphanumeric.take(5).mkString def writeToTempFile(s: String): Unit = { val printer = new java.io.PrintWriter(new File(tempFile)) diff --git a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala index cdae36aba0..e62c0c8acb 100644 --- a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala @@ -8,8 +8,8 @@ import org.scalatest.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) class CloseAwaitablyTest extends FunSuite { class Context extends Closable with CloseAwaitably { - val p = new Promise[Unit] - var n = 0 + val p: Promise[Unit] = new Promise[Unit] + var n: Int = 0 def close(deadline: Time): Future[Unit] = closeAwaitably { n += 1 p diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index ecde962406..fa35290017 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -22,7 +22,7 @@ import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) -class DurationTest extends { val ops = Duration } with TimeLikeSpec[Duration] { +class DurationTest extends { val ops: Duration.type = Duration } with TimeLikeSpec[Duration] { "Duration" should { "*" in { diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index 16437fd764..d9c5cced13 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -13,7 +13,7 @@ import scala.util.control.NonFatal @RunWith(classOf[JUnitRunner]) class FuturePoolTest extends FunSuite with Eventually { - implicit override val patienceConfig = + implicit override val patienceConfig: PatienceConfig = PatienceConfig(timeout = scaled(Span(15, Seconds)), interval = scaled(Span(5, Millis))) test("FuturePool should dispatch to another thread") { @@ -173,10 +173,10 @@ class FuturePoolTest extends FunSuite with Eventually { } class PoolCtx { - val executor = Executors.newFixedThreadPool(1) - val pool = FuturePool(executor) + val executor: ExecutorService = Executors.newFixedThreadPool(1) + val pool: ExecutorServiceFuturePool = FuturePool(executor) - val pools = Seq(FuturePool.immediatePool, pool) + val pools: Seq[FuturePool] = Seq(FuturePool.immediatePool, pool) } test("handles NonLocalReturnControl properly") { diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index 72d97cb3ce..310dabc4d3 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -128,8 +128,8 @@ class PromiseTest extends FunSuite { } class HandledMonitor extends Monitor { - var handled = null: Throwable - def handle(exc: Throwable) = { + var handled: Throwable = null: Throwable + def handle(exc: Throwable): Boolean = { handled = exc true } diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 74fac0d2a1..a1ad02d134 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -399,7 +399,7 @@ class TimeFormatTest extends WordSpec { } @RunWith(classOf[JUnitRunner]) -class TimeTest extends { val ops = Time } with TimeLikeSpec[Time] with Eventually +class TimeTest extends { val ops: Time.type = Time } with TimeLikeSpec[Time] with Eventually with IntegrationPatience { "Time" should { diff --git a/util-core/src/test/scala/com/twitter/util/TryTest.scala b/util-core/src/test/scala/com/twitter/util/TryTest.scala index 2317b059d4..ffc1453cb2 100644 --- a/util-core/src/test/scala/com/twitter/util/TryTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TryTest.scala @@ -7,7 +7,7 @@ import org.scalatest.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) class TryTest extends FunSuite { class MyException extends Exception - val e = new Exception("this is an exception") + val e: Exception = new Exception("this is an exception") test("Try.apply(): should catch exceptions and lift into the Try type") { assert(Try[Int](1) == Return(1)) From 337e270f8fbdb5dd76ec01573e5262c0ab2054bd Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 23 Jan 2019 19:27:25 +0000 Subject: [PATCH 101/796] util-security: Allow Pkcs8KeyManagerFactory to Handle Multiple Certs Problem The `Pkcs8KeyManagerFactory` within util-security currently only supports a certificate file with a single certificate stored within it. There are scenarios where it's beneficial to support a single certificate file with the full certificate chain stored within it. Solution Add support to Pkcs8KeyManagerFactory for a single certificate file with multiple certificates contained within it. JIRA Issues: CSL-7519 Differential Revision: https://phabricator.twitter.biz/D263190 --- CHANGELOG.rst | 3 + .../security/Pkcs8KeyManagerFactory.scala | 22 ++-- .../certs/test-rsa-full-cert-chain.crt | 107 ++++++++++++++++++ .../security/Pkcs8KeyManagerFactoryTest.scala | 25 +++- 4 files changed, 142 insertions(+), 15 deletions(-) create mode 100644 util-security/src/test/resources/certs/test-rsa-full-cert-chain.crt diff --git a/CHANGELOG.rst b/CHANGELOG.rst index caaec64d9e..7d70987a5f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,9 @@ New Features * util-core: Added `Duration.fromMinutes` to return a `Duration` from a given number of minutes. ``PHAB_ID=D259795`` +* util-security: `Pkcs8KeyManagerFactory` now supports a certificates file which contains multiple + certificates that are part of the same certificate chain. ``PHAB_ID=D263190`` + 19.1.0 ------- diff --git a/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala b/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala index 3b2629a230..40bdcd3a46 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala @@ -12,13 +12,13 @@ import javax.net.ssl.{KeyManager, KeyManagerFactory} /** * A factory which can create a [[javax.net.ssl.KeyManager KeyManager]] which contains - * an X.509 Certificate and a PKCS#8 private key. + * an X.509 Certificate (or Certificate chain) and a PKCS#8 private key. */ -class Pkcs8KeyManagerFactory(certFile: File, keyFile: File) { +class Pkcs8KeyManagerFactory(certsFile: File, keyFile: File) { private[this] def logException(ex: Throwable): Unit = log.warning( - s"Pkcs8KeyManagerFactory (${certFile.getName()}, ${keyFile.getName()}) " + + s"Pkcs8KeyManagerFactory (${certsFile.getName()}, ${keyFile.getName()}) " + s"failed to create key manager: ${ex.getMessage()}." ) @@ -27,11 +27,14 @@ class Pkcs8KeyManagerFactory(certFile: File, keyFile: File) { kf.generatePrivate(keySpec) } - private[this] def createKeyStore(cert: X509Certificate, privateKey: PrivateKey): KeyStore = { + private[this] def createKeyStore( + certs: Seq[X509Certificate], + privateKey: PrivateKey + ): KeyStore = { val alias: String = UUID.randomUUID().toString() val ks: KeyStore = KeyStore.getInstance("JKS") ks.load(null) - ks.setKeyEntry(alias, privateKey, "".toCharArray(), Array(cert)) + ks.setKeyEntry(alias, privateKey, "".toCharArray(), certs.toArray) ks } @@ -42,19 +45,20 @@ class Pkcs8KeyManagerFactory(certFile: File, keyFile: File) { } /** - * Attempts to read the contents of both the X.509 Certificate file and the PKCS#8 + * Attempts to read the contents of both the X.509 Certificates file and the PKCS#8 * Private Key file and combine the contents into a [[javax.net.ssl.KeyManager KeyManager]]. * The singular value is returned in an Array for ease of use with * [[javax.net.ssl.SSLContext SSLContext's]] init method. */ def getKeyManagers(): Try[Array[KeyManager]] = { - val tryCert: Try[X509Certificate] = new X509CertificateFile(certFile).readX509Certificate() + val tryCerts: Try[Seq[X509Certificate]] = + new X509CertificateFile(certsFile).readX509Certificates() val tryKeySpec: Try[PKCS8EncodedKeySpec] = new Pkcs8EncodedKeySpecFile(keyFile).readPkcs8EncodedKeySpec() val tryPrivateKey: Try[PrivateKey] = tryKeySpec.map(keySpecToPrivateKey) - val tryCertKey: Try[(X509Certificate, PrivateKey)] = join(tryCert, tryPrivateKey) - val tryKeyStore: Try[KeyStore] = tryCertKey.map((createKeyStore _).tupled) + val tryCertsKey: Try[(Seq[X509Certificate], PrivateKey)] = join(tryCerts, tryPrivateKey) + val tryKeyStore: Try[KeyStore] = tryCertsKey.map((createKeyStore _).tupled) tryKeyStore.map(keyStoreToKeyManagers).onFailure(logException) } diff --git a/util-security/src/test/resources/certs/test-rsa-full-cert-chain.crt b/util-security/src/test/resources/certs/test-rsa-full-cert-chain.crt new file mode 100644 index 0000000000..37217b6e2b --- /dev/null +++ b/util-security/src/test/resources/certs/test-rsa-full-cert-chain.crt @@ -0,0 +1,107 @@ +-----BEGIN CERTIFICATE----- +MIIGNTCCBB2gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwgZUxCzAJBgNVBAYTAlVT +MRMwEQYDVQQIDApDYWxpZm9ybmlhMRAwDgYDVQQKDAdUd2l0dGVyMR8wHQYDVQQL +DBZDb3JlIFN5c3RlbXMgTGlicmFyaWVzMRwwGgYDVQQDDBNDU0wgSW50ZXJtZWRp +YXRlIENBMSAwHgYJKoZIhvcNAQkBFhFyeWFub0B0d2l0dGVyLmNvbTAeFw0xNzA4 +MTUxNzUyMzZaFw0yNzA4MTMxNzUyMzZaMIGvMQswCQYDVQQGEwJVUzETMBEGA1UE +CAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwH +VHdpdHRlcjEfMB0GA1UECwwWQ29yZSBTeXN0ZW1zIExpYnJhcmllczEeMBwGA1UE +AwwVbG9jYWxob3N0LnR3aXR0ZXIuY29tMSAwHgYJKoZIhvcNAQkBFhFyeWFub0B0 +d2l0dGVyLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ+we1c9 +H8m2p+pj3ycQozSKKmbDRQG8ajp/IgoayR1N3FqYR5CnAGRdCTAZLfQ+dMbiE45z +ascl8a/g9e02ZZOovpIYzzVbR24N+qn5jmSP0ibRuiUsNeZ/N9IZ0YjotK+qfdiE +0qE12TiWhctBpyyn9OXLwMpn+TnXi4ejTyr1xp4OwWKzgnZXVhKzZQ+pqCkA9Edj +jKCbHcgdwPDa0LFYeakGdEkoyqTcC/jfW0uzVLPdPbC2VLA3Sr+gXbpltbb378iq +btk+/OG1YCYN5rtS0JypbC5lPzuGN/6al8arTCtJ66+5w+d0OMy0MsIIINcKP6/v +u9fXXaNQ5AUA0e8CAwEAAaOCAXEwggFtMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEB +BAQDAgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIg +Q2VydGlmaWNhdGUwHQYDVR0OBBYEFL5RSPjQgZ2VVFuBpy9dBHm0HMzeMIHTBgNV +HSMEgcswgciAFFoRgBgNkweNV0f60G8uQ5IUwoXxoYGrpIGoMIGlMQswCQYDVQQG +EwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNj +bzEQMA4GA1UECgwHVHdpdHRlcjEfMB0GA1UECwwWQ29yZSBTeXN0ZW1zIExpYnJh +cmllczEUMBIGA1UEAwwLQ1NMIFJvb3QgQ0ExIDAeBgkqhkiG9w0BCQEWEXJ5YW5v +QHR3aXR0ZXIuY29tggIQADAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwDQYJKoZIhvcNAQELBQADggIBAKHlSXrMfCFE6McLvz47gfH8171+1tzB +D5FyEaG7YUqkHxN6jjLiXE/rIrq1+zTwa9YniTw+yLUnuA6ddFxNnRu4JEVYGkFV +YJNPUUsGSSIKBPPamui156BMyt4BkA1c9HrGO+L/3QE4mjJOmLQguorHlK2GGhiv +bW4r587CmPuDzNIFb6wmoU0S4xrRRVyETQE+LbdbsErZC2T8/nq8XyII7EpbzNpo +XX6CP59ajvGX11b5aAo9GmLKwQHmpBvreY8RdciW7YViNlUNdGY786tVxkUvMPwi +hsSK1ikik7Ffn7tZ3VIj0wyJFQOU08dnCrx8FTga2AuctExv1lbFL8xHwy/2ZXFB +O6df/CuIUAKyA++L8/MFyKJFb4zXQb+D8Zu5L8lmvYWhy2awfjjEtZIAfdO6BICD +nIej/4Wd/FDlTMNnE1vrKvcRPPAdWr7EPLSb3Le+O5SnQHh4VLmrPZrmsmGLp0ca +bMxyalEcrUJZr39fq6Y85c4gkn2EeYNozZZeMbhirvp3JnlTNLnhLm5v8Uo/xb8T +RAq9oGYV/QNT7gc55jP/3HXt3IHWZHZ3Uw++lg1ZKbFSm4NDmUOP/kaZREcislun +4hq72GLAGnJMEa7W0jW16hBCVDEJCRJWtDzwJXKi56DDNSoVydKLNmmqTDWswrJk +d0zC9i9vSawE +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIGHjCCBAagAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwgaUxCzAJBgNVBAYTAlVT +MRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNpc2NvMRAw +DgYDVQQKDAdUd2l0dGVyMR8wHQYDVQQLDBZDb3JlIFN5c3RlbXMgTGlicmFyaWVz +MRQwEgYDVQQDDAtDU0wgUm9vdCBDQTEgMB4GCSqGSIb3DQEJARYRcnlhbm9AdHdp +dHRlci5jb20wHhcNMTcwODE1MTc0ODE4WhcNMzQwMTE4MTc0ODE4WjCBlTELMAkG +A1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEDAOBgNVBAoMB1R3aXR0ZXIx +HzAdBgNVBAsMFkNvcmUgU3lzdGVtcyBMaWJyYXJpZXMxHDAaBgNVBAMME0NTTCBJ +bnRlcm1lZGlhdGUgQ0ExIDAeBgkqhkiG9w0BCQEWEXJ5YW5vQHR3aXR0ZXIuY29t +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA5kLopHd6fKjk3AzjGkj5 +9TnybLw85SId8MOGtEbdx88AiQA+Omidoc7efBk2c8dVu9KFrb3noQcKM6feNz9I +eul/s2hIJDlJXt3Psg/HzGH2Z46MvIc8Ebj/QtfRXDbtfVInY+xdL01o1l8o+aOD +5fQ9UE0/nPkKgCFWnQ8IAiEY9uSb7I5ujp6x68D7NUO38DhQ+6KMsU3141xb3RsV +15c6qxCIUxktNOAYv9WTkrKO4OBp64W72P2RQs6paot0001B2ARATyYYmWPWjivt +rc/eg8msBYh+xfjbqwM8XJjyvDUXwJ69TIE8KP6JwqI/elf8VPV7LAlOXwP4O8XN +fq/z/l9yNOEc6yH+25S2rA304LqD8ymJv3EtoOOERMuP/P2JtNjC5anB/96Dnjy2 +AvUY1bLVqC/A00QfW0XZw/rFd5B80rFfrdCRy6RohvaAT0EvTDZb/Q2CKKGV4KBE +aQnmWBzEjICX9OuaTWWwj8b98jCKlF00rZvJLeypoNHdjiMRmiKp+9jDbbZ2X4OS +/vz+82SgCGc/7zwAl2qR4mJa5m80JlmtFvIkipTbTfyVbxbMi6H7uXOCWyD5myAC +cNM5iO60DhbfC9yj1orO+Gy6i12A8DQmTXzVM/TLeInx3yVeoOBldK6OkLrUIc3N +VLWkwruQIYGg3cco3om2qh8CAwEAAaNmMGQwHQYDVR0OBBYEFFoRgBgNkweNV0f6 +0G8uQ5IUwoXxMB8GA1UdIwQYMBaAFH7TBrmeXnA2h6GQWlitpnUwMrGRMBIGA1Ud +EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4IC +AQBHYf7aPftsjmUDRvsBxYUPncrUaY8qE1YA8JFFP7DnI2KehyITXfJd6An933IV +89SjfBb0nsDYiI36CbkFZVAJIxtYoqtpBwIMJXjgQcYI0ZHDQH3ijFrpHFYAQGm4 +2uaruekatUzrf8Mn5kgvnD72AOpqGnUXLGvxyhYK6VczDbFylNqPtJ/wDndI9xmb +Q8rbVKC6JWkllhOOROO8sG96jJuzJ9t2ob4bwfCIsXadIsHrNSDSTDH8bMqAaw8y +WprrGev80jkK2cS43w2o4hiY4L1aREGBkEja+cTJ6bb34Ebkc3Wrojtu76cHr95W +qYE0RT5/eqO04YnyPmr2/D1YTdn0z9HtJbTs7TlpxA173X7QpCG8TWki3KssBE+P +RfeHh8jVtwYT51WTZBwbf5M0bnQ1c7D3RTOAPtYkzLw65iK+ibliiqeXvXbV2FW1 +UkJCct5YuIF/NKJ0bJ75l08h86QrkS1dW5xW0IO3pN3BcVlvn+EQQRo7pKkFA6H7 +jGYbbSZP/AMZZswi9qrEp7qDt5bNjEaRQlkKLRWhOKC0GkvLEw8drbs4DcapcGwJ +vAGxcsEym1JGZQ8cQxVEq/K8Tk1blZDmJjPpplWG6YCljnvAx3Kc/BFChoprtVzg +amy+LAlsCrlMtvb9VKZShZw/IPil2d9wWQqdDOqq/Nz2gg== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIGMjCCBBqgAwIBAgIJANugAHtkAULIMA0GCSqGSIb3DQEBCwUAMIGlMQswCQYD +VQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5j +aXNjbzEQMA4GA1UECgwHVHdpdHRlcjEfMB0GA1UECwwWQ29yZSBTeXN0ZW1zIExp +YnJhcmllczEUMBIGA1UEAwwLQ1NMIFJvb3QgQ0ExIDAeBgkqhkiG9w0BCQEWEXJ5 +YW5vQHR3aXR0ZXIuY29tMB4XDTE3MDgxNTE3NDM0MloXDTM3MDgxMDE3NDM0Mlow +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1T +YW4gRnJhbmNpc2NvMRAwDgYDVQQKDAdUd2l0dGVyMR8wHQYDVQQLDBZDb3JlIFN5 +c3RlbXMgTGlicmFyaWVzMRQwEgYDVQQDDAtDU0wgUm9vdCBDQTEgMB4GCSqGSIb3 +DQEJARYRcnlhbm9AdHdpdHRlci5jb20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQC5sYrwlKIsytZbclVnrYs+KK0qY8izZ8KK4g4hQ/SRNL5YkvOecvfi +qCq19Dm5PE024Y/dAb0M4zq6HEHzsF5+M81o2FSKraiqUeJe3YwYxCwM+5msGym1 +Bys2rHnboe15Uj0KSRzW1Fouokw5GXdqTqKtmSYgHuRm0IYU8f6B2T6/tjjObUCW +Mm66xpR6YkFQtWLWgoCwAcBclvhwLyQ4GT3CBIxWzXW/AyXv09XcjgQIp2ftm2ej +bNhO/UMS2Juwzl35Sk2ilSpHGtSIwJvAwcrPFZ/LCenrC/bslbp2GMZ1JmT1+Kgt +xOUbyHRPsJGmf05kRL3+JlPhU2tBkMP5hhbr3/W+HqoSGrhGDt199HHZDy3lvESX +Hu46PsIakQj4LG58eJ98rHEa+LG4bqiROjDfOL39Fy2pkdq2Sp/nYXNS05H/uBaT +sePaWXeUAL9VcCq4sCFfeL1n7aic0Dwjv2+i1hiETvVhFzuQk1ZumNIUJBC6IOKI +nr65WWf+KQnUPQX1coOUbFSHJF/ZQAi9WqoYyLWDA3rUWVFaS6cbaF0gAsPEbYKQ +TkC1ApPxpQhCYAp8djTYzzkjsIEc6rFRGrGSHAxITtLlNZ/SLD6D4WS8n+JQMhAp +UsG0fPlEfyr/IT/MmytZgC8NgARCjpA72fMqhCXJ90RU0hq3BvPqowIDAQABo2Mw +YTAdBgNVHQ4EFgQUftMGuZ5ecDaHoZBaWK2mdTAysZEwHwYDVR0jBBgwFoAUftMG +uZ5ecDaHoZBaWK2mdTAysZEwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC +AYYwDQYJKoZIhvcNAQELBQADggIBADbYfooAL6S0i+zecjZShEjKCZL9BFZAyy68 +HSgvyoOToinu9XRwLSb6OzMTQcsJ/zlETc+sb7hP0jtpIDHgNOv+Wrreh/7otKsp +/ijYGG9RfvZZQGZs0Q98/GjhqTJryXwOqL+eBX8V4SnJbOKaQgURH0eTcs2Se8r8 +4TNhwZFtwMPGbCW3HQVkx3VONVYKQAzGvbJBydj+wJVB76aO98DgsYchdgr21Mkv +elm953mnqMqWxyZzr9Ok8gsYeM23uX1OTCFMAzMxtrRTQWCRjq3NAedNVqufd4WF +eVda3NM3L1dipzyvwePh1XAQmYOxPoyKiDIKlYuumSO6CtRWT/wMRnNlEq4VHgTd +3JqeOv5gSM8Sd99/l6PdCxAJGhUPdHUOst3S4H6fy21BWd4srfivJuSRRYtWrs89 +Ij4sxaYq7PzcSTZkVV5Fn0Dkf8F5qna3AKKODbefqSssG9tw1UeavDranNiUzg3O +1LPnBurGDEWRyX+0bg60+cB6BY3mkWaBlYyFSs+8AGHgwfBn/g+rVapPA6XdJGvz +7VZXexyTe+IMWU6013JIXiwcZEUD3s3xNHJ8bdyOHf3kEz2gYMVF4q1uP8OHpE9Y +/6/0JfEjlRoG7A8mOi+QS6aYYSiCHuYLAaUL+ABbRFrLq6ppKWNuVgwsTRlqL3im +2C/dTIlS +-----END CERTIFICATE----- diff --git a/util-security/src/test/scala/com/twitter/util/security/Pkcs8KeyManagerFactoryTest.scala b/util-security/src/test/scala/com/twitter/util/security/Pkcs8KeyManagerFactoryTest.scala index 232795eee4..f4d02818c0 100644 --- a/util-security/src/test/scala/com/twitter/util/security/Pkcs8KeyManagerFactoryTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/Pkcs8KeyManagerFactoryTest.scala @@ -3,14 +3,11 @@ package com.twitter.util.security import com.twitter.io.TempFile import java.io.File import javax.net.ssl.{KeyManager, X509KeyManager} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class PKCS8KeyManagerFactoryTest extends FunSuite { - private[this] def assertContainsCertAndKey(keyManager: KeyManager): Unit = { + private[this] def assertContainsCertsAndKey(keyManager: KeyManager, numCerts: Int = 1): Unit = { assert(keyManager.isInstanceOf[X509KeyManager]) val x509km = keyManager.asInstanceOf[X509KeyManager] @@ -22,7 +19,7 @@ class PKCS8KeyManagerFactoryTest extends FunSuite { val alias = clientAliases.head val certChain = x509km.getCertificateChain(alias) - assert(certChain.length == 1) + assert(certChain.length == numCerts) val cert = certChain.head assert(cert.getSubjectX500Principal.getName().contains("localhost.twitter.com")) @@ -83,7 +80,23 @@ class PKCS8KeyManagerFactoryTest extends FunSuite { assert(tryKeyManagers.isReturn) val keyManagers = tryKeyManagers.get() assert(keyManagers.length == 1) - assertContainsCertAndKey(keyManagers.head) + assertContainsCertsAndKey(keyManagers.head) + } + + test("Chain of certificates is good") { + val tempCertsFile = TempFile.fromResourcePath("/certs/test-rsa-full-cert-chain.crt") + // deleteOnExit is handled by TempFile + + val tempKeyFile = TempFile.fromResourcePath("/keys/test-pkcs8.key") + // deleteOnExit is handled by TempFile + + val factory = new Pkcs8KeyManagerFactory(tempCertsFile, tempKeyFile) + val tryKeyManagers = factory.getKeyManagers() + + assert(tryKeyManagers.isReturn) + val keyManagers = tryKeyManagers.get() + assert(keyManagers.length == 1) + assertContainsCertsAndKey(keyManagers.head, numCerts = 3) } } From 8948aea464e561ff43d70d133e96e68a60fb4fdf Mon Sep 17 00:00:00 2001 From: Borja Lorente Date: Fri, 1 Feb 2019 11:41:47 +0000 Subject: [PATCH 102/796] util-tunable: Add missing dependency on jackson-annotations Problem / Solution `JsonTunableMapper.scala` uses `com.fasterxml.jackson.annotation.JsonProperty` but doesn't declare a pants dependency on its providing target. Differential Revision: https://phabricator.twitter.biz/D267812 --- util-tunable/src/main/scala/com/twitter/util/tunable/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD index fcb4206f5a..396f9779ac 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD @@ -6,6 +6,7 @@ scala_library( repo = artifactory, ), dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", From 6c8f2522eba067fbb47b23ec5b521cb89d0c3daf Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 4 Feb 2019 19:38:05 +0000 Subject: [PATCH 103/796] util-core: Correct the Reader.flatten behavior Problem 1. Reader.flatten didn't satisfy the onClose after fully read. 2. Reader.flatten keep reading after encountering an exception. Solution 1. satisfy the promise after finish reading the stream. 2. while encountering an exception, stop `updateCurrentReader()` and fulfill the onClose with the same exception. JIRA Issues: CSL-7496, CSL-7497 Differential Revision: https://phabricator.twitter.biz/D263751 --- .../main/scala/com/twitter/io/Reader.scala | 30 +++++---- .../scala/com/twitter/io/ReaderTest.scala | 64 +++++++++---------- 2 files changed, 49 insertions(+), 45 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 55443cd08f..a1851842ef 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -216,13 +216,17 @@ object Reader { private[this] val closep = Promise[StreamTermination]() final def read(): Future[Option[A]] = - if (closep.updateIfEmpty(StreamTermination.FullyRead.Return)) - fa.map(Some.apply) - else + if (!closep.isDefined) { + fa.map(Some.apply).respond { + case Throw(e) => closep.updateIfEmpty(Throw(e)) + case Return(_) => closep.updateIfEmpty(StreamTermination.FullyRead.Return) + } + } else { closep.flatMap { case StreamTermination.FullyRead => Future.None case StreamTermination.Discarded => Future.exception(new ReaderDiscardedException) } + } final def discard(): Unit = closep.updateIfEmpty(StreamTermination.Discarded.Return) @@ -416,25 +420,29 @@ object Reader { // access this currentReader is synchronized on `self` private[this] var currentReader: Reader[A] = Reader.empty - // this method should be called inside a synchronized block - def updateCurrentAndRead(): Future[Option[A]] = + private def updateCurrentAndRead(): Future[Option[A]] = readers.read().flatMap { case Some(reader) => - currentReader = reader + self.synchronized { currentReader = reader } read() - case None => Future.None + case None => + closep.updateIfEmpty(StreamTermination.FullyRead.Return) + Future.None } def read(): Future[Option[A]] = self.synchronized { - currentReader.read().flatMap { - case sa @ Some(_) => Future.value(sa) - case _ => updateCurrentAndRead() + currentReader.read().transform { + case Return(None) => updateCurrentAndRead() + case Return(sa) => Future.value(sa) + case Throw(t) => + closep.updateIfEmpty(Throw(t)) + Future.exception(t) } } def discard(): Unit = { closep.updateIfEmpty(StreamTermination.Discarded.Return) - currentReader.discard() + self.synchronized(currentReader).discard() readers.discard() } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index ff775e700b..7f2ebb5ad3 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -261,26 +261,31 @@ class ReaderTest assertFailed(reader, p) } - test("Reader.flatten - lazy tail") { - val head = new Reader[Buf] { - def read(): Future[Option[Buf]] = Future.exception(new Exception) + test("Reader.flatten - onClose is satisfied when fully read") { + val reader = Reader.flatten(Reader.fromSeq(Seq(Reader.value(10), Reader.value(20)))) + assert(await(reader.read()) == Some(10)) + assert(await(reader.read()) == Some(20)) + assert(await(reader.read()) == None) + assert(await(reader.onClose) == StreamTermination.FullyRead) + } - def discard(): Unit = () - def onClose: Future[StreamTermination] = Future.never + test("Reader.flatten - stop when encounter exceptions") { + val reader = Reader.flatten( + Reader.fromSeq( + Seq(Reader.value(10), Reader.exception(new Exception("stop")), Reader.value(20)))) + assert(await(reader.read()) == Some(10)) + val ex1 = intercept[Exception] { + await(reader.read()) } - val p = new Promise[Unit] - - def tail: Reader[Reader[Buf]] = { - p.setDone() - Reader.empty + assert(ex1.getMessage == "stop") + val ex2 = intercept[Exception] { + await(reader.read()) } - - val combined = Reader.flatten(Reader.fromSeq(head +: undefinedReaders)) - val buf = Reader.readAll(combined) - intercept[Exception] { - await(buf) + assert(ex2.getMessage == "stop") + val ex3 = intercept[Exception] { + await(reader.onClose) } - assert(!p.isDefined) + assert(ex3.getMessage == "stop") } test("Reader.fromSeq works on infinite streams") { @@ -291,19 +296,6 @@ class ReaderTest assert(await(reader.read()) == Some(1)) } - test("Reader.flatten - concurrent read on a Reader with empty Reader prefixed") { - val p = new Promise[Int] - val reader = Reader.flatten( - Reader.fromSeq(Seq(Reader.empty, Reader.fromFuture(p), Reader.fromSeq(Seq(2, 3))))) - - val x = reader.read() - val y = reader.read() - val z = reader.read() - - p.setValue(1) - assert(Seq(Some(1), Some(2), Some(3)) == Seq(await(x), await(y), await(z))) - } - test("Reader.fromStream closes resources on EOF read") { val in = spy(new ByteArrayInputStream(arr(0, 10))) val r = Reader.fromStream(in, 4) @@ -322,10 +314,10 @@ class ReaderTest } test("Reader.fromAsyncStream completes when stream is empty") { - val as = AsyncStream(buf(1, 10)) + val as = AsyncStream(buf(1, 10)) ++ AsyncStream.exception[Buf](new Exception()) ++ AsyncStream( + buf(1, 10)) val r = Reader.fromAsyncStream(as) - val f = Reader.readAll(r) - assert(await(f) == buf(1, 10)) + assert(await(r.read()) == Some(buf(1, 10))) } test("Reader.fromAsyncStream fails on exceptional stream") { @@ -477,10 +469,14 @@ class ReaderTest test("Reader.exception") { forAll { ex: Exception => val r = Reader.exception(ex) - intercept[Exception] { + val exr = intercept[Exception] { await(r.read()) } - assert(await(r.read()) == None) + assert(exr == ex) + val exc = intercept[Exception] { + await(r.onClose) + } + assert(exc == ex) } } From b9914f15b14e28cc6705cc6b5c876b22087f326b Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 6 Feb 2019 00:50:08 +0000 Subject: [PATCH 104/796] util-core: Remove silly shim CountDownLatch Problem / Solution `com.twitter.util.CountDownLatch` is an extremely thin shim around the java stdlib one. We should stop using it. JIRA Issues: CSL-7533 Differential Revision: https://phabricator.twitter.biz/D269404 --- CHANGELOG.rst | 10 ++++++++++ .../scala/com/twitter/util/CountDownLatch.scala | 16 ---------------- .../twitter/concurrent/AsyncSemaphoreTest.scala | 2 +- .../test/scala/com/twitter/util/EventTest.scala | 2 +- .../scala/com/twitter/util/FuturePoolTest.scala | 4 ++-- .../test/scala/com/twitter/util/TimerTest.scala | 6 +++--- 6 files changed, 17 insertions(+), 23 deletions(-) delete mode 100644 util-core/src/main/scala/com/twitter/util/CountDownLatch.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7d70987a5f..04b4c0b609 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,16 @@ New Features * util-security: `Pkcs8KeyManagerFactory` now supports a certificates file which contains multiple certificates that are part of the same certificate chain. ``PHAB_ID=D263190`` +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: Remove `c.t.u.CountDownLatch` which is an extremely thin shim around + `j.u.c.CountDownLatch` that provides pretty limited value. To migrate to `j.u.c.CountDownLatch`, + instead of `c.t.u.CountDownLatch#await(Duration)`, please use + `j.u.c.CountDownLatch#await(int, TimeUnit)`, and instead of + `c.t.u.CountDownLatch#within(Duration)`, please throw an exception yourself after awaiting. + ``PHAB_ID=D269404`` + 19.1.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/CountDownLatch.scala b/util-core/src/main/scala/com/twitter/util/CountDownLatch.scala deleted file mode 100644 index 45f9da9725..0000000000 --- a/util-core/src/main/scala/com/twitter/util/CountDownLatch.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.util - -import java.util.concurrent.TimeUnit - -class CountDownLatch(val initialCount: Int) { - val underlying: java.util.concurrent.CountDownLatch = - new java.util.concurrent.CountDownLatch(initialCount) - def count: Long = underlying.getCount - def isZero: Boolean = count == 0 - def countDown(): Unit = underlying.countDown() - def await(): Unit = underlying.await() - def await(timeout: Duration): Boolean = underlying.await(timeout.inMillis, TimeUnit.MILLISECONDS) - def within(timeout: Duration): Boolean = await(timeout) || { - throw new TimeoutException(timeout.toString) - } -} diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index 6cf21bda0c..63085ff026 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -2,7 +2,7 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ import com.twitter.util._ -import java.util.concurrent.{ConcurrentLinkedQueue, RejectedExecutionException} +import java.util.concurrent.{ConcurrentLinkedQueue, RejectedExecutionException, CountDownLatch} import org.junit.runner.RunWith import org.scalatest.fixture.FunSpec import org.scalatest.junit.JUnitRunner diff --git a/util-core/src/test/scala/com/twitter/util/EventTest.scala b/util-core/src/test/scala/com/twitter/util/EventTest.scala index 416d336cf1..612788b997 100644 --- a/util-core/src/test/scala/com/twitter/util/EventTest.scala +++ b/util-core/src/test/scala/com/twitter/util/EventTest.scala @@ -1,7 +1,7 @@ package com.twitter.util import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} -import java.util.concurrent.Executors +import java.util.concurrent.{CountDownLatch, Executors} import org.scalatest.FunSuite import org.scalatest.junit.JUnitRunner import org.junit.runner.RunWith diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index d9c5cced13..6899468db1 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -93,7 +93,7 @@ class FuturePoolTest extends FunSuite with Eventually { runCount.get } - startedLatch.await(1.second) + startedLatch.await(1, TimeUnit.SECONDS) result.raise(new Exception) cancelledLatch.countDown() @@ -212,7 +212,7 @@ class FuturePoolTest extends FunSuite with Eventually { // execute a task we can control val latch = new CountDownLatch(1) val future = pool { - latch.await(10.seconds) + latch.await(10, TimeUnit.SECONDS) true } eventually { assert(pool.poolSize == 1) } diff --git a/util-core/src/test/scala/com/twitter/util/TimerTest.scala b/util-core/src/test/scala/com/twitter/util/TimerTest.scala index b899e2b96f..42131993f6 100644 --- a/util-core/src/test/scala/com/twitter/util/TimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimerTest.scala @@ -1,7 +1,7 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import java.util.concurrent.{CancellationException, ExecutorService} +import java.util.concurrent.{CancellationException, ExecutorService, TimeUnit, CountDownLatch} import java.util.concurrent.atomic.AtomicInteger import org.junit.runner.RunWith import org.mockito.ArgumentCaptor @@ -140,7 +140,7 @@ class TimerTest extends FunSuite with MockitoSugar with Eventually with Integrat throw new scala.MatchError("huh") } - latch.await(30.seconds) + latch.await(30, TimeUnit.SECONDS) assert(errors == 1) @@ -151,7 +151,7 @@ class TimerTest extends FunSuite with MockitoSugar with Eventually with Integrat latch.countDown() } - latch.await(30.seconds) + latch.await(30, TimeUnit.SECONDS) assert(result == 2) assert(errors == 1) From 0a142872237b2709c1228bcedf9441cde2164af6 Mon Sep 17 00:00:00 2001 From: David Rusek Date: Wed, 6 Feb 2019 18:47:53 +0000 Subject: [PATCH 105/796] util-core: Pipe honor deadline on close Problem When closing a Pipe, the deadline is not honored. This is not strictly incorrect but it is possible to try to allow reads to occur before the deadline. Solution Introduce an asynchronous close. Result Reads will succeed up until the close deadline. The default behavior will not be affected, the close will only be async if the Pipe is instantiated with a Timer. JIRA Issues: CSL-3843 Differential Revision: https://phabricator.twitter.biz/D229728 --- CHANGELOG.rst | 10 ++ .../src/main/scala/com/twitter/io/Pipe.scala | 87 +++++++++---- .../test/scala/com/twitter/io/PipeTest.scala | 114 ++++++++++++++++-- 3 files changed, 172 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 04b4c0b609..2d4496c494 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,16 @@ New Features * util-security: `Pkcs8KeyManagerFactory` now supports a certificates file which contains multiple certificates that are part of the same certificate chain. ``PHAB_ID=D263190`` +* util-core: If given a `Timer` upon construction, `c.t.io.Pipe` will respect the close + deadline and wait the given amount of time for any pending writes to be read. ``PHAB_ID=D229728`` + +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-core: Closing a `c.t.io.Pipe` will notify `onClose` when the deadline has passed whereas + before the pipe would wait indefinitely for a read before transitioning to the Closed state. + ``PHAB_ID=D229728`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 6d43e5892f..c0b915db6d 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -1,6 +1,6 @@ package com.twitter.io -import com.twitter.util.{Future, Promise, Return, Time, Throw} +import com.twitter.util.{Future, Promise, Return, Throw, Time, Timer} /** * A synchronous in-memory pipe that connects [[Reader]] and [[Writer]] in the sense @@ -66,12 +66,14 @@ import com.twitter.util.{Future, Promise, Return, Time, Throw} * The following rules should help reasoning about closure signals in pipes: * * - Closing a pipe with a pending read resolves said read into EOF and returns a [[Future.Unit]] - * - Closing a pipe with a pending write fails said write with [[IllegalStateException]] and - * returns a future that will be satisfied when a consumer observes the closure (EOF) via read + * - Closing a pipe with a pending write by default fails said write with [[IllegalStateException]] and + * returns a future that will be satisfied when a consumer observes the closure (EOF) via read. If + * a timer is provided, the pipe will wait until the provided deadline for a successful read before + * failing the write. * - Closing an idle pipe returns a future that will be satisfied when a consumer observes the - * closure (EOF) via read + * closure (EOF) via read when a timer is provided, otherwise the pipe will be closed immedidately. */ -final class Pipe[A] extends Reader[A] with Writer[A] { +final class Pipe[A](timer: Timer) extends Reader[A] with Writer[A] { // Implementation Notes: // @@ -87,6 +89,11 @@ final class Pipe[A] extends Reader[A] with Writer[A] { import Pipe._ + /** + * For Java compatability + */ + def this() = this(Timer.Nil) + // thread-safety provided by synchronization on `this` private[this] var state: State[A] = State.Idle @@ -99,10 +106,6 @@ final class Pipe[A] extends Reader[A] with Writer[A] { case State.Failed(exc) => (null, Future.exception(exc)) - case State.Closing => - state = State.Closed - (null, Future.None) - case State.Closed => (null, Future.None) @@ -112,12 +115,22 @@ final class Pipe[A] extends Reader[A] with Writer[A] { (null, p) case State.Writing(buf, p) => - // pending write can fully fit into this requested read state = State.Idle (p, Future.value(Some(buf))) case State.Reading(_) => (null, Future.exception(new IllegalStateException("read() while read is pending"))) + + case State.Closing(buf, p) => + val result = buf match { + case None => + state = State.Closed + Future.None + case _ => + state = State.Closing(None, null) + Future.value(buf) + } + (p, result) } } @@ -136,7 +149,7 @@ final class Pipe[A] extends Reader[A] with Writer[A] { case State.Failed(exc) => (null, null, Future.exception(exc)) - case State.Closed | State.Closing => + case State.Closed | State.Closing(_, _) => (null, null, Future.exception(new IllegalStateException("write() while closed"))) case State.Idle => @@ -171,9 +184,12 @@ final class Pipe[A] extends Reader[A] with Writer[A] { case State.Closed | State.Failed(_) => // do not update state to failing (null, null, null) - case State.Idle | State.Closing => + case State.Idle => state = State.Failed(cause) (closep, null, null) + case State.Closing(_, p) => + state = State.Failed(cause) + (closep, null, p) case State.Reading(p) => state = State.Failed(cause) (closep, p, null) @@ -192,23 +208,34 @@ final class Pipe[A] extends Reader[A] with Writer[A] { } } + private def closeLater(deadline: Time): Future[Unit] = { + timer.doLater(Time.now.until(deadline)) { + Pipe.this.synchronized { + state match { + case State.Failed(_) | State.Closed => + case _ => state = State.Closed + } + } + } + } + def close(deadline: Time): Future[Unit] = { - val (closer, reader, writer) = synchronized { + val (reader, writer) = synchronized { state match { - case State.Failed(_) | State.Closed | State.Closing => - (null, null, null) + case State.Failed(_) | State.Closed | State.Closing(_, _) => + (null, null) case State.Idle => - state = State.Closing - (null, null, null) + state = State.Closing(None, null) + (null, null) case State.Reading(p) => state = State.Closed - (closep, p, null) + (p, null) - case State.Writing(_, p) => - state = State.Closing - (null, null, p) + case State.Writing(buf, p) => + state = State.Closing(Some(buf), p) + (null, p) } } @@ -216,9 +243,11 @@ final class Pipe[A] extends Reader[A] with Writer[A] { reader.update(Return.None) closep.update(StreamTermination.FullyRead.Return) } else if (writer != null) { - val exn = new IllegalStateException("close() while write is pending") - writer.setException(exn) - if (closer != null) closer.updateIfEmpty(Throw(exn)) + closeLater(deadline).respond { _ => + val exn = new IllegalStateException("close() while write is pending") + writer.updateIfEmpty(Throw(exn)) + closep.updateIfEmpty(Throw(exn)) + } } onClose.unit @@ -256,8 +285,14 @@ object Pipe { /** Indicates the pipe was failed. */ final case class Failed(exc: Throwable) extends State[Nothing] - /** Indicates a close occurred while `Idle` — no reads or writes were pending.*/ - case object Closing extends State[Nothing] + /** + * Indicates a close occurred while a write was pending. + * + * @param value the pending write + * @param p when satisfied it indicates that this write has been fully read or the + * close deadline has expired before it has been fully read. + * */ + final case class Closing[A](value: Option[A], p: Promise[Unit]) extends State[A] /** Indicates the reader has seen the EOF. No more reads or writes are allowed. */ case object Closed extends State[Nothing] diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index 27aa91d192..5cd0f329f4 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -1,8 +1,7 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ -import com.twitter.io.ReaderDiscardedException -import com.twitter.util.{Await, Future, Return} +import com.twitter.util.{Await, Future, MockTimer, Return, Time} import org.scalatest.{FunSuite, Matchers} class PipeTest extends FunSuite with Matchers { @@ -36,13 +35,13 @@ class PipeTest extends FunSuite with Matchers { val buf = Buf.ByteArray.Owned(Array.range(i, j).map(_.toByte)) val f = w.write(buf) assert(f.isDefined) - assert(await(f.liftToTry) == Return(())) + assert(await(f.liftToTry) == Return.Unit) } private def assertWriteEmpty(w: Writer[Buf]): Unit = { val f = w.write(Buf.Empty) assert(f.isDefined) - assert(await(f.liftToTry) == Return(())) + assert(await(f.liftToTry) == Return.Unit) } private def assertReadEofAndClosed(rw: Pipe[Buf]): Unit = { @@ -231,7 +230,10 @@ class PipeTest extends FunSuite with Matchers { intercept[IllegalStateException] { await(wf) } - assertReadEofAndClosed(rw) + assertReadNone(rw) + intercept[IllegalStateException] { + await(rw.onClose) + } } test("read while reading") { @@ -322,15 +324,34 @@ class PipeTest extends FunSuite with Matchers { } test("close before fail") { - val rw = new Pipe[Buf] - val cf = rw.close() - assert(!cf.isDone) + val timer = new MockTimer() + Time.withCurrentTimeFrozen { ctrl => + val rw = new Pipe[Buf](timer) + val cf = rw.close(1.second) + assert(!cf.isDone) - rw.fail(failedEx) - assert(!cf.isDone) + ctrl.advance(1.second) + timer.tick() - assertFailedEx(rw.read()) - assertFailedEx(cf) + rw.fail(failedEx) + + assertFailedEx(rw.read()) + } + } + + test("close before fail within deadline") { + val timer = new MockTimer() + Time.withCurrentTimeFrozen { _ => + val rw = new Pipe[Buf](timer) + val cf = rw.close(1.second) + assert(!cf.isDone) + + rw.fail(failedEx) + assert(!cf.isDone) + + assertFailedEx(rw.read()) + assertFailedEx(cf) + } } test("close while write pending") { @@ -342,6 +363,73 @@ class PipeTest extends FunSuite with Matchers { intercept[IllegalStateException] { await(wf) } - assertReadEofAndClosed(rw) + assertReadNone(rw) + intercept[IllegalStateException] { + await(rw.onClose) + } + } + + test("close respects deadline") { + val mockTimer = new MockTimer() + Time.withCurrentTimeFrozen { timeCtrl => + val rw = new Pipe[Buf](mockTimer) + val wf = rw.write(buf(0, 6)) + + rw.close(1.second) + + assert(!wf.isDefined) + assert(!rw.onClose.isDefined) + + timeCtrl.advance(1.second) + mockTimer.tick() + + intercept[IllegalStateException] { + await(wf) + } + + intercept[IllegalStateException] { + await(rw.onClose) + } + assertReadNone(rw) + } + } + + test("read complete data before close deadline") { + val mockTimer = new MockTimer() + val rw = new Pipe[Buf](mockTimer) + Time.withCurrentTimeFrozen { timeCtrl => + val wf = rw.write(buf(0, 6)) before rw.close(1.second) + + assert(!wf.isDefined) + assertRead(rw, 0, 6) + assertReadNone(rw) + assert(wf.isDefined) + + timeCtrl.advance(1.second) + mockTimer.tick() + + assertReadEofAndClosed(rw) + } + } + + test("multiple reads read complete data before close deadline") { + val mockTimer = new MockTimer() + val buf = Buf.Utf8("foo") + Time.withCurrentTimeFrozen { timeCtrl => + val rw = new Pipe[Buf](mockTimer) + val writef = rw.write(buf) + + rw.close(1.second) + + assert(!writef.isDefined) + assert(await(Reader.readAll(rw)) == buf) + assertReadNone(rw) + assert(writef.isDefined) + + timeCtrl.advance(1.second) + mockTimer.tick() + + assertReadEofAndClosed(rw) + } } } From ed3f7069ffd5640e53776b09bb490b09fd1b8b25 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 8 Feb 2019 16:20:49 +0000 Subject: [PATCH 106/796] util-core: Rename BlacklistStatsReceiver to DenylistStatsReceiver Problem BlacklistStatsReceiver uses non-inclusive language. Solution Rename it to DenylistStatsReceiver JIRA Issues: CSL-7049 Differential Revision: https://phabricator.twitter.biz/D270526 --- CHANGELOG.rst | 2 ++ doc/src/sphinx/util-stats/user_guide.rst | 6 +++--- ...iver.scala => DenylistStatsReceiver.scala} | 16 +++++++------- .../stats/DelegatingStatsReceiverTest.scala | 6 +++--- ....scala => DenylistStatsReceiverTest.scala} | 21 ++++++++----------- .../finagle/stats/StatsReceiverTest.scala | 4 ++-- 6 files changed, 27 insertions(+), 28 deletions(-) rename util-stats/src/main/scala/com/twitter/finagle/stats/{BlacklistStatsReceiver.scala => DenylistStatsReceiver.scala} (55%) rename util-stats/src/test/scala/com/twitter/finagle/stats/{BlacklistStatsReceiverTest.scala => DenylistStatsReceiverTest.scala} (69%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2d4496c494..a027c302fa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -39,6 +39,8 @@ Breaking API Changes `c.t.u.CountDownLatch#within(Duration)`, please throw an exception yourself after awaiting. ``PHAB_ID=D269404`` +* util-core: Rename `BlacklistStatsReceiver` to `DenylistStatsReceiver`. ``PHAB_ID=D270526`` + 19.1.0 ------- diff --git a/doc/src/sphinx/util-stats/user_guide.rst b/doc/src/sphinx/util-stats/user_guide.rst index 667d29949b..c53bd5d12c 100644 --- a/doc/src/sphinx/util-stats/user_guide.rst +++ b/doc/src/sphinx/util-stats/user_guide.rst @@ -124,7 +124,7 @@ operations. Limiting the number of exported metrics via verbosity levels can reduce applications' operational cost. However taking this to extremes may drastically affect operability of your service. We -recommend using your judgment to make sure blacklisting a given metric will not reduce a process' +recommend using your judgment to make sure denylisting a given metric will not reduce a process' visibility. Access needed to a StatsReceiver in an inconvenient place @@ -145,7 +145,7 @@ developers may find useful. - `NullStatsReceiver`_ for when you do not care about all metrics. -- `BlacklistStatsReceiver`_ programmatically decide which metrics to ignore. +- `DenylistStatsReceiver`_ programmatically decide which metrics to ignore. - `BroadcastStatsReceiver`_ allows for sending metrics to two or more ``StatsReceiver``\s. @@ -167,4 +167,4 @@ Via TwitterServer/finagle-stats — the `HTTP admin interface`_ responds with js .. _java.lang.ref.WeakReference: http://docs.oracle.com/javase/8/docs/api/java/lang/ref/WeakReference.html .. _InMemoryStatsReceiver: https://github.com/twitter/util/blob/master/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala .. _HTTP admin interface: https://twitter.github.io/twitter-server/Features.html#http-admin-interface -.. _BlacklistStatsReceiver: https://github.com/twitter/util/blob/develop/util-stats/src/main/scala/com/twitter/finagle/stats/BlacklistStatsReceiver.scala +.. _DenylistStatsReceiver: https://github.com/twitter/util/blob/develop/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BlacklistStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala similarity index 55% rename from util-stats/src/main/scala/com/twitter/finagle/stats/BlacklistStatsReceiver.scala rename to util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala index 0c1e4ed2a7..a158bcca4c 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BlacklistStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala @@ -1,15 +1,15 @@ package com.twitter.finagle.stats /** - * A blacklisting [[StatsReceiver]]. If the name for a metric is found to be - * blacklisted, nothing is recorded. + * A denylisting [[StatsReceiver]]. If the name for a metric is found to be + * denylisted, nothing is recorded. * * @param self a base [[StatsReceiver]], used for metrics that aren't - * blacklisted - * @param blacklisted a predicate that reads a name and returns true to - * blacklist, and false to let it pass through + * denylisted + * @param denylisted a predicate that reads a name and returns true to + * denylist, and false to let it pass through */ -class BlacklistStatsReceiver(protected val self: StatsReceiver, blacklisted: Seq[String] => Boolean) +class DenylistStatsReceiver(protected val self: StatsReceiver, denylisted: Seq[String] => Boolean) extends StatsReceiverProxy { override def counter(verbosity: Verbosity, name: String*): Counter = @@ -22,7 +22,7 @@ class BlacklistStatsReceiver(protected val self: StatsReceiver, blacklisted: Seq getStatsReceiver(name).addGauge(verbosity, name: _*)(f) private[this] def getStatsReceiver(name: Seq[String]): StatsReceiver = - if (blacklisted(name)) NullStatsReceiver else self + if (denylisted(name)) NullStatsReceiver else self - override def toString: String = s"BlacklistStatsReceiver($self)" + override def toString: String = s"DenylistStatsReceiver($self)" } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala index 287220fc5f..5c60e415e2 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala @@ -17,9 +17,9 @@ class DelegatingStatsReceiverTest extends FunSuite with GeneratorDrivenPropertyC Gen.const(new InMemoryStatsReceiver()) } - private[this] def blacklistStatsReceiver(depth: Int): Gen[StatsReceiver] = + private[this] def denylistStatsReceiver(depth: Int): Gen[StatsReceiver] = if (depth > 3) inMemoryStatsReceiver - else statsReceiverTopology(depth).map(new BlacklistStatsReceiver(_, { case _ => false })) + else statsReceiverTopology(depth).map(new DenylistStatsReceiver(_, { case _ => false })) private[this] def rollupStatsReceiver(depth: Int): Gen[StatsReceiver] = if (depth > 3) inMemoryStatsReceiver @@ -58,7 +58,7 @@ class DelegatingStatsReceiverTest extends FunSuite with GeneratorDrivenPropertyC private[this] def statsReceiverTopology(depth: Int): Gen[StatsReceiver] = { val seq = Seq( inMemoryStatsReceiver, - Gen.lzy(blacklistStatsReceiver(depth + 1)), + Gen.lzy(denylistStatsReceiver(depth + 1)), Gen.lzy(scopedStatsReceiver(depth + 1)), Gen.lzy(broadcastStatsReceiver(depth + 1)), Gen.lzy(proxyStatsReceiver(depth + 1)), diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BlacklistStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala similarity index 69% rename from util-stats/src/test/scala/com/twitter/finagle/stats/BlacklistStatsReceiverTest.scala rename to util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala index 3b232ee316..1dcc9941bd 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/BlacklistStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala @@ -1,15 +1,12 @@ package com.twitter.finagle.stats -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) -class BlacklistStatsReceiverTest extends FunSuite { +class DenylistStatsReceiverTest extends FunSuite { // scalafix:off StoreGaugesAsMemberVariables - test("BlacklistStatsReceiver blacklists properly") { + test("DenylistStatsReceiver denylists properly") { val inmemory = new InMemoryStatsReceiver() - val bsr = new BlacklistStatsReceiver(inmemory, { case _ => true }) + val bsr = new DenylistStatsReceiver(inmemory, { case _ => true }) val ctr = bsr.counter("foo", "bar") ctr.incr() val gauge = bsr.addGauge("foo", "baz") { 3.0f } @@ -21,9 +18,9 @@ class BlacklistStatsReceiverTest extends FunSuite { assert(inmemory.stats.isEmpty) } - test("BlacklistStatsReceiver allows through properly") { + test("DenylistStatsReceiver allows through properly") { val inmemory = new InMemoryStatsReceiver() - val bsr = new BlacklistStatsReceiver(inmemory, { case _ => false }) + val bsr = new DenylistStatsReceiver(inmemory, { case _ => false }) val ctr = bsr.counter("foo", "bar") ctr.incr() val gauge = bsr.addGauge("foo", "baz") { 3.0f } @@ -35,9 +32,9 @@ class BlacklistStatsReceiverTest extends FunSuite { assert(inmemory.stats == Map(Seq("qux") -> Seq(3.0))) } - test("BlacklistStatsReceiver can go both ways properly") { + test("DenylistStatsReceiver can go both ways properly") { val inmemory = new InMemoryStatsReceiver() - val bsr = new BlacklistStatsReceiver(inmemory, { case seq => seq.length != 2 }) + val bsr = new DenylistStatsReceiver(inmemory, { case seq => seq.length != 2 }) val ctr = bsr.counter("foo", "bar") ctr.incr() val gauge = bsr.addGauge("foo", "baz") { 3.0f } @@ -49,10 +46,10 @@ class BlacklistStatsReceiverTest extends FunSuite { assert(inmemory.stats.isEmpty) } - test("BlacklistStatsReceiver works scoped") { + test("DenylistStatsReceiver works scoped") { val inmemory = new InMemoryStatsReceiver() val bsr = - new BlacklistStatsReceiver(inmemory, { case seq => seq == Seq("foo", "bar") }).scope("foo") + new DenylistStatsReceiver(inmemory, { case seq => seq == Seq("foo", "bar") }).scope("foo") val ctr = bsr.counter("foo", "bar") ctr.incr() val gauge = bsr.addGauge("foo", "baz") { 3.0f } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 3c600a0524..79bd3a8421 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -143,8 +143,8 @@ class StatsReceiverTest extends FunSuite { assert("NullStatsReceiver" == NullStatsReceiver.scope("hi").scopeSuffix("bye").toString) assert( - "BlacklistStatsReceiver(NullStatsReceiver)" == - new BlacklistStatsReceiver(NullStatsReceiver, { _ => + "DenylistStatsReceiver(NullStatsReceiver)" == + new DenylistStatsReceiver(NullStatsReceiver, { _ => false }).toString ) From 60eff0a430fe16912accc5dec4b8c9f574ddc588 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 8 Feb 2019 20:05:22 +0000 Subject: [PATCH 107/796] util-collection: Delete it Problem util-collection does not have very much stuff in it anymore, and of those things they don't seem to be very valuable. Solution Let's delete it. Result Never have to worry about util-collection ever again. JIRA Issues: CSL-6493 Differential Revision: https://phabricator.twitter.biz/D270548 --- CHANGELOG.rst | 8 + README.md | 12 - build.sbt | 13 -- util-collection/BUILD | 12 - util-collection/src/main/scala/BUILD | 17 -- .../src/main/scala/PantsWorkaroundColl.scala | 4 - .../main/scala/com/twitter/collection/BUILD | 14 -- .../collection/GenerationalQueue.scala | 160 ------------- .../com/twitter/collection/RecordSchema.scala | 221 ------------------ .../src/main/scala/com/twitter/util/BUILD | 8 - .../scala/com/twitter/util/ImmutableLRU.scala | 103 -------- .../main/scala/com/twitter/util/LruMap.scala | 96 -------- .../com/twitter/util/MapToSetAdapter.scala | 17 -- util-collection/src/test/scala/BUILD | 6 - .../test/scala/com/twitter/collection/BUILD | 11 - .../collection/GenerationalQueueTest.scala | 132 ----------- .../twitter/collection/RecordSchemaTest.scala | 130 ----------- .../src/test/scala/com/twitter/util/BUILD | 10 - .../com/twitter/util/ImmutableLRUTest.scala | 76 ------ .../twitter/util/LRUEntriesGenerator.scala | 20 -- .../scala/com/twitter/util/LRUMapTest.scala | 50 ---- 21 files changed, 8 insertions(+), 1112 deletions(-) delete mode 100644 util-collection/BUILD delete mode 100644 util-collection/src/main/scala/BUILD delete mode 100644 util-collection/src/main/scala/PantsWorkaroundColl.scala delete mode 100644 util-collection/src/main/scala/com/twitter/collection/BUILD delete mode 100644 util-collection/src/main/scala/com/twitter/collection/GenerationalQueue.scala delete mode 100644 util-collection/src/main/scala/com/twitter/collection/RecordSchema.scala delete mode 100644 util-collection/src/main/scala/com/twitter/util/BUILD delete mode 100644 util-collection/src/main/scala/com/twitter/util/ImmutableLRU.scala delete mode 100644 util-collection/src/main/scala/com/twitter/util/LruMap.scala delete mode 100644 util-collection/src/main/scala/com/twitter/util/MapToSetAdapter.scala delete mode 100644 util-collection/src/test/scala/BUILD delete mode 100644 util-collection/src/test/scala/com/twitter/collection/BUILD delete mode 100644 util-collection/src/test/scala/com/twitter/collection/GenerationalQueueTest.scala delete mode 100644 util-collection/src/test/scala/com/twitter/collection/RecordSchemaTest.scala delete mode 100644 util-collection/src/test/scala/com/twitter/util/BUILD delete mode 100644 util-collection/src/test/scala/com/twitter/util/ImmutableLRUTest.scala delete mode 100644 util-collection/src/test/scala/com/twitter/util/LRUEntriesGenerator.scala delete mode 100644 util-collection/src/test/scala/com/twitter/util/LRUMapTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a027c302fa..cae399bf09 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -39,6 +39,14 @@ Breaking API Changes `c.t.u.CountDownLatch#within(Duration)`, please throw an exception yourself after awaiting. ``PHAB_ID=D269404`` +* util-collection: Delete util-collection. We deleted `GenerationalQueue`, `MapToSetAdapter`, and + `ImmutableLRU`, because we found that they were of little utility. We deleted `LruMap` because it + was a very thin shim around a `j.u.LinkedHashMap`, where you override `removeEldestEntry`. If you + need `SynchronizedLruMap`, you can wrap your `LinkedHashMap` with + `j.u.Collection.synchronizedMap`. We moved `RecordSchema` into finagle-base-http because it was + basically only used for HTTP messages, so its new package name is `c.t.f.http.collection`. + ``PHAB_ID=D270548`` + * util-core: Rename `BlacklistStatsReceiver` to `DenylistStatsReceiver`. ``PHAB_ID=D270526`` 19.1.0 diff --git a/README.md b/README.md index 67427832fa..59016c4d42 100644 --- a/README.md +++ b/README.md @@ -89,18 +89,6 @@ for { } ``` -# Collections - -## LruMap - -The LruMap is an LRU with a maximum size passed in. If the map is full it expires items in FIFO order. Reading a value will move an item to the top of the stack. - -```scala -import com.twitter.util.LruMap - -val map = new LruMap[String, String](15) // this is of type mutable.Map[String, String] -``` - # Object Pool The pool order is FIFO. diff --git a/build.sbt b/build.sbt index 25076315ba..0f20b71d45 100644 --- a/build.sbt +++ b/build.sbt @@ -124,7 +124,6 @@ lazy val util = Project( utilCacheGuava, utilClassPreloader, utilCodec, - utilCollection, utilCore, utilDoc, utilFunction, @@ -203,18 +202,6 @@ lazy val utilCodec = Project( name := "util-codec" ).dependsOn(utilCore) -lazy val utilCollection = Project( - id = "util-collection", - base = file("util-collection") -).settings( - sharedSettings -).settings( - name := "util-collection", - libraryDependencies ++= Seq( - scalacheckLib - ) -).dependsOn(utilCore % "compile->compile;test->test") - lazy val utilCore = Project( id = "util-core", base = file("util-core") diff --git a/util-collection/BUILD b/util-collection/BUILD deleted file mode 100644 index ef294a8e31..0000000000 --- a/util-collection/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -target( - dependencies = [ - "util/util-collection/src/main/scala", - ], -) - -target( - name = "tests", - dependencies = [ - "util/util-collection/src/test/scala", - ], -) diff --git a/util-collection/src/main/scala/BUILD b/util-collection/src/main/scala/BUILD deleted file mode 100644 index 8857ace469..0000000000 --- a/util-collection/src/main/scala/BUILD +++ /dev/null @@ -1,17 +0,0 @@ -scala_library( - # This target exists only to act as an aggregator jar. - sources = ["PantsWorkaroundColl.scala"], - provides = scala_artifact( - org = "com.twitter", - name = "util-collection", - repo = artifactory, - ), - dependencies = [ - "util/util-collection/src/main/scala/com/twitter/collection", - "util/util-collection/src/main/scala/com/twitter/util", - ], - exports = [ - "util/util-collection/src/main/scala/com/twitter/collection", - "util/util-collection/src/main/scala/com/twitter/util", - ], -) diff --git a/util-collection/src/main/scala/PantsWorkaroundColl.scala b/util-collection/src/main/scala/PantsWorkaroundColl.scala deleted file mode 100644 index b4b36f594b..0000000000 --- a/util-collection/src/main/scala/PantsWorkaroundColl.scala +++ /dev/null @@ -1,4 +0,0 @@ -// workaround for https://github.com/pantsbuild/pants/issues/6678 -private final class PantsWorkaroundColl { - throw new IllegalStateException("workaround for https://github.com/pantsbuild/pants/issues/6678") -} diff --git a/util-collection/src/main/scala/com/twitter/collection/BUILD b/util-collection/src/main/scala/com/twitter/collection/BUILD deleted file mode 100644 index b615a1425b..0000000000 --- a/util-collection/src/main/scala/com/twitter/collection/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -scala_library( - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-collection-collection", - repo = artifactory, - ), - dependencies = [ - "util/util-core/src/main/scala/com/twitter/util", - ], - exports = [ - "util/util-core/src/main/scala/com/twitter/util", - ], -) diff --git a/util-collection/src/main/scala/com/twitter/collection/GenerationalQueue.scala b/util-collection/src/main/scala/com/twitter/collection/GenerationalQueue.scala deleted file mode 100644 index a14d65dac9..0000000000 --- a/util-collection/src/main/scala/com/twitter/collection/GenerationalQueue.scala +++ /dev/null @@ -1,160 +0,0 @@ -package com.twitter.collection - -import scala.collection._ - -import com.twitter.util.{Duration, Time} - -trait GenerationalQueue[A] { - def touch(a: A): Unit - def add(a: A): Unit - def remove(a: A): Unit - def collect(d: Duration): Option[A] - def collectAll(d: Duration): Iterable[A] -} - -/** - * Generational Queue keep track of elements based on their last activity. - * You can refresh activity of an element by calling touch(a: A) on it. - * There is 2 ways of retrieving old elements: - * - collect(age: Duration) collect the oldest element (age of elem must be > age) - * - collectAll(age: Duration) collect all the elements which age > age in parameter - */ -class ExactGenerationalQueue[A] extends GenerationalQueue[A] { - private[this] val container = mutable.HashMap.empty[A, Time] - private[this] implicit val ordering = Ordering.by[(A, Time), Time] { case (_, ts) => ts } - - /** - * touch insert the element if it is not yet present - */ - def touch(a: A) = synchronized { container.update(a, Time.now) } - - def add(a: A) = synchronized { container += ((a, Time.now)) } - - def remove(a: A) = synchronized { container.remove(a) } - - def collect(age: Duration): Option[A] = synchronized { - if (container.isEmpty) - None - else - container.min match { - case (a, t) if (t.untilNow > age) => Some(a) - case _ => None - } - } - - def collectAll(age: Duration): Iterable[A] = synchronized { - (container filter { case (_, t) => t.untilNow > age }).keys - } -} - -/** - * Improved GenerationalQueue: using a list of buckets responsible for containing elements belonging - * to a slice of time. - * For instance: 3 Buckets, First contains elements from 0 to 10, second elements from 11 to 20 - * and third elements from 21 to 30 - * We expand the list when we need a new bucket, and compact the list to stash all old buckets - * into one. - * There is a slightly difference with the other implementation, when we collect elements we only - * choose randomly an element in the oldest bucket, as we don't have activity date in this bucket - * we consider the worst case and then we can miss some expired elements by never find elements - * that aren't expired. - */ -class BucketGenerationalQueue[A](timeout: Duration) extends GenerationalQueue[A] { - object TimeBucket { - def empty[B] = new TimeBucket[B](Time.now, timeSlice) - } - class TimeBucket[B](var origin: Time, var span: Duration) extends mutable.HashSet[B] { - - // return the age of the potential youngest element of the bucket (may be negative if the - // bucket is not yet expired) - def age(now: Time = Time.now): Duration = (origin + span).until(now) - - def ++=(other: TimeBucket[B]) = { - origin = List(origin, other.origin).min - span = List(origin + span, other.origin + other.span).max - origin - - super.++=(other) - this - } - - override def toString() = "TimeBucket(origin=%d, size=%d, age=%s, count=%d)".format( - origin.inMilliseconds, - span.inMilliseconds, - age().toString, - super.size - ) - } - - private[this] val timeSlice = timeout / 3 - private[this] var buckets = List[TimeBucket[A]]() - - private[this] def maybeGrowChain() = { - // NB: age of youngest element is negative when bucket isn't expired - val growChain = buckets.headOption - .map((bucket) => { - bucket.age() > Duration.Zero - }) - .getOrElse(true) - - if (growChain) - buckets = TimeBucket.empty[A] :: buckets - growChain - } - - private[this] def compactChain(): List[TimeBucket[A]] = { - val now = Time.now - // partition equivalent to takeWhile/dropWhile because buckets are ordered - val (news, olds) = buckets.partition(_.age(now) < timeout) - if (olds.isEmpty) - news - else { - val tailBucket = olds.head - olds drop 1 foreach { tailBucket ++= _ } - if (tailBucket.isEmpty) - news - else - news ::: List(tailBucket) - } - } - - private[this] def updateBuckets(): Unit = { - if (maybeGrowChain()) - buckets = compactChain() - } - - def touch(a: A) = synchronized { - buckets drop 1 foreach { _.remove(a) } - add(a) - } - - def add(a: A) = synchronized { - updateBuckets() - buckets.head.add(a) - } - - def remove(a: A) = synchronized { - buckets foreach { _.remove(a) } - buckets = compactChain() - } - - def collect(d: Duration): Option[A] = synchronized { - if (buckets.isEmpty) - return None - - if (buckets.last.isEmpty) - buckets = compactChain() - - if (buckets.isEmpty) - return None - - val oldestBucket = buckets.last - if (d < oldestBucket.age()) - oldestBucket.headOption - else - None - } - - def collectAll(d: Duration): Iterable[A] = synchronized { - (buckets dropWhile (_.age() < d)).flatten - } -} diff --git a/util-collection/src/main/scala/com/twitter/collection/RecordSchema.scala b/util-collection/src/main/scala/com/twitter/collection/RecordSchema.scala deleted file mode 100644 index 3e5cc51eb8..0000000000 --- a/util-collection/src/main/scala/com/twitter/collection/RecordSchema.scala +++ /dev/null @@ -1,221 +0,0 @@ -package com.twitter.collection - -import java.util.IdentityHashMap - -/** - * RecordSchema represents the declaration of a heterogeneous - * [[com.twitter.collection.RecordSchema.Record Record]] type, with - * [[com.twitter.collection.RecordSchema.Field Fields]] that are determined at runtime. A Field - * declares the static type of its associated values, so although the record itself is dynamic, - * field access is type-safe. - * - * Given a RecordSchema declaration `schema`, any number of - * [[com.twitter.collection.RecordSchema.Record Records]] of that schema can be obtained with - * `schema.newRecord`. The type that Scala assigns to this value is what Scala calls a - * "[[http://lampwww.epfl.ch/~amin/dot/fpdt.pdf path-dependent type]]," meaning that - * `schema1.Record` and `schema2.Record` name distinct types. The same is true of fields: - * `schema1.Field[A]` and `schema2.Field[A]` are distinct, and can only be used with the - * corresponding Record. - */ -final class RecordSchema { - - private[RecordSchema] final class Entry(var value: Any) { - var locked: Boolean = false - } - - /** - * Record is an instance of a [[com.twitter.collection.RecordSchema RecordSchema]] declaration. - * Records are mutable; the `update` method assigns or reassigns a value to a given field. If - * the user requires that a field's assigned value is never reassigned later, the user can `lock` - * that field. - * - * '''Note that this implementation is not synchronized.''' If multiple threads access a record - * concurrently, and at least one of the threads modifies the record, it ''must'' be synchronized - * externally. - */ - final class Record private[RecordSchema] ( - fields: IdentityHashMap[Field[_], Entry] = new IdentityHashMap[Field[_], Entry]) { - - private[this] def getOrInitializeEntry(field: Field[_]): Entry = { - var entry = fields.get(field) - if (entry eq null) { - entry = new Entry(field.default()) - fields.put(field, entry) - } - entry - } - - /** - * Returns the current value assigned to `field` in this record. If there is no value currently - * assigned (either explicitly by a previous `update`, or by the field's declared default), this - * will throw an `IllegalStateException`, indicating the field was never initialized. - * - * Note that Scala provides two syntactic equivalents for invoking this method: - * - * {{{ - * record.apply(field) - * record(field) - * }}} - * - * @param field the field to access in this record - * @return the value associated with `field`. - * @throws IllegalStateException - */ - @throws(classOf[IllegalStateException]) - def apply[A](field: Field[A]): A = - getOrInitializeEntry(field).value.asInstanceOf[A] - - /** - * Locks the current value for a given `field` in this record, preventing further `update`s. If - * there is no value currently assigned (either explicitly by a previous `update`, or by the - * field's declared default), this will throw an `IllegalStateException`, indicating the field - * was never initialized. - * - * @param field the field to lock in this record - * @return this record - * @throws IllegalStateException - */ - @throws(classOf[IllegalStateException]) - def lock(field: Field[_]): Record = { - getOrInitializeEntry(field).locked = true - this - } - - /** - * Assigns (or reassigns) a `value` to a `field` in this record. If this field was previously - * `lock`ed, this will throw an `IllegalStateException` to indicate failure. - * - * Note that Scala provides two syntactic equivalents for invoking this method: - * - * {{{ - * record.update(field, value) - * record(field) = value - * }}} - * - * @param field the field to assign in this record - * @param value the value to assign to `field` in this record - * @return this record - * @throws IllegalStateException - */ - @throws(classOf[IllegalStateException]) - def update[A](field: Field[A], value: A): Record = { - val entry = fields.get(field) - if (entry eq null) { - fields.put(field, new Entry(value)) - } else if (entry.locked) { - throw new IllegalStateException( - s"attempt to assign $value to a locked field (with current value ${entry.value})" - ) - } else { - entry.value = value - } - this - } - - /** - * Assigns (or reassigns) a `value` to a `field` in this record, and locks it to prevent further - * `update`s. This method is provided for convenience only; the following are equivalent: - * - * {{{ - * record.updateAndLock(field, value) - * record.update(field, value).lock(field) - * }}} - * - * @param field the field to assign and lock in this record - * @param value the value to assign to `field` in this record - * @return this record - * @throws IllegalStateException - */ - @throws(classOf[IllegalStateException]) - def updateAndLock[A](field: Field[A], value: A): Record = - update(field, value).lock(field) - - private[this] def copyFields(): IdentityHashMap[Field[_], Entry] = { - val newFields = new IdentityHashMap[Field[_], Entry] - val iter = fields.entrySet().iterator() - while (iter.hasNext()) { - val kv = iter.next() - val entry = kv.getValue() - val newEntry = new Entry(entry.value) - newEntry.locked = entry.locked - newFields.put(kv.getKey(), newEntry) - } - newFields - } - - /** - * Create a copy of this record. Fields are locked in the copy iff they were locked in the - * original record. - * - * @return a copy of this record - */ - def copy(): Record = { - new Record(copyFields()) - } - - /** - * Create a copy of this record with `value` assigned to `field`. `field` will be locked in the - * copy iff it was present and locked in the original record. If `field` was not present in the - * original then the following are equivalent: - * - * {{{ - * record.copy(field, value) - * record.copy().update(field, value) - * }}} - * - * @param field the field to assign in the copy - * @param value the value to assign to `field` in the copy - * @return a copy of this record - */ - def copy[A](field: Field[A], value: A): Record = { - val newFields = copyFields() - val entry = newFields.get(field) - if (entry eq null) { - newFields.put(field, new Entry(value)) - } else { - entry.value = value - } - new Record(newFields) - } - } - - /** - * Creates a new [[com.twitter.collection.RecordSchema.Record Record]] from this Schema. - * - * @return a new [[com.twitter.collection.RecordSchema.Record Record]] - */ - def newRecord(): Record = new Record - - /** - * Field is a handle used to access some corresponding value in a - * [[com.twitter.collection.RecordSchema.Record Record]]. A field may also declare a default, - * which is computed for each record it is associated with, at most once per record instance. - */ - sealed trait Field[A] { - def default(): A - } - - /** - * Creates a new [[com.twitter.collection.RecordSchema.Field Field]] with no default value, to be - * used only with [[com.twitter.collection.RecordSchema.Record Records]] from this schema. - * - * @return a [[com.twitter.collection.RecordSchema.Field Field]] with no default value - */ - def newField[A](): Field[A] = new Field[A] { - override def default(): A = - throw new IllegalStateException("attempt to access uninitialized field") - } - - /** - * Creates a new [[com.twitter.collection.RecordSchema.Field Field]] with the given - * `defaultSupplier`, to be used only with [[com.twitter.collection.RecordSchema.Record Records]] - * from this schema. - * - * @param defaultSupplier a computation producing the default value to use, when there is no value - * previously assigned to this field in a given record. - * @return a [[com.twitter.collection.RecordSchema.Field Field]] with the given `defaultSupplier` - */ - def newField[A](defaultSupplier: => A): Field[A] = new Field[A] { - override def default(): A = defaultSupplier - } -} diff --git a/util-collection/src/main/scala/com/twitter/util/BUILD b/util-collection/src/main/scala/com/twitter/util/BUILD deleted file mode 100644 index e44adb8f25..0000000000 --- a/util-collection/src/main/scala/com/twitter/util/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -scala_library( - compiler_option_sets = {"fatal_warnings"}, - provides = scala_artifact( - org = "com.twitter", - name = "util-collection-util", - repo = artifactory, - ), -) diff --git a/util-collection/src/main/scala/com/twitter/util/ImmutableLRU.scala b/util-collection/src/main/scala/com/twitter/util/ImmutableLRU.scala deleted file mode 100644 index 17581d53d4..0000000000 --- a/util-collection/src/main/scala/com/twitter/util/ImmutableLRU.scala +++ /dev/null @@ -1,103 +0,0 @@ -package com.twitter.util - -import scala.collection.SortedMap - -/** - * Immutable implementation of an LRU cache. - */ -object ImmutableLRU { - - /** - * Build an immutable LRU key/value store that cannot grow larger than `maxSize` - */ - def apply[K, V](maxSize: Int): ImmutableLRU[K, V] = { - new ImmutableLRU(maxSize, 0, Map.empty[K, (Long, V)], SortedMap.empty[Long, K]) - } -} - -/** - * An immutable key/value store that evicts the least recently accessed elements - * to stay constrained in a maximum size bound. - */ -// "map" is the backing store used to hold key->(index,value) -// pairs. The index tracks the access time for a particular key. "ord" -// is used to determine the Least-Recently-Used key in "map" by taking -// the minimum index. -class ImmutableLRU[K, V] private ( - maxSize: Int, - idx: Long, - map: Map[K, (Long, V)], - ord: SortedMap[Long, K]) { - - // Scala's SortedMap requires a key ordering; ImmutableLRU doesn't - // care about pulling a minimum value out of the SortedMap, so the - // following kOrd treats every value as equal. - protected implicit val kOrd = new Ordering[K] { def compare(l: K, r: K) = 0 } - - /** - * the number of entries in the cache - */ - def size: Int = map.size - - /** - * the `Set` of all keys in the LRU - * @note accessing this set does not update the element LRU ordering - */ - def keySet: Set[K] = map.keySet - - /** - * Build a new LRU containing the given key/value - * @return a tuple with of the following two items: - * _1 represents the evicted entry (if the given lru is at the maximum size) - * or None if the lru is not at capacity yet. - * _2 is the new lru with the given key/value pair inserted. - */ - def +(kv: (K, V)): (Option[K], ImmutableLRU[K, V]) = { - val (key, value) = kv - val newIdx = idx + 1 - val newMap = map + (key -> ((newIdx, value))) - // Now update the ordered cache: - val baseOrd = map.get(key).map { case (id, _) => ord - id }.getOrElse(ord) - val ordWithNewKey = baseOrd + (newIdx -> key) - // Do we need to remove an old key: - val (evicts, finalMap, finalOrd) = if (ordWithNewKey.size > maxSize) { - val (minIdx, eKey) = ordWithNewKey.min - (Some(eKey), newMap - eKey, ordWithNewKey - minIdx) - } else { - (None, newMap, ordWithNewKey) - } - (evicts, new ImmutableLRU[K, V](maxSize, newIdx, finalMap, finalOrd)) - } - - /** - * If the key is present in the cache, returns the pair of - * Some(value) and the cache with the key's entry as the most recently accessed. - * Else, returns None and the unmodified cache. - */ - def get(k: K): (Option[V], ImmutableLRU[K, V]) = { - val (optionalValue, lru) = remove(k) - val newLru = optionalValue.map { v => - (lru + (k -> v))._2 - } getOrElse (lru) - (optionalValue, newLru) - } - - /** - * If the key is present in the cache, returns the pair of - * Some(value) and the cache with the key removed. - * Else, returns None and the unmodified cache. - */ - def remove(k: K): (Option[V], ImmutableLRU[K, V]) = - map - .get(k) - .map { - case (kidx, v) => - val newMap = map - k - val newOrd = ord - kidx - // Note we don't increase the idx on a remove, only on put: - (Some(v), new ImmutableLRU[K, V](maxSize, idx, newMap, newOrd)) - } - .getOrElse((None, this)) - - override def toString = { "ImmutableLRU(" + map.toList.mkString(",") + ")" } -} diff --git a/util-collection/src/main/scala/com/twitter/util/LruMap.scala b/util-collection/src/main/scala/com/twitter/util/LruMap.scala deleted file mode 100644 index 79a5c32dae..0000000000 --- a/util-collection/src/main/scala/com/twitter/util/LruMap.scala +++ /dev/null @@ -1,96 +0,0 @@ -package com.twitter.util - -import java.{util => ju} -import java.util.LinkedHashMap -import scala.collection.JavaConverters._ -import scala.collection.mutable.{Map, MapLike} - -/** - * A wrapper trait for java.util.Map implementations to make them behave as scala Maps. - * This is useful if you want to have more specifically-typed wrapped objects instead - * of the generic maps returned by JavaConverters - */ -trait JMapWrapperLike[A, B, +Repr <: MapLike[A, B, Repr] with Map[A, B]] - extends Map[A, B] - with MapLike[A, B, Repr] { - def underlying: ju.Map[A, B] - - override def size = underlying.size - - override def get(k: A) = underlying.asScala.get(k) - - override def +=(kv: (A, B)): this.type = { underlying.put(kv._1, kv._2); this } - override def -=(key: A): this.type = { underlying remove key; this } - - override def put(k: A, v: B): Option[B] = underlying.asScala.put(k, v) - - override def update(k: A, v: B): Unit = { underlying.put(k, v) } - - override def remove(k: A): Option[B] = underlying.asScala.remove(k) - - override def clear() = underlying.clear() - - override def empty: Repr = null.asInstanceOf[Repr] - - override def iterator = underlying.asScala.iterator -} - -object LruMap { - - // initial capacity and load factor are the normal defaults for LinkedHashMap - def makeUnderlying[K, V](maxSize: Int): ju.Map[K, V] = - new LinkedHashMap[K, V]( - 16, /* initial capacity */ - 0.75f, /* load factor */ - true /* access order (as opposed to insertion order) */ - ) { - override protected def removeEldestEntry(eldest: ju.Map.Entry[K, V]): Boolean = { - this.size() > maxSize - } - } -} - -/** - * A scala `Map` backed by a [[java.util.LinkedHashMap]] - */ -class LruMap[K, V](val maxSize: Int, val underlying: ju.Map[K, V]) - extends JMapWrapperLike[K, V, LruMap[K, V]] { - override def empty: LruMap[K, V] = new LruMap[K, V](maxSize) - def this(maxSize: Int) = this(maxSize, LruMap.makeUnderlying(maxSize)) -} - -/** - * A synchronized scala `Map` backed by an [[java.util.LinkedHashMap]] - */ -class SynchronizedLruMap[K, V](maxSize: Int, underlying: ju.Map[K, V]) - extends LruMap[K, V](maxSize, ju.Collections.synchronizedMap(underlying)) { - override def get(key: K): Option[V] = synchronized { super.get(key) } - override def iterator: Iterator[(K, V)] = synchronized { super.iterator } - override def +=(kv: (K, V)): this.type = synchronized[this.type] { super.+=(kv) } - override def -=(key: K): this.type = synchronized[this.type] { super.-=(key) } - - override def size: Int = synchronized { super.size } - override def put(key: K, value: V): Option[V] = synchronized { super.put(key, value) } - override def update(key: K, value: V): Unit = synchronized { super.update(key, value) } - override def remove(key: K): Option[V] = synchronized { super.remove(key) } - override def clear(): Unit = synchronized { super.clear() } - override def getOrElseUpdate(key: K, default: => V): V = synchronized { - super.getOrElseUpdate(key, default) - } - override def transform(f: (K, V) => V): this.type = synchronized[this.type] { super.transform(f) } - override def retain(p: (K, V) => Boolean): this.type = synchronized[this.type] { super.retain(p) } - override def values: scala.collection.Iterable[V] = synchronized { super.values } - override def valuesIterator: Iterator[V] = synchronized { super.valuesIterator } - override def clone(): Self = synchronized { super.clone() } - override def foreach[U](f: ((K, V)) => U) = synchronized { super.foreach(f) } - override def apply(key: K): V = synchronized { super.apply(key) } - override def keySet: scala.collection.Set[K] = synchronized { super.keySet } - override def keys: scala.collection.Iterable[K] = synchronized { super.keys } - override def keysIterator: Iterator[K] = synchronized { super.keysIterator } - override def isEmpty: Boolean = synchronized { super.isEmpty } - override def contains(key: K): Boolean = synchronized { super.contains(key) } - override def isDefinedAt(key: K) = synchronized { super.isDefinedAt(key) } - override def empty: SynchronizedLruMap[K, V] = new SynchronizedLruMap[K, V](maxSize) - //!!! todo: also add all other methods - def this(maxSize: Int) = this(maxSize, LruMap.makeUnderlying(maxSize)) -} diff --git a/util-collection/src/main/scala/com/twitter/util/MapToSetAdapter.scala b/util-collection/src/main/scala/com/twitter/util/MapToSetAdapter.scala deleted file mode 100644 index c2d5da5c68..0000000000 --- a/util-collection/src/main/scala/com/twitter/util/MapToSetAdapter.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.util - -import scala.collection.mutable - -class MapToSetAdapter[A](map: mutable.Map[A, A]) extends mutable.Set[A] { - def +=(elem: A): this.type = { - map(elem) = elem - this - } - def -=(elem: A): this.type = { - map -= elem - this - } - override def size: Int = map.size - def iterator: Iterator[A] = map.keysIterator - def contains(elem: A): Boolean = map.contains(elem) -} diff --git a/util-collection/src/test/scala/BUILD b/util-collection/src/test/scala/BUILD deleted file mode 100644 index aa0cffd6bd..0000000000 --- a/util-collection/src/test/scala/BUILD +++ /dev/null @@ -1,6 +0,0 @@ -target( - dependencies = [ - "util/util-collection/src/test/scala/com/twitter/collection", - "util/util-collection/src/test/scala/com/twitter/util", - ], -) diff --git a/util-collection/src/test/scala/com/twitter/collection/BUILD b/util-collection/src/test/scala/com/twitter/collection/BUILD deleted file mode 100644 index 6c4fc61ea8..0000000000 --- a/util-collection/src/test/scala/com/twitter/collection/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -junit_tests( - sources = globs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalatest", - "util/util-collection/src/main/scala/com/twitter/collection", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", - ], -) diff --git a/util-collection/src/test/scala/com/twitter/collection/GenerationalQueueTest.scala b/util-collection/src/test/scala/com/twitter/collection/GenerationalQueueTest.scala deleted file mode 100644 index 28cea593e1..0000000000 --- a/util-collection/src/test/scala/com/twitter/collection/GenerationalQueueTest.scala +++ /dev/null @@ -1,132 +0,0 @@ -package com.twitter.collection - -import org.junit.runner.RunWith -import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner - -import com.twitter.conversions.DurationOps._ -import com.twitter.util.{Duration, Time} - -@RunWith(classOf[JUnitRunner]) -class GenerationalQueueTest extends FunSuite { - - def genericGenerationalQueueTest[A]( - name: String, - newQueue: () => GenerationalQueue[String], - timeout: Duration - ): Unit = { - - test("%s: Don't collect fresh data".format(name)) { - Time.withCurrentTimeFrozen { _ => - val queue = newQueue() - queue.add("foo") - queue.add("bar") - val collectedValue = queue.collect(timeout) - assert(collectedValue.isEmpty) - } - } - - test("%s: Don't collect old data recently refreshed".format(name)) { - var t = Time.now - Time.withTimeFunction(t) { _ => - val queue = newQueue() - queue.add("foo") - t += timeout * 3 - queue.add("bar") - t += timeout * 3 - queue.touch("foo") - t += timeout * 3 - val collectedValue = queue.collect(timeout) - assert(!collectedValue.isEmpty) - assert(collectedValue.get == "bar") - } - } - - test("%s: collect one old data".format(name)) { - var t = Time.now - Time.withTimeFunction(t) { _ => - val queue = newQueue() - queue.add("foo1") - queue.add("foo2") - queue.add("foo3") - - t += timeout * 3 - queue.add("bar") - - val collectedData = queue.collect(timeout + 1.millisecond) - assert(!collectedData.isEmpty) - assert(collectedData.get.indexOf("foo") != -1) - } - } - - test("%s: collectAll old data".format(name)) { - var t = Time.now - Time.withTimeFunction(t) { _ => - val queue = newQueue() - queue.add("foo") - queue.add("bar") - - t += timeout - queue.add("foo2") - queue.add("bar2") - - t += timeout - val collectedValues = queue.collectAll(timeout + 1.millisecond) - assert(collectedValues.size == 2) - assert(collectedValues.toSeq.contains("foo")) - assert(collectedValues.toSeq.contains("bar")) - } - } - - test("%s: generate a long chain of bucket (if applicable)".format(name)) { - var t = Time.now - Time.withTimeFunction(t) { _ => - val queue = newQueue() - queue.add("foo") - t += timeout * 2 - queue.add("bar") - t += timeout * 2 - queue.add("foobar") - t += timeout * 2 - queue.add("barfoo") - - val collectedValues = queue.collect(timeout + 1.millisecond) - assert(!collectedValues.isEmpty) - assert(List("foo", "bar").contains(collectedValues.get)) - } - } - - test("%s: Don't collect old data".format(name)) { - var t = Time.now - Time.withTimeFunction(t) { _ => - val queue = newQueue() - t += 3.seconds - queue.add("1") - queue.add("2") - queue.add("3") - t += 4.seconds - assert(!queue.collect(2.seconds).isEmpty) - } - } - } - - val generationalQueue = () => new ExactGenerationalQueue[String]() - genericGenerationalQueueTest("ExactGenerationalQueue", generationalQueue, 100.milliseconds) - - val timeout = 100.milliseconds - val bucketQueue = () => new BucketGenerationalQueue[String](timeout) - genericGenerationalQueueTest("BucketGenerationalQueue", bucketQueue, timeout) - - test("BucketGenerationalQueue check removals") { - var t = Time.now - Time.withTimeFunction(t) { _ => - val queue = bucketQueue() - queue.add("1") - queue.add("2") - queue.remove("1") - queue.remove("2") - t += timeout * 3 - assert(queue.collect(timeout).isEmpty) - } - } -} diff --git a/util-collection/src/test/scala/com/twitter/collection/RecordSchemaTest.scala b/util-collection/src/test/scala/com/twitter/collection/RecordSchemaTest.scala deleted file mode 100644 index 913cf21bc2..0000000000 --- a/util-collection/src/test/scala/com/twitter/collection/RecordSchemaTest.scala +++ /dev/null @@ -1,130 +0,0 @@ -package com.twitter.collection - -import org.junit.runner.RunWith -import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner - -@RunWith(classOf[JUnitRunner]) -class RecordSchemaTest extends FunSuite { - - val schema = new RecordSchema - val field = schema.newField[Object]() - val fieldWithDefault = schema.newField[Object](new Object) - val fields = Seq(field, fieldWithDefault) - - test("apply should throw IllegalStateException when field is uninitialized") { - val record = schema.newRecord() - intercept[IllegalStateException] { - record(field) - } - } - - test("apply should compute, store and return default when field is initialized with default") { - val record = schema.newRecord() - assert(record(fieldWithDefault) eq record(fieldWithDefault)) - } - - test("apply should return field value when field is explicitly initialized") { - val record = schema.newRecord() - val value = new Object - - for (f <- fields) { - record(f) = value - assert(record(f) eq value) - } - } - - test("lock should throw IllegalStateException when field is uninitialized") { - val record = schema.newRecord() - intercept[IllegalStateException] { - record.lock(field) - } - } - - test("lock should compute and store default when field is initialized with default") { - val record = schema.newRecord() - record.lock(fieldWithDefault) - assert(record(fieldWithDefault) ne null) - } - - test("update should reassign when field is not locked") { - val record = schema.newRecord() - val value = new Object - - for (f <- fields) { - record(f) = new Object - record(f) = value - assert(record(f) eq value) - } - } - - test("update should throw IllegalStateException when field is locked") { - val record = schema.newRecord() - val value = new Object - - for (f <- fields) { - record(f) = value - record.lock(f) - intercept[IllegalStateException] { - record(f) = value - } - } - } - - test("updateAndLock should update and lock") { - val record = schema.newRecord() - val value = new Object - - for (f <- fields) { - record.updateAndLock(f, value) - intercept[IllegalStateException] { - record(f) = value - } - } - } - - test("copy should copy") { - val record = schema.newRecord() - val value = new Object - - for (f <- fields) { - record.update(f, value) - val copy = record.copy() - assert(record(f) eq copy(f)) - } - } - - test("copy should not be modified when the original is updated") { - val record = schema.newRecord() - val copy = record.copy - - for (f <- fields) { - record.update(f, new Object) - val copy = record.copy() - record.update(f, new Object) - assert(record(f) ne copy(f)) - } - } - - test("locked state should be copied") { - val record = schema.newRecord() - - for (f <- fields) { - record.updateAndLock(f, new Object) - intercept[IllegalStateException] { - record.copy().update(f, new Object) - } - } - } - - test("copy should be able to overwrite a locked field") { - val record = schema.newRecord() - - for (f <- fields) { - record.updateAndLock(f, new Object) - val value = new Object - val copy = record.copy(f, value) - assert(copy(f) eq value) - } - } -} diff --git a/util-collection/src/test/scala/com/twitter/util/BUILD b/util-collection/src/test/scala/com/twitter/util/BUILD deleted file mode 100644 index 2ce910c542..0000000000 --- a/util-collection/src/test/scala/com/twitter/util/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -junit_tests( - sources = globs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalacheck", - "3rdparty/jvm/org/scalatest", - "util/util-collection/src/main/scala/com/twitter/util", - ], -) diff --git a/util-collection/src/test/scala/com/twitter/util/ImmutableLRUTest.scala b/util-collection/src/test/scala/com/twitter/util/ImmutableLRUTest.scala deleted file mode 100644 index f5a85ee154..0000000000 --- a/util-collection/src/test/scala/com/twitter/util/ImmutableLRUTest.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.util - -import org.junit.runner.RunWith -import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -import org.scalacheck.Gen -import org.scalacheck.Arbitrary.arbitrary -import org.scalatest.prop.GeneratorDrivenPropertyChecks - -@RunWith(classOf[JUnitRunner]) -class ImmutableLRUTest extends FunSuite with GeneratorDrivenPropertyChecks { - - // don't waste too much time testing this and keep things small - implicit override val generatorDrivenConfig = - PropertyCheckConfiguration(minSuccessful = 5, minSize = 2, sizeRange = 8) - - test("ImmutableLRU insertion") { - forAll(Gen.zip(Gen.identifier, arbitrary[Int])) { - case (s: String, i: Int) => - val lru = ImmutableLRU[String, Int](50) - - // first-time insertion should not evict - val (key, lru2) = lru + (s -> i) - assert(key == None) - - // test get method - val (key2, _) = lru2.get(s) - assert(key2 == Some(i)) - } - } - - // given a list of entries, build an LRU containing them - private def buildLRU[V]( - lru: ImmutableLRU[String, V], - entries: List[(String, V)] - ): ImmutableLRU[String, V] = { - entries match { - case Nil => lru - case head :: tail => buildLRU((lru + head)._2, tail) - } - } - - test("ImmutableLRU eviction") { - forAll(LRUEntriesGenerator[Int]) { entries => - val lru = buildLRU(ImmutableLRU[String, Int](4), entries) - assert(lru.keySet == entries.map(_._1).slice(entries.size - 4, entries.size).toSet) - } - } - - test("ImmutableLRU removal") { - val gen = for { - entries <- LRUEntriesGenerator[Double] - entry <- Gen.oneOf(entries) - } yield (entries, entry._1, entry._2) - - forAll(gen) { - case (entries, k, v) => - val lru = buildLRU(ImmutableLRU[String, Double](entries.size + 1), entries) - - // make sure it's there - assert(lru.get(k)._1 == Some(v)) - - // remove once - val (key, newLru) = lru.remove(k) - assert(key == Some(v)) - - // remove it again - val (key2, _) = newLru.remove(k) - assert(key2 == None) - - // still should be able to get it out of the original - val (key3, _) = lru.remove(k) - assert(key3 == Some(v)) - } - } -} diff --git a/util-collection/src/test/scala/com/twitter/util/LRUEntriesGenerator.scala b/util-collection/src/test/scala/com/twitter/util/LRUEntriesGenerator.scala deleted file mode 100644 index fdbdba0399..0000000000 --- a/util-collection/src/test/scala/com/twitter/util/LRUEntriesGenerator.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.util - -import org.scalacheck.Arbitrary.arbitrary -import org.scalacheck.{Gen, Arbitrary} - -object LRUEntriesGenerator { - - /** - * Generate a nonempty list of map entries keyed with strings - * and having any arbitrarily typed values. - */ - def apply[V: Arbitrary]: Gen[List[(String, V)]] = { - val gen = for { - size <- Gen.choose(1, 200) - uniqueKeys <- Gen.listOfN(size, Gen.identifier).map(_.toSet) - vals <- Gen.listOfN(uniqueKeys.size, arbitrary[V]) - } yield uniqueKeys.toList.zip(vals) - gen suchThat (_.nonEmpty) - } -} diff --git a/util-collection/src/test/scala/com/twitter/util/LRUMapTest.scala b/util-collection/src/test/scala/com/twitter/util/LRUMapTest.scala deleted file mode 100644 index 2dea99015b..0000000000 --- a/util-collection/src/test/scala/com/twitter/util/LRUMapTest.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.util - -import org.junit.runner.RunWith -import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -import org.scalacheck.Gen -import org.scalatest.prop.GeneratorDrivenPropertyChecks - -@RunWith(classOf[JUnitRunner]) -class LRUMapTest extends FunSuite with GeneratorDrivenPropertyChecks { - - // don't waste too much time testing this and keep things small - implicit override val generatorDrivenConfig = - PropertyCheckConfiguration(minSuccessful = 5, minSize = 2, sizeRange = 8) - - test("LRUMap creation") { - forAll(Gen.choose(1, 200)) { size => - val lru = new LruMap[String, String](size) - assert(lru.maxSize == size) - - val slru = new SynchronizedLruMap[String, String](size) - assert(slru.maxSize == size) - } - } - - test("LRUMap insertion") { - forAll(LRUEntriesGenerator[Int]) { entries => - val lru = new LruMap[String, Int](entries.size + 10) // headroom - for (entry <- entries) { - lru += entry - } - - for ((key, value) <- entries) { - assert(lru.get(key) == Some(value)) - } - } - } - - test("LRUMap eviction") { - forAll(LRUEntriesGenerator[Double]) { entries => - val slru = new SynchronizedLruMap[String, Double](5) - for (entry <- entries) { - slru += entry - } - - val expectedKeys = entries.slice(entries.size - 5, entries.size).map(_._1) - assert(slru.keySet == expectedKeys.toSet) - } - } -} From b3083ac2677e4ce1365798304b4cec27ddeaed4b Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 8 Feb 2019 22:33:17 +0000 Subject: [PATCH 108/796] {util, finagle}: remove remnants of util-collection from sbt Problem util-collection isn't a thing, but we're still trying to make it a thing. Solution Stop trying to make util-collection happen JIRA Issues: CSL-7623 Differential Revision: https://phabricator.twitter.biz/D271602 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 0f20b71d45..b83842414d 100644 --- a/build.sbt +++ b/build.sbt @@ -421,7 +421,7 @@ lazy val utilZk = Project( ).settings( name := "util-zk", libraryDependencies += zkDependency -).dependsOn(utilCore, utilCollection, utilLogging) +).dependsOn(utilCore, utilLogging) lazy val utilZkTest = Project( id = "util-zk-test", From de4cebda30c90f2461990e623e8a842f57249534 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Mon, 11 Feb 2019 16:43:54 +0000 Subject: [PATCH 109/796] util-core: Optimize `ConstFuture.proxyTo` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem `Future.flatMap` and `Future.transform` are not as fast as `Future.map` when the function's result is a `ConstFuture`. Solution Optimize `ConstFuture.proxyTo` to use `Promise.update`. Result The implementation now matches or beats `map`. This is because `map` is written in terms of `transformTry` which can use `Promise.update`. This optimization allows this case, and others, to behave like that as well. JMH microbenchmarks: ``` Baseline Benchmark Score Error Units FutureBenchmark.fromConstMap 55.937 ± 2.988 ns/op FutureBenchmark.fromConstMap:·gc.alloc.rate.norm 104.000 ± 0.001 B/op FutureBenchmark.fromConstFlatMapToConst 89.606 ± 6.285 ns/op FutureBenchmark.fromConstFlatMapToConst:·gc.alloc.rate.norm 128.000 ± 0.001 B/op FutureBenchmark.fromConstTransformToConst 80.562 ± 2.750 ns/op FutureBenchmark.fromConstTransformToConst:·gc.alloc.rate.norm 112.000 ± 0.001 B/op Patched Benchmark Score Error Units FutureBenchmark.fromConstMap 54.129 ± 3.030 ns/op FutureBenchmark.fromConstMap:·gc.alloc.rate.norm 104.000 ± 0.001 B/op FutureBenchmark.fromConstFlatMapToConst 54.044 ± 9.021 ns/op FutureBenchmark.fromConstFlatMapToConst:·gc.alloc.rate.norm 88.000 ± 0.001 B/op FutureBenchmark.fromConstTransformToConst 50.474 ± 3.087 ns/op FutureBenchmark.fromConstTransformToConst:·gc.alloc.rate.norm 72.000 ± 0.001 B/op ``` JIRA Issues: CSL-7605 Differential Revision: https://phabricator.twitter.biz/D271358 --- CHANGELOG.rst | 9 ++++++--- .../scala/com/twitter/util/FutureBenchmark.scala | 15 +++++++++++++++ .../main/scala/com/twitter/util/ConstFuture.scala | 5 +++++ .../src/main/scala/com/twitter/util/Promise.scala | 10 ++++++---- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cae399bf09..22dbb62698 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,12 +16,15 @@ New Features * util-core: Added `Duration.fromMinutes` to return a `Duration` from a given number of minutes. ``PHAB_ID=D259795`` -* util-security: `Pkcs8KeyManagerFactory` now supports a certificates file which contains multiple - certificates that are part of the same certificate chain. ``PHAB_ID=D263190`` - * util-core: If given a `Timer` upon construction, `c.t.io.Pipe` will respect the close deadline and wait the given amount of time for any pending writes to be read. ``PHAB_ID=D229728`` +* util-core: Optimized `ConstFuture.proxyTo` which brings the performance of + `flatMap` and `transform` of a `ConstFuture` in line with `map`. ``PHAB_ID=D271358`` + +* util-security: `Pkcs8KeyManagerFactory` now supports a certificates file which contains multiple + certificates that are part of the same certificate chain. ``PHAB_ID=D263190`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala index 9316880f72..07c6f3e3d9 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala @@ -77,6 +77,18 @@ class FutureBenchmark extends StdBenchAnnotations { new Promise[Unit].flatMap(FlatMapFn) } + @Benchmark + def fromConstMap: Future[Unit] = + Future.Done.map(MapFn) + + @Benchmark + def fromConstFlatMapToConst: Future[Unit] = + Future.Done.flatMap(FlatMapFn) + + @Benchmark + def fromConstTransformToConst: Future[Unit] = + Future.Done.transform(TransformFn) + @Benchmark def timeSelect(state: SelectState): Future[(Try[Unit], Seq[Future[Unit]])] = { import state._ @@ -257,6 +269,9 @@ object FutureBenchmark { private val MapFn: Unit => Unit = { _: Unit => () } + private val TransformFn: Try[Unit] => Future[Unit] = { _: Try[Unit] => + Future.Done + } @State(Scope.Benchmark) class ByState { diff --git a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala index bb8bac1dc7..6e9c35d076 100644 --- a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala @@ -32,6 +32,11 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { this } + override def proxyTo[B >: A](other: Promise[B]): Unit = { + // avoid an extra call to `isDefined` as `update` checks + other.update(result) + } + def raise(interrupt: Throwable): Unit = () protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 3d6710bdc4..44fced4ecc 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -703,14 +703,16 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ * @see [[com.twitter.util.Future.proxyTo]] */ def become(other: Future[A]): Unit = { - if (isDefined) { - val current = Await.result(liftToTry) - throw new IllegalStateException(s"cannot become() on an already satisfied promise: $current") - } if (other.isInstanceOf[Promise[_]]) { + if (isDefined) { + val current = Await.result(liftToTry) + throw new IllegalStateException( + s"cannot become() on an already satisfied promise: $current") + } val that = other.asInstanceOf[Promise[A]] that.link(compress()) } else { + // avoid an extra call to `isDefined` as `proxyTo` checks other.proxyTo(this) forwardInterruptsTo(other) } From 41e24395a0d38c2c64c5451a358e7e130a93f6fe Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 12 Feb 2019 17:22:12 +0000 Subject: [PATCH 110/796] util-core: Collapse Buf.Composite and Buf.Composite.Impl make it private Problem / Solution `Buf.Composite` is a sealed abstract class yet we only supply one implementation. Let's collapse them together and have fewer abstractions. Additionally, we can mark `Buf.Composite` private: nobody should program against it. JIRA Issues: CSL-4652 Differential Revision: https://phabricator.twitter.biz/D270916 --- CHANGELOG.rst | 3 + .../src/main/scala/com/twitter/io/Buf.scala | 217 +++++++----------- .../test/scala/com/twitter/io/BufTest.scala | 20 -- 3 files changed, 92 insertions(+), 148 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 22dbb62698..ad94e03243 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -52,6 +52,9 @@ Breaking API Changes * util-core: Rename `BlacklistStatsReceiver` to `DenylistStatsReceiver`. ``PHAB_ID=D270526`` +* util-core: `Buf.Composite` is now private. Program against more generic, `Buf` interface instead. + ``PHAB_ID=D270916`` + 19.1.0 ------- diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 706782f9f4..8ccba6c07e 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -94,58 +94,58 @@ abstract class Buf { outer => else this match { // This could be much cleaner as a Tuple match, but we want to avoid the allocation. - case Buf.Composite.Impl(left: Vector[Buf], leftLength) => + case Buf.Composite(left: Vector[Buf], leftLength) => right match { - case Buf.Composite.Impl(right: Vector[Buf], rightLength) => - Buf.Composite.Impl(Buf.fastConcat(left, right), leftLength + rightLength) - case Buf.Composite.Impl(right: IndexedTwo, rightLength) => - Buf.Composite.Impl(left :+ right.a :+ right.b, leftLength + rightLength) - case Buf.Composite.Impl(right: IndexedThree, rightLength) => - Buf.Composite.Impl(left :+ right.a :+ right.b :+ right.c, leftLength + rightLength) + case Buf.Composite(right: Vector[Buf], rightLength) => + Buf.Composite(Buf.fastConcat(left, right), leftLength + rightLength) + case Buf.Composite(right: IndexedTwo, rightLength) => + Buf.Composite(left :+ right.a :+ right.b, leftLength + rightLength) + case Buf.Composite(right: IndexedThree, rightLength) => + Buf.Composite(left :+ right.a :+ right.b :+ right.c, leftLength + rightLength) case _ => - Buf.Composite.Impl(left :+ right, leftLength + right.length) + Buf.Composite(left :+ right, leftLength + right.length) } - case Buf.Composite.Impl(left: IndexedTwo, leftLength) => + case Buf.Composite(left: IndexedTwo, leftLength) => right match { - case Buf.Composite.Impl(right: Vector[Buf], rightLength) => - Buf.Composite.Impl(left.a +: left.b +: right, leftLength + rightLength) - case Buf.Composite.Impl(right: IndexedTwo, rightLength) => - Buf.Composite.Impl( + case Buf.Composite(right: Vector[Buf], rightLength) => + Buf.Composite(left.a +: left.b +: right, leftLength + rightLength) + case Buf.Composite(right: IndexedTwo, rightLength) => + Buf.Composite( (new VectorBuilder[Buf] += left.a += left.b += right.a += right.b).result(), leftLength + rightLength ) - case Buf.Composite.Impl(right: IndexedThree, rightLength) => - Buf.Composite.Impl( + case Buf.Composite(right: IndexedThree, rightLength) => + Buf.Composite( (new VectorBuilder[Buf] += left.a += left.b += right.a += right.b += right.c) .result(), leftLength + rightLength ) case _ => - Buf.Composite.Impl( + Buf.Composite( new IndexedThree(left.a, left.b, right), leftLength + right.length ) } - case Buf.Composite.Impl(left: IndexedThree, leftLength) => + case Buf.Composite(left: IndexedThree, leftLength) => right match { - case Buf.Composite.Impl(right: Vector[Buf], rightLength) => - Buf.Composite.Impl(left.a +: left.b +: left.c +: right, leftLength + rightLength) - case Buf.Composite.Impl(right: IndexedTwo, rightLength) => - Buf.Composite.Impl( + case Buf.Composite(right: Vector[Buf], rightLength) => + Buf.Composite(left.a +: left.b +: left.c +: right, leftLength + rightLength) + case Buf.Composite(right: IndexedTwo, rightLength) => + Buf.Composite( (new VectorBuilder[Buf] += left.a += left.b += left.c += right.a += right.b) .result(), leftLength + rightLength ) - case Buf.Composite.Impl(right: IndexedThree, rightLength) => - Buf.Composite.Impl( + case Buf.Composite(right: IndexedThree, rightLength) => + Buf.Composite( (new VectorBuilder[Buf] += left.a += left.b += left.c += right.a += right.b += right.c) .result(), leftLength + rightLength ) case _ => - Buf.Composite.Impl( + Buf.Composite( (new VectorBuilder[Buf] += left.a += left.b += left.c += right).result(), leftLength + right.length ) @@ -153,18 +153,17 @@ abstract class Buf { outer => case left => right match { - case Buf.Composite.Impl(right: Vector[Buf], rightLength) => - Buf.Composite.Impl(left +: right, left.length + rightLength) - case Buf.Composite.Impl(right: IndexedTwo, rightLength) => - Buf.Composite - .Impl(new IndexedThree(left, right.a, right.b), left.length + rightLength) - case Buf.Composite.Impl(right: IndexedThree, rightLength) => - Buf.Composite.Impl( + case Buf.Composite(right: Vector[Buf], rightLength) => + Buf.Composite(left +: right, left.length + rightLength) + case Buf.Composite(right: IndexedTwo, rightLength) => + Buf.Composite(new IndexedThree(left, right.a, right.b), left.length + rightLength) + case Buf.Composite(right: IndexedThree, rightLength) => + Buf.Composite( (new VectorBuilder[Buf] += left += right.a += right.b += right.c).result(), left.length + rightLength ) case _ => - Buf.Composite.Impl(new IndexedTwo(left, right), left.length + right.length) + Buf.Composite(new IndexedTwo(left, right), left.length + right.length) } } } @@ -337,10 +336,10 @@ object Buf { val builder = Vector.newBuilder[Buf] var length = 0 bufs.foreach { - case b: Composite.Impl => + case Composite(b, l) => // Guaranteed to be non-empty by construction - length += b.length - builder ++= b.bufs + length += l + builder ++= b case b => val len = b.length @@ -356,7 +355,7 @@ object Buf { else if (filtered.size == 1) filtered.head else - new Composite.Impl(filtered, length) + Composite(filtered, length) } /** Helps Buf implementations validate the arguments to slicing functions. */ @@ -392,20 +391,10 @@ object Buf { def length: Int = 3 } - /** - * A `Buf` which is composed of other `Bufs`. - * - * @see [[Buf.apply]] for creating new instances. - */ - sealed abstract class Composite extends Buf { - def bufs: IndexedSeq[Buf] - protected def computedLength: Int - + private final case class Composite(bufs: IndexedSeq[Buf], length: Int) extends Buf { // the factory method requires non-empty Bufs override def isEmpty: Boolean = false - def length: Int = computedLength - override def toString: String = s"Buf.Composite(length=$length)" def write(output: Array[Byte], off: Int): Unit = { @@ -488,6 +477,58 @@ object Buf { } } + def get(index: Int): Byte = { + var bufIdx = 0 + var byteIdx = 0 + while (bufIdx < bufs.length) { + val buf = bufs(bufIdx) + val bufLen = buf.length + if (index < byteIdx + bufLen) { + return buf.get(index - byteIdx) + } else { + byteIdx += bufLen + } + bufIdx += 1 + } + throw new IndexOutOfBoundsException(s"Index out of bounds: $index") + } + + def process(from: Int, until: Int, processor: Processor): Int = { + checkSliceArgs(from, until) + if (isSliceEmpty(from, until)) return -1 + var i = 0 + var bufIdx = 0 + var continue = true + while (continue && i < until && bufIdx < bufs.length) { + val buf = bufs(bufIdx) + val bufLen = buf.length + + if (i + bufLen < from) { + // skip ahead to the right Buf for `from` + bufIdx += 1 + i += bufLen + } else { + // ensure we are positioned correctly in the first Buf + var byteIdx = + if (i >= from) 0 + else from - i + val endAt = math.min(bufLen, until - i) + while (continue && byteIdx < endAt) { + val byte = buf.get(byteIdx) + if (processor(byte)) { + byteIdx += 1 + } else { + continue = false + } + } + bufIdx += 1 + i += byteIdx + } + } + if (continue) -1 + else i + } + protected def unsafeByteArrayBuf: Option[ByteArray] = None private[this] def equalsIndexed(other: Buf): Boolean = { @@ -511,7 +552,7 @@ object Buf { override def equals(other: Any): Boolean = other match { case otherBuf: Buf if length == otherBuf.length => otherBuf match { - case Composite(otherBufs) => + case Composite(otherBufs, _) => // this is 2 nested loops, with the outer loop tracking which // Buf's they are on. The inner loop compares individual bytes across // the Bufs "segments". @@ -553,86 +594,6 @@ object Buf { } } - object Composite { - def unapply(buf: Composite): Some[IndexedSeq[Buf]] = - Some(buf.bufs) - - /** - * Basic implementation of a [[Buf]] created from n-`Bufs`. - */ - private[Buf] final case class Impl(bufs: IndexedSeq[Buf], protected val computedLength: Int) - extends Buf.Composite { - - bufs match { - case _: Vector[Buf] => - case _: IndexedTwo => - case _: IndexedThree => - case _ => throw new IllegalArgumentException(s"Type not allowed: ${bufs.getClass.getName}") - } - - // ensure there is a need for a `Composite` - if (bufs.length <= 1) - throw new IllegalArgumentException(s"Must have 2 or more bufs: $bufs") - if (computedLength <= 0) - throw new IllegalArgumentException(s"Length must be positive: $computedLength") - - // the factory method requires non-empty Bufs - override def isEmpty: Boolean = false - - def get(index: Int): Byte = { - var bufIdx = 0 - var byteIdx = 0 - while (bufIdx < bufs.length) { - val buf = bufs(bufIdx) - val bufLen = buf.length - if (index < byteIdx + bufLen) { - return buf.get(index - byteIdx) - } else { - byteIdx += bufLen - } - bufIdx += 1 - } - throw new IndexOutOfBoundsException(s"Index out of bounds: $index") - } - - def process(from: Int, until: Int, processor: Processor): Int = { - checkSliceArgs(from, until) - if (isSliceEmpty(from, until)) return -1 - var i = 0 - var bufIdx = 0 - var continue = true - while (continue && i < until && bufIdx < bufs.length) { - val buf = bufs(bufIdx) - val bufLen = buf.length - - if (i + bufLen < from) { - // skip ahead to the right Buf for `from` - bufIdx += 1 - i += bufLen - } else { - // ensure we are positioned correctly in the first Buf - var byteIdx = - if (i >= from) 0 - else from - i - val endAt = math.min(bufLen, until - i) - while (continue && byteIdx < endAt) { - val byte = buf.get(byteIdx) - if (processor(byte)) { - byteIdx += 1 - } else { - continue = false - } - } - bufIdx += 1 - i += byteIdx - } - } - if (continue) -1 - else i - } - } - } - /** * A buffer representing an array of bytes. */ diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index 569d8184d9..f83b4e01e1 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -615,26 +615,6 @@ class BufTest assert(Buf.Utf8("abcabc") == Buf(Seq(abc, Buf.Empty, abc))) } - test("Buf.Composite.unapply") { - Buf.Empty match { - case Buf.Composite(_) => fail() - case _ => - } - - val abc = Buf.Utf8("abc") - abc match { - case Buf.Composite(_) => fail() - case _ => - } - - val xyz = Buf.Utf8("xyz") - abc.concat(xyz) match { - case Buf.Composite(bufs) => - assert(bufs == IndexedSeq(abc, xyz)) - case _ => fail() - } - } - test("Buf.Utf8: English") { val buf = Buf.Utf8("Hello, world!") assert(buf.length == 13) From df397f5338fb608d20e103307cb57985428fce47 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 12 Feb 2019 22:05:08 +0000 Subject: [PATCH 111/796] util-core: Remove deprecated conversions Problem / Solution It's time for the deprecated `com.twitter.conversions` to be removed. They all have Ops-named analogs. JIRA Issues: CSL-7356 Differential Revision: https://phabricator.twitter.biz/D272206 --- CHANGELOG.rst | 10 ++ .../com/twitter/conversions/percent.scala | 38 ---- .../com/twitter/conversions/storage.scala | 45 ----- .../com/twitter/conversions/string.scala | 163 ------------------ .../com/twitter/conversions/thread.scala | 30 ---- .../scala/com/twitter/conversions/time.scala | 57 ------ .../scala/com/twitter/conversions/u64.scala | 22 --- .../com/twitter/conversions/PercentTest.scala | 50 ------ .../com/twitter/conversions/TimeTest.scala | 20 --- 9 files changed, 10 insertions(+), 425 deletions(-) delete mode 100644 util-core/src/main/scala/com/twitter/conversions/percent.scala delete mode 100644 util-core/src/main/scala/com/twitter/conversions/storage.scala delete mode 100644 util-core/src/main/scala/com/twitter/conversions/string.scala delete mode 100644 util-core/src/main/scala/com/twitter/conversions/thread.scala delete mode 100644 util-core/src/main/scala/com/twitter/conversions/time.scala delete mode 100644 util-core/src/main/scala/com/twitter/conversions/u64.scala delete mode 100644 util-core/src/test/scala/com/twitter/conversions/PercentTest.scala delete mode 100644 util-core/src/test/scala/com/twitter/conversions/TimeTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ad94e03243..86c5a44e39 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -42,6 +42,16 @@ Breaking API Changes `c.t.u.CountDownLatch#within(Duration)`, please throw an exception yourself after awaiting. ``PHAB_ID=D269404`` +* util-core: Deprecated conversions in `c.t.conversions` have new implementations + that follow a naming scheme of `SomethingOps`. ``PHAB_ID=D272206`` + + - `percent` is now `PercentOps` + - `storage` is now `StorageUnitOps` + - `string` is now `StringOps` + - `thread` is now `ThreadOps` + - `time` is now `DurationOps` + - `u64` is now `U64Ops` + * util-collection: Delete util-collection. We deleted `GenerationalQueue`, `MapToSetAdapter`, and `ImmutableLRU`, because we found that they were of little utility. We deleted `LruMap` because it was a very thin shim around a `j.u.LinkedHashMap`, where you override `removeEldestEntry`. If you diff --git a/util-core/src/main/scala/com/twitter/conversions/percent.scala b/util-core/src/main/scala/com/twitter/conversions/percent.scala deleted file mode 100644 index 90a5fa942d..0000000000 --- a/util-core/src/main/scala/com/twitter/conversions/percent.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.twitter.conversions - -import scala.language.implicitConversions - -/** - * Implicits for turning x.percent (where x is an Int or Double) into a Double scaled to - * where 1.0 is 100 percent. - * - * @note Negative values, fractional values, and values greater than 100 are permitted. - * - * @example - * {{{ - * 1.percent == 0.01 - * 100.percent == 1.0 - * 99.9.percent == 0.999 - * 500.percent == 5.0 - * -10.percent == -0.1 - * }}} - */ -@deprecated("Use the AnyVal version `com.twitter.conversions.PercentOps`", "2018-12-05") -object percent { - - private val BigDecimal100 = BigDecimal(100.0) - - class RichDoublePercent private[conversions] (val wrapped: Double) extends AnyVal { - // convert wrapped to BigDecimal to preserve precision when dividing Doubles - def percent: Double = - if (wrapped.equals(Double.NaN) - || wrapped.equals(Double.PositiveInfinity) - || wrapped.equals(Double.NegativeInfinity)) wrapped - else (BigDecimal(wrapped) / BigDecimal100).doubleValue - } - - implicit def intPercentToFractionalDouble(i: Int): RichDoublePercent = - new RichDoublePercent(i) - implicit def doublePercentToFractionalDouble(d: Double): RichDoublePercent = - new RichDoublePercent(d) -} diff --git a/util-core/src/main/scala/com/twitter/conversions/storage.scala b/util-core/src/main/scala/com/twitter/conversions/storage.scala deleted file mode 100644 index a34d5155b7..0000000000 --- a/util-core/src/main/scala/com/twitter/conversions/storage.scala +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2010 Twitter Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter.conversions - -import com.twitter.util.StorageUnit -import scala.language.implicitConversions - -@deprecated("Use the AnyVal version `com.twitter.conversions.StorageUnitOps`", "2018-12-05") -object storage { - class RichWholeNumber(wrapped: Long) { - def byte: StorageUnit = bytes - def bytes: StorageUnit = StorageUnit.fromBytes(wrapped) - def kilobyte: StorageUnit = kilobytes - def kilobytes: StorageUnit = StorageUnit.fromKilobytes(wrapped) - def megabyte: StorageUnit = megabytes - def megabytes: StorageUnit = StorageUnit.fromMegabytes(wrapped) - def gigabyte: StorageUnit = gigabytes - def gigabytes: StorageUnit = StorageUnit.fromGigabytes(wrapped) - def terabyte: StorageUnit = terabytes - def terabytes: StorageUnit = StorageUnit.fromTerabytes(wrapped) - def petabyte: StorageUnit = petabytes - def petabytes: StorageUnit = StorageUnit.fromPetabytes(wrapped) - - def thousand: Long = wrapped * 1000 - def million: Long = wrapped * 1000 * 1000 - def billion: Long = wrapped * 1000 * 1000 * 1000 - } - - implicit def intToStorageUnitableWholeNumber(i: Int): RichWholeNumber = new RichWholeNumber(i) - implicit def longToStorageUnitableWholeNumber(l: Long): RichWholeNumber = new RichWholeNumber(l) -} diff --git a/util-core/src/main/scala/com/twitter/conversions/string.scala b/util-core/src/main/scala/com/twitter/conversions/string.scala deleted file mode 100644 index 7496726d0f..0000000000 --- a/util-core/src/main/scala/com/twitter/conversions/string.scala +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright 2010 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter.conversions - -import scala.language.implicitConversions -import scala.util.matching.Regex - -@deprecated("Use the AnyVal version `com.twitter.conversions.StringOps`", "2018-12-05") -object string { - final class RichString(wrapped: String) { - - /** - * For every section of a string that matches a regular expression, call - * a function to determine a replacement (as in python's - * `re.sub`). The function will be passed the Matcher object - * corresponding to the substring that matches the pattern, and that - * substring will be replaced by the function's result. - * - * For example, this call: - * {{{ - * "ohio".regexSub("""h.""".r) { m => "n" } - * }}} - * will return the string `"ono"`. - * - * The matches are found using `Matcher.find()` and so - * will obey all the normal java rules (the matches will not overlap, - * etc). - * - * @param re the regex pattern to replace - * @param replace a function that takes Regex.MatchData objects and - * returns a string to substitute - * @return the resulting string with replacements made - */ - def regexSub(re: Regex)(replace: Regex.MatchData => String): String = { - var offset = 0 - val out = new StringBuilder() - - for (m <- re.findAllIn(wrapped).matchData) { - if (m.start > offset) { - out.append(wrapped.substring(offset, m.start)) - } - - out.append(replace(m)) - offset = m.end - } - - if (offset < wrapped.length) { - out.append(wrapped.substring(offset)) - } - out.toString - } - - private val QUOTE_RE = "[\u0000-\u001f\u007f-\uffff\\\\\"]".r - - /** - * Quote a string so that unprintable chars (in ASCII) are represented by - * C-style backslash expressions. For example, a raw linefeed will be - * translated into `"\n"`. Control codes (anything below 0x20) - * and unprintables (anything above 0x7E) are turned into either - * `"\xHH"` or `"\\uHHHH"` expressions, depending on - * their range. Embedded backslashes and double-quotes are also quoted. - * - * @return a quoted string, suitable for ASCII display - */ - def quoteC(): String = { - regexSub(QUOTE_RE) { m => - m.matched.charAt(0) match { - case '\r' => "\\r" - case '\n' => "\\n" - case '\t' => "\\t" - case '"' => "\\\"" - case '\\' => "\\\\" - case c => - if (c <= 255) { - "\\x%02x".format(c.asInstanceOf[Int]) - } else { - "\\u%04x" format c.asInstanceOf[Int] - } - } - } - } - - // we intentionally don't unquote "\$" here, so it can be used to escape interpolation later. - private val UNQUOTE_RE = """\\(u[\dA-Fa-f]{4}|x[\dA-Fa-f]{2}|[/rnt\"\\])""".r - - /** - * Unquote an ASCII string that has been quoted in a style like - * [[quoteC()]] and convert it into a standard unicode string. - * `"\\uHHHH"` and `"\xHH"` expressions are unpacked - * into unicode characters, as well as `"\r"`, `"\n"`, - * `"\t"`, `"\\"`, and `'\"'`. - * - * @return an unquoted unicode string - */ - def unquoteC(): String = { - regexSub(UNQUOTE_RE) { m => - val ch = m.group(1).charAt(0) match { - // holy crap! this is terrible: - case 'u' => - Character.valueOf(Integer.valueOf(m.group(1).substring(1), 16).asInstanceOf[Int].toChar) - case 'x' => - Character.valueOf(Integer.valueOf(m.group(1).substring(1), 16).asInstanceOf[Int].toChar) - case 'r' => '\r' - case 'n' => '\n' - case 't' => '\t' - case x => x - } - ch.toString - } - } - - /** - * Turn a string of hex digits into a byte array. This does the exact - * opposite of `Array[Byte]#hexlify`. - */ - def unhexlify(): Array[Byte] = { - val buffer = new Array[Byte]((wrapped.length + 1) / 2) - wrapped.grouped(2).toSeq.zipWithIndex.foreach { - case (substr, i) => - buffer(i) = Integer.parseInt(substr, 16).toByte - } - buffer - } - } - - def hexlify(array: Array[Byte], from: Int, to: Int): String = { - val out = new StringBuffer - for (i <- from until to) { - val b = array(i) - val s = (b.toInt & 0xff).toHexString - if (s.length < 2) { - out append '0' - } - out append s - } - out.toString - } - - final class RichByteArray(wrapped: Array[Byte]) { - - /** - * Turn an `Array[Byte]` into a string of hex digits. - */ - def hexlify: String = string.hexlify(wrapped, 0, wrapped.length) - } - - implicit def stringToConfiggyString(s: String): RichString = new RichString(s) - implicit def byteArrayToConfiggyByteArray(b: Array[Byte]): RichByteArray = new RichByteArray(b) -} diff --git a/util-core/src/main/scala/com/twitter/conversions/thread.scala b/util-core/src/main/scala/com/twitter/conversions/thread.scala deleted file mode 100644 index b03209c153..0000000000 --- a/util-core/src/main/scala/com/twitter/conversions/thread.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2010 Twitter Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter.conversions - -import java.util.concurrent.Callable -import scala.language.implicitConversions - -/** - * Implicits for turning a block of code into a Runnable or Callable. - */ -@deprecated("Use `com.twitter.conversions.ThreadOps`", "2018-12-05") -object thread { - implicit def makeRunnable(f: => Unit): Runnable = new Runnable() { def run(): Unit = f } - - implicit def makeCallable[T](f: => T): Callable[T] = new Callable[T]() { def call(): T = f } -} diff --git a/util-core/src/main/scala/com/twitter/conversions/time.scala b/util-core/src/main/scala/com/twitter/conversions/time.scala deleted file mode 100644 index d8e4d9edf2..0000000000 --- a/util-core/src/main/scala/com/twitter/conversions/time.scala +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2010 Twitter Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter.conversions - -import com.twitter.util.Duration -import java.util.concurrent.TimeUnit -import scala.language.implicitConversions - -@deprecated("Use the AnyVal version `com.twitter.conversions.DurationOps`", "2018-12-05") -object time { - class RichWholeNumber(wrapped: Long) { - def nanoseconds: Duration = Duration(wrapped, TimeUnit.NANOSECONDS) - def nanosecond: Duration = nanoseconds - def microseconds: Duration = Duration(wrapped, TimeUnit.MICROSECONDS) - def microsecond: Duration = microseconds - def milliseconds: Duration = Duration(wrapped, TimeUnit.MILLISECONDS) - def millisecond: Duration = milliseconds - def millis: Duration = milliseconds - def seconds: Duration = Duration(wrapped, TimeUnit.SECONDS) - def second: Duration = seconds - def minutes: Duration = Duration(wrapped, TimeUnit.MINUTES) - def minute: Duration = minutes - def hours: Duration = Duration(wrapped, TimeUnit.HOURS) - def hour: Duration = hours - def days: Duration = Duration(wrapped, TimeUnit.DAYS) - def day: Duration = days - } - - private val ZeroRichWholeNumber = new RichWholeNumber(0) { - override def nanoseconds: Duration = Duration.Zero - override def microseconds: Duration = Duration.Zero - override def milliseconds: Duration = Duration.Zero - override def seconds: Duration = Duration.Zero - override def minutes: Duration = Duration.Zero - override def hours: Duration = Duration.Zero - override def days: Duration = Duration.Zero - } - - implicit def intToTimeableNumber(i: Int): RichWholeNumber = - if (i == 0) ZeroRichWholeNumber else new RichWholeNumber(i) - implicit def longToTimeableNumber(l: Long): RichWholeNumber = - if (l == 0) ZeroRichWholeNumber else new RichWholeNumber(l) -} diff --git a/util-core/src/main/scala/com/twitter/conversions/u64.scala b/util-core/src/main/scala/com/twitter/conversions/u64.scala deleted file mode 100644 index 6e898db129..0000000000 --- a/util-core/src/main/scala/com/twitter/conversions/u64.scala +++ /dev/null @@ -1,22 +0,0 @@ -package com.twitter.conversions - -@deprecated("Use `com.twitter.conversions.U64Ops`", "2018-12-05") -object u64 { - - /** - * Parses this HEX string as an unsigned 64-bit long value. Be careful, this can throw - * [[NumberFormatException]]. - * - * @see [[java.lang.Long.parseUnsignedLong()]] - */ - implicit class StringOps(val self: String) extends AnyVal { - def toU64Long: Long = java.lang.Long.parseUnsignedLong(self, 16) - } - - /** - * Converts this unsigned 64-bit long value into a 16-character HEX string. - */ - implicit class LongOps(val self: Long) extends AnyVal { - def toU64HexString: String = "%016x".format(self) - } -} diff --git a/util-core/src/test/scala/com/twitter/conversions/PercentTest.scala b/util-core/src/test/scala/com/twitter/conversions/PercentTest.scala deleted file mode 100644 index 2777e68cfa..0000000000 --- a/util-core/src/test/scala/com/twitter/conversions/PercentTest.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.twitter.conversions - -import org.scalacheck.Arbitrary.arbitrary -import org.scalacheck.Gen -import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks - -class PercentTest extends FunSuite with GeneratorDrivenPropertyChecks { - import percent._ - - private[this] val Precision = 0.0000000001 - - private[this] def doubleEq(d1: Double, d2: Double): Boolean = { - Math.abs(d1 - d2) <= Precision - } - - test("percent can be fractional and precision is preserved") { - assert(99.9.percent == 0.999) - assert(99.99.percent == 0.9999) - assert(99.999.percent == 0.99999) - assert(12.3456.percent == 0.123456) - } - - test("percent can be > 100") { - assert(101.percent == 1.01) - assert(500.percent == 5.0) - } - - test("percent can be < 0") { - assert(-0.1.percent == -0.001) - assert(-500.percent == -5.0) - } - - test("assorted percentages") { - forAll(arbitrary[Int]) { i => - assert(new RichDoublePercent(i).percent == i / 100.0) - } - - // We're not as accurate when we get into high double-digit exponents, but that's acceptable. - forAll(Gen.choose(-10000D, 10000D)) { d => - assert(doubleEq(new RichDoublePercent(d).percent, d / 100.0)) - } - } - - test("doesn't blow up on edge cases") { - assert(Double.NaN.percent.equals(Double.NaN)) - assert(Double.NegativeInfinity.percent.equals(Double.NegativeInfinity)) - assert(Double.PositiveInfinity.percent.equals(Double.PositiveInfinity)) - } -} diff --git a/util-core/src/test/scala/com/twitter/conversions/TimeTest.scala b/util-core/src/test/scala/com/twitter/conversions/TimeTest.scala deleted file mode 100644 index 937c0e7f30..0000000000 --- a/util-core/src/test/scala/com/twitter/conversions/TimeTest.scala +++ /dev/null @@ -1,20 +0,0 @@ -package com.twitter.conversions - -import org.scalatest.FunSuite - -import com.twitter.util.Duration - -class TimeTest extends FunSuite { - import time._ - - test("converts Duration.Zero") { - assert(0.seconds eq Duration.Zero) - assert(0.milliseconds eq Duration.Zero) - assert(0.seconds eq 0.seconds) - } - - test("converts nonzero durations") { - assert(1.seconds == Duration.fromSeconds(1)) - assert(123.milliseconds == Duration.fromMilliseconds(123)) - } -} From 0111389c7e8a6500554f270ab11490429df83c58 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 13 Feb 2019 18:35:34 +0000 Subject: [PATCH 112/796] util-app: Remove warning when registering App Problem In many cases, there is a single `c.t.app.App` instance is registered in the singleton `c.t.app.App` object, however in testing there may be cases where multiple instances are started and thus registered. We log a warning but don't do anything specifically to prevent multiple registrations as the expectation is that the "last one wins". Thus the utility of the warning is suspect. Solution Remove the warning log statement. JIRA Issues: CSL-6034 Differential Revision: https://phabricator.twitter.biz/D273094 --- util-app/src/main/scala/com/twitter/app/App.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index 7b90e14063..b560b9c359 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -395,10 +395,5 @@ object App { */ def registered: Option[App] = ref.get - private[app] def register(app: App): Unit = - ref.getAndSet(Some(app)).foreach { existing => - log.warning( - s"Multiple com.twitter.app.App main methods called. ${existing.name}, then ${app.name}" - ) - } + private[app] def register(app: App): Unit = ref.set(Some(app)) } From 0a98d99198864308a22522a95ba22d1fe3a1fca7 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 13 Feb 2019 21:07:40 +0000 Subject: [PATCH 113/796] Fix usage of com.twitter.conversions.time in comments Problem / Solution A few usages of `com.twitter.conversions.time` slipped past in comments. This replaces them with `com.twitter.conversions.DurationOps`. Differential Revision: https://phabricator.twitter.biz/D273370 --- util-core/src/main/scala/com/twitter/util/Duration.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index 64d27090d9..c6c0617623 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -325,7 +325,7 @@ sealed class Duration private[util] (protected val nanos: Long) extends { * * - loses no information * - is easy to read - * - can be read back in if com.twitter.conversions.time._ is imported + * - can be read back in if com.twitter.conversions.DurationOps._ is imported * * An example: * From 67d50ad9368532f084a8740a4b8395a794cfdc34 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 13 Feb 2019 22:36:41 +0000 Subject: [PATCH 114/796] util-core: Improve Buf documentation on owned and shared Problem The nomenclature of owned and shared is not intuitive. Solution Clarify this part of Buf's documentation. JIRA Issues: CSL-7575 Differential Revision: https://phabricator.twitter.biz/D273351 --- .../src/main/scala/com/twitter/io/Buf.scala | 77 +++++++++++++++---- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 8ccba6c07e..153accb7c8 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -13,7 +13,7 @@ import scala.collection.immutable.VectorBuilder * * @see [[com.twitter.io.Buf.ByteArray]] for an `Array[Byte]` backed * implementation. - * @see [[com.twitter.io.Buf.ByteBuffer]] for an `nio.ByteBuffer` backed + * @see [[com.twitter.io.Buf.ByteBuffer]] for a `java.nio.ByteBuffer` backed * implementation. * @see [[com.twitter.io.Buf.apply]] for creating a `Buf` from other `Bufs` * @see [[com.twitter.io.Buf.Empty]] for an empty `Buf`. @@ -273,16 +273,20 @@ abstract class Buf { outer => } /** - * Buf wrapper-types (like Buf.ByteArray and Buf.ByteBuffer) provide Shared and - * Owned APIs, each of which with construction & extraction utilities. + * [[Buf]] wrapper-types (like [[Buf.ByteArray]] and [[Buf.ByteBuffer]]) provide + * "shared" and "owned" APIs, each of which contain construction and extraction + * utilities. While the owned and shared nomenclature is unintuitive, their + * purpose and use cases are straightforward. * - * The Owned APIs may provide direct access to a Buf's underlying - * implementation; and so mutating the data structure invalidates a Buf's - * immutability constraint. Users must take care to handle this data - * immutably. + * A "shared" [[Buf]] means that steps are taken to ensure that the produced + * [[Buf]] shares no state with the producer. This typically implies defensive + * copying and should be used when you do not control the lifecycle or usage + * of the passed in data. * - * The Shared variants, on the other hand, ensure that the Buf shares no state - * with the caller (at the cost of additional allocation). + * The "owned" APIs may provide direct access to a [[Buf]]'s underlying + * implementation; and so mutating the underlying data structure invalidates + * a [[Buf]]'s immutability constraint. Users must take care to handle this data + * immutably. * * Note: There are Java-friendly APIs for this object at `com.twitter.io.Bufs`. */ @@ -698,12 +702,15 @@ object Buf { /** * Construct a buffer representing the given bytes. + * + * The contract is that these bytes will not be modified after this has + * been called. */ def apply(bytes: Byte*): Buf = Owned(bytes.toArray) /** - * Safely coerce a buffer to a Buf.ByteArray, potentially without copying its underlying - * data. + * Safely coerce a buffer to a [[Buf.ByteArray]], potentially without + * copying its underlying data. */ def coerce(buf: Buf): Buf.ByteArray = buf match { case buf: Buf.ByteArray => buf @@ -716,7 +723,16 @@ object Buf { } } - /** Owned non-copying constructors/extractors for Buf.ByteArray. */ + /** + * Non-copying constructors and extractors for [[Buf.ByteArray]]. + * + * The "owned" APIs may provide direct access to a [[Buf]]'s underlying + * implementation; and so mutating the underlying data structure invalidates + * a [[Buf]]'s immutability constraint. Users must take care to handle this data + * immutably. + * + * @see [[Shared]] for analogs that make defensive copies. + */ object Owned { /** @@ -752,7 +768,16 @@ object Buf { } } - /** Safe copying constructors / extractors for Buf.ByteArray. */ + /** + * Copying constructors and extractors for [[Buf.ByteArray]]. + * + * A "shared" [[Buf]] means that steps are taken to ensure that the produced + * [[Buf]] shares no state with the producer. This typically implies defensive + * copying and should be used when you do not control the lifecycle or usage + * of the passed in data. + * + * @see [[Owned]] for analogs that do not make defensive copies. + */ object Shared { /** @@ -782,7 +807,7 @@ object Buf { /** * A buffer representing the remaining bytes in the - * given ByteBuffer. The given buffer will not be + * given `java.nio.ByteBuffer`. The given buffer will not be * affected. * * Modifications to the ByteBuffer's content will be @@ -859,7 +884,7 @@ object Buf { def unapply(buf: ByteBuffer): Some[java.nio.ByteBuffer] = Some(buf.underlying.asReadOnlyBuffer) - /** Coerce a generic buffer to a Buf.ByteBuffer, potentially without copying data. */ + /** Coerce a generic buffer to a [[Buf.ByteBuffer]], potentially without copying data. */ def coerce(buf: Buf): ByteBuffer = buf match { case buf: ByteBuffer => buf case _ => @@ -872,7 +897,16 @@ object Buf { new ByteBuffer(bb) } - /** Owned non-copying constructors/extractors for Buf.ByteBuffer. */ + /** + * Non-copying constructors and extractors for [[Buf.ByteBuffer]]. + * + * The "owned" APIs may provide direct access to a [[Buf]]'s underlying + * implementation; and so mutating the underlying data structure invalidates + * a [[Buf]]'s immutability constraint. Users must take care to handle this data + * immutably. + * + * @see [[Shared]] for analogs that make defensive copies. + */ object Owned { // N.B. We cannot use ByteBuffer.asReadOnly to ensure correctness because @@ -896,7 +930,16 @@ object Buf { def extract(buf: Buf): java.nio.ByteBuffer = Buf.ByteBuffer.coerce(buf).underlying } - /** Safe copying constructors/extractors for Buf.ByteBuffer. */ + /** + * Copying constructors and extractors for [[Buf.ByteBuffer]]. + * + * A "shared" [[Buf]] means that steps are taken to ensure that the produced + * [[Buf]] shares no state with the producer. This typically implies defensive + * copying and should be used when you do not control the lifecycle or usage + * of the passed in data. + * + * @see [[Owned]] for analogs that do not make defensive copies. + */ object Shared { private[this] def copy(orig: java.nio.ByteBuffer): java.nio.ByteBuffer = { val copy = java.nio.ByteBuffer.allocate(orig.remaining) From ea24c9510623bdc4d0e35f21fa4ba2c7a66d4279 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Fri, 15 Feb 2019 02:04:35 +0000 Subject: [PATCH 115/796] util-core: Don't allow AsyncSemaphore Permits to be released several times Problem It is possible to call release on the same permit twice, thus releasing the permit and a waiter. Solution AsyncSemaphore permits should be idempotent so that if we call release twice on a permit, it will only release the permit that is being released. Result With this change, you can call release on the same permit twice but it will only release that one permit. JIRA Issues: CSL-6358 Differential Revision: https://phabricator.twitter.biz/D274017 --- CHANGELOG.rst | 4 ++++ .../twitter/concurrent/AsyncSemaphore.scala | 23 +++++++++++-------- .../concurrent/AsyncSemaphoreTest.scala | 19 +++++++++++++++ 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 86c5a44e39..2e41d53175 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -32,6 +32,10 @@ Runtime Behavior Changes before the pipe would wait indefinitely for a read before transitioning to the Closed state. ``PHAB_ID=D229728`` +* util-core: Don't allow `AsyncSemaphore` `Permit`s to be released multiple times. Before it was + possible to release a permit more than once and incorrectly remove waiters from the queue. + With this change, the `release` permit operation is idempotent. ``PHAB_ID=D274017`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala index 75e64b59f4..a3efe46e29 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala @@ -59,30 +59,33 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { // Serves as our intrinsic lock. private[this] final def lock: Object = waitq - private[this] val semaphorePermit = new Permit { - private[this] val ReturnThis = Return(this) + private[this] def semaphorePermit(): Permit = new Permit { + // access to `released` is synchronized by locking on the `lock` above. + private[this] var released = false @tailrec override def release(): Unit = { val waiter = lock.synchronized { - val next = waitq.pollFirst() - if (next == null) { - availablePermits += 1 + if (released) null + else { + val next = waitq.pollFirst() + if (next == null) { + availablePermits += 1 + } + released = true + next } - next } if (waiter != null) { // If the waiter is already satisfied, it must // have been interrupted, so we can simply move on. - if (!waiter.updateIfEmpty(ReturnThis)) { + if (!waiter.updateIfEmpty(Return(semaphorePermit()))) { release() } } } } - private[this] val futurePermit = Future.value(semaphorePermit) - def numWaiters: Int = lock.synchronized(waitq.size) def numInitialPermits: Int = initialPermits def numPermitsAvailable: Int = lock.synchronized(availablePermits) @@ -122,7 +125,7 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { if (availablePermits > 0) { availablePermits -= 1 - futurePermit + Future.value(semaphorePermit()) } else { maxWaiters match { case Some(max) if waitq.size >= max => diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index 63085ff026..e7ff1dda0b 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -390,5 +390,24 @@ class AsyncSemaphoreTest extends FunSpec { val msgs = results.collect { case Some(Throw(e)) => e.getMessage } assert(msgs.forall(_ == "woop")) } + + it("should be idempotent and not release a permit twice") { _ => + val semaphore = new AsyncSemaphore(1) + val (p1, p2, p3) = (semaphore.acquire(), semaphore.acquire(), semaphore.acquire()) + + assert(p1.isDefined) + assert(!p2.isDefined) + assert(!p3.isDefined) + assert(semaphore.numWaiters == 2) + + val permit1 = await(p1) + permit1.release() + permit1.release() + + assert(p1.isDefined) + assert(p2.isDefined) + assert(!p3.isDefined) + assert(semaphore.numWaiters == 1) + } } } From f2a05474ec41f34146d710bdc2a789efd6da9d21 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Sat, 16 Feb 2019 01:56:53 +0000 Subject: [PATCH 116/796] util-core: Add onClose callback from parent reader in Reader.flatMap Problem Reader.flatMap doesn't propagate parent reader's onClose. Solution Add onClose callback from parent reader in Reader.flatMap. JIRA Issues: CSL-7505 Differential Revision: https://phabricator.twitter.biz/D269413 --- CHANGELOG.rst | 9 ++ .../main/scala/com/twitter/io/Reader.scala | 93 +++++++++++++++---- .../scala/com/twitter/io/ReaderTest.scala | 21 +++++ 3 files changed, 106 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2e41d53175..7e3533bcba 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-core: updated `Reader#fromFuture` to resolve its `onClose` when reading of end-of-stream. + ``PHAB_ID=D269413`` + * util-core: Added `Reader.flatten` to flatten a `Reader[Reader[_]]` to `Reader[_]`, and `Reader.fromSeq` to create a new Reader from a Seq. ``PHAB_ID=D255424`` @@ -25,6 +28,12 @@ New Features * util-security: `Pkcs8KeyManagerFactory` now supports a certificates file which contains multiple certificates that are part of the same certificate chain. ``PHAB_ID=D263190`` +Bug Fixes +~~~~~~~~~ + +* util-core: Fixed the behavior in `c.t.io.Reader` where `Reader#flatMap` fails to propagate + parent reader's `onClose`. ``PHAB_ID=D269413`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index a1851842ef..03c17b776c 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -3,6 +3,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.util._ import java.io.{File, FileInputStream, InputStream} +import java.util.concurrent.atomic.AtomicReference import scala.annotation.tailrec import scala.collection.mutable.ListBuffer @@ -95,7 +96,7 @@ trait Reader[+A] { self => def discard(): Unit /** - * A [[Future]] that resolves once this reader is closed. + * A [[Future]] that resolves once this reader is closed upon reading of end-of-stream. * * If the result is a failed future, this indicates that it was not closed either by reading * until the end of the stream nor by discarding. This is useful for any extra resource cleanup @@ -210,31 +211,68 @@ object Reader { /** * Construct a `Reader` from a `Future` + * + * We want to ensure that this reader always satisfies these invariants: + * 1. Satisfied closep is always aligned with the state + * 2. Reading from a discarded reader will always return ReaderDiscardedException + * 3. Reading from a fully read reader will always return None + * 4. Reading from an exceptioned reader will always return the exception it has thrown + * + * We achieved this with a state machine where any access to the state is synchronized, + * and by ensuring that we always verify the update of state succeeded before setting closep, + * and by preventing changing the state when the reader is exceptioned, fully read, or discarded + * + * @note Multiple outstanding reads are not allowed on this reader + * */ def fromFuture[A](fa: Future[A]): Reader[A] = new Reader[A] { private[this] val closep = Promise[StreamTermination]() - - final def read(): Future[Option[A]] = - if (!closep.isDefined) { - fa.map(Some.apply).respond { - case Throw(e) => closep.updateIfEmpty(Throw(e)) - case Return(_) => closep.updateIfEmpty(StreamTermination.FullyRead.Return) - } - } else { - closep.flatMap { - case StreamTermination.FullyRead => Future.None - case StreamTermination.Discarded => Future.exception(new ReaderDiscardedException) - } + private[this] val state = new AtomicReference[State](State.Idle) + + final def read(): Future[Option[A]] = { + state.get() match { + case State.Idle => + fa.map(Some.apply).respond { + case t: Throw[_] => + if (state.compareAndSet(State.Idle, State.Exception)) { + closep.update(t.cast[StreamTermination]) + } + case Return(_) => + state.compareAndSet(State.Idle, State.Read) + } + case State.Read => + if (state.compareAndSet(State.Read, State.FullyRead)) { + closep.update(StreamTermination.FullyRead.Return) + } + closep.flatMap { + case StreamTermination.FullyRead => Future.None + case StreamTermination.Discarded => Future.exception(new ReaderDiscardedException) + } + case State.Exception => + /** closep is guaranteed to be an exception, flatMap should never be triggered but return the exception */ + closep.flatMap(_ => Future.None) + case State.FullyRead => + Future.None + case State.Discard => + Future.exception(new ReaderDiscardedException) } + } - final def discard(): Unit = closep.updateIfEmpty(StreamTermination.Discarded.Return) + final def discard(): Unit = { + if (state.compareAndSet(State.Idle, State.Discard) || state + .compareAndSet(State.Read, State.Discard)) { + closep.update(StreamTermination.Discarded.Return) + fa.raise(new ReaderDiscardedException) + } + } final def onClose: Future[StreamTermination] = closep } /** * Construct a `Reader` from a value `a` + * @note Multiple outstanding reads are not allowed on this reader */ def value[A](a: A): Reader[A] = fromFuture[A](Future.value(a)) @@ -420,13 +458,14 @@ object Reader { // access this currentReader is synchronized on `self` private[this] var currentReader: Reader[A] = Reader.empty + readers.onClose.respond(closep.updateIfEmpty) + private def updateCurrentAndRead(): Future[Option[A]] = readers.read().flatMap { case Some(reader) => self.synchronized { currentReader = reader } read() case None => - closep.updateIfEmpty(StreamTermination.FullyRead.Return) Future.None } @@ -441,7 +480,6 @@ object Reader { } def discard(): Unit = { - closep.updateIfEmpty(StreamTermination.Discarded.Return) self.synchronized(currentReader).discard() readers.discard() } @@ -486,7 +524,7 @@ object Reader { // We have to do this because discarding the writer doesn't interrupt read // operations, it only fails the next write operation. loop().proxyTo(p) - p.setInterruptHandler { case exc => r.discard() } + p.setInterruptHandler { case _ => r.discard() } p } @@ -497,4 +535,25 @@ object Reader { * of the framer. */ def framed(r: Reader[Buf], framer: Buf => Seq[Buf]): Reader[Buf] = new Framed(r, framer) + + private sealed trait State + + private object State { + + /** Indicates no actions are taken on the Reader. */ + case object Idle extends State + + /** Indicates the reader has been read once. */ + case object Read extends State + + /** Indicates an exception occurred during reading */ + case object Exception extends State + + /** Indicates the reader is fully read. */ + case object FullyRead extends State + + /** Indicates the reader has been discarded. */ + case object Discard extends State + } + } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 7f2ebb5ad3..6c6d588bf1 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -458,11 +458,32 @@ class ReaderTest } } + test("Reader.flatMap: propagate parent's onClose when parent reader failed with an exception") { + val ExceptionMessage = "boom" + val parentReader = new Pipe[Double] + val childReader = parentReader.flatMap(Reader.value) + parentReader.fail(new Exception(ExceptionMessage)) + val parentException = intercept[Exception](await(parentReader.onClose)) + val readerException = intercept[Exception](await(childReader.onClose)) + assert(parentException.getMessage == ExceptionMessage) + assert(readerException.getMessage == ExceptionMessage) + } + + test("Reader.flatMap: propagate parent's onClose when parent reader is discarded") { + val parentReader = Reader.empty[Int] + val childReader = parentReader.flatMap(Reader.value) + parentReader.discard() + assert(await(parentReader.onClose) == StreamTermination.Discarded) + assert(await(childReader.onClose) == StreamTermination.Discarded) + } + test("Reader.value") { forAll { a: AnyVal => val r = Reader.value(a) assert(await(r.read()) == Some(a)) + assert(r.onClose.isDefined == false) assert(await(r.read()) == None) + assert(await(r.onClose) == StreamTermination.FullyRead) } } From aeafba1a4f547dc0e543a3bcea596ceb78ff4e63 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Tue, 19 Feb 2019 19:46:18 +0000 Subject: [PATCH 117/796] util-core: bypass scheduler for map/transformTry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem transformTry and map can't express recursive computations so there's no stack to unwind. Solution Just apply the function passed to map. Benchmarks ``` Before Benchmark Mode Cnt Score Error Units FutureBenchmark.fromConstMap avgt 10 56.712 ± 1.415 ns/op FutureBenchmark.fromConstMap:·gc.alloc.rate.norm avgt 10 104.000 ± 0.001 B/op After Benchmark Mode Cnt Score Error Units FutureBenchmark.fromConstMap avgt 10 36.494 ± 0.243 ns/op FutureBenchmark.fromConstMap:·gc.alloc.rate.norm avgt 10 72.000 ± 0.001 B/op ``` JIRA Issues: CSL-7650 Differential Revision: https://phabricator.twitter.biz/D271962 --- CHANGELOG.rst | 7 ++++ .../scala/com/twitter/util/ConstFuture.scala | 41 +++++++++++-------- .../main/scala/com/twitter/util/Future.scala | 10 +++++ 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7e3533bcba..6a96a06f51 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,13 @@ New Features * util-core: Optimized `ConstFuture.proxyTo` which brings the performance of `flatMap` and `transform` of a `ConstFuture` in line with `map`. ``PHAB_ID=D271358`` +* util-core: Experimental toggle (com.twitter.util.BypassScheduler) for speeding up + `ConstFuture.map` (`transformTry`). The mechanism, when turned on, runs map operations + immediately (why not when we have a concrete value), instead of via the Scheduler, where it may + be queued and potentially reordered, e.g.: + `f.flatMap { _ => println(1); g.map { _ => println(2) }; println(3) }` will print `1 2 3`, + where it would have printed `1 3 2`. ``PHAB_ID=D271962`` + * util-security: `Pkcs8KeyManagerFactory` now supports a certificates file which contains multiple certificates that are part of the same certificate chain. ``PHAB_ID=D263190`` diff --git a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala index 6e9c35d076..fee635c4fe 100644 --- a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala @@ -41,24 +41,31 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { val p = new Promise[B] - // see the note on `respond` for an explanation of why `Scheduler` is used. - val saved = Local.save() - Scheduler.submit(new Runnable { - def run(): Unit = { - val current = Local.save() - if (current ne saved) - Local.restore(saved) - val computed = try f(result) - catch { - case e: NonLocalReturnControl[_] => Throw(new FutureNonLocalReturnControl(e)) - case scala.util.control.NonFatal(e) => Throw(e) - case t: Throwable => - Monitor.handle(t) - throw t - } finally Local.restore(current) - p.update(computed) + if (Future.BypassScheduler) { + try p.update(f(result)) + catch { + case scala.util.control.NonFatal(e) => p.update(Throw(e)) } - }) + } else { + // see the note on `respond` for an explanation of why `Scheduler` is used. + val saved = Local.save() + Scheduler.submit(new Runnable { + def run(): Unit = { + val current = Local.save() + if (current ne saved) + Local.restore(saved) + val computed = try f(result) + catch { + case e: NonLocalReturnControl[_] => Throw(new FutureNonLocalReturnControl(e)) + case scala.util.control.NonFatal(e) => Throw(e) + case t: Throwable => + Monitor.handle(t) + throw t + } finally Local.restore(current) + p.update(computed) + } + }) + } p } diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 3cf0995995..2713638a17 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -14,6 +14,16 @@ import scala.util.control.NoStackTrace * on concurrent programming with Futures. */ object Future { + // An experimental property for bypassing the scheduler in Future.map + // operations. + @volatile private[util] var BypassScheduler: Boolean = + System.getProperty("com.twitter.util.BypassScheduler", "false").toBoolean + + // Set to true to bypass the scheduler in Future.map operations. + private[twitter] def bypassScheduler(setting: Boolean): Unit = { + BypassScheduler = setting + } + val DEFAULT_TIMEOUT: Duration = Duration.Top /** A successfully satisfied constant `Unit`-typed `Future` of `()` */ From 5bf12f3a414e08aced2834d279e8548a5f63f689 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Wed, 20 Feb 2019 20:12:40 +0000 Subject: [PATCH 118/796] util-core: revise Var docs Problem / Solution Explain Var in terms perhaps more familiar. Also, add an example to illustrate what it can be used for. JIRA Issues: CSL-2985 Differential Revision: https://phabricator.twitter.biz/D275308 --- .../src/main/scala/com/twitter/util/Var.scala | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 5a6fae9024..c1db345a08 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -11,12 +11,45 @@ import scala.language.higherKinds import scala.reflect.ClassTag /** - * Trait Var represents a variable. It is a reference cell which is - * composable: dependent Vars (derived through flatMap) are - * recomputed automatically when independent variables change -- they - * implement a form of self-adjusting computation. + * Vars are values that vary over time. To create one, you must give it an + * initial value. * - * Vars are observed, notifying users whenever the variable changes. + * {{{ + * val a = Var[Int](1) + * }}} + * + * A Var created this way can be sampled to retrieve its current value, + * + * {{{ + * println(Var.sample(a)) // prints 1 + * }}} + * + * or, invoked to assign it new values. + * + * {{{ + * a.update(2) + * println(Var.sample(a)) // prints 2 + * }}} + * + * Vars can be derived from other Vars. + * + * {{{ + * val b = a.flatMap { x => Var(x + 2) } + * println(Var.sample(b)) // prints 4 + * }}} + * + * And, if the underlying is assigned a new value, the derived Var is updated. + * Updates are computed lazily, so while assignment is cheap, sampling is where + * we pay the cost of the computation required to derive the new Var. + * + * {{{ + * a.update(1) + * println(Var.sample(b)) // prints 3 + * }}} + * + * A key difference between the derived Var and its underlying is that derived + * Vars can't be assigned new values. That's why `b`, from the example above, + * can't be invoked to assign it a new value, it can only be sampled. * * @note Vars do not always perform the minimum amount of * re-computation. From 58ff4f33abd63212b5944cf4391747fe209a12dc Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Wed, 20 Feb 2019 21:35:28 +0000 Subject: [PATCH 119/796] util-core: revise docs for Event Problem / Solution It's not obvious from Event docs that Events are actually quite familiar. Clarify what is meant by instantaneous values, but also relate Events to streams. JIRA Issues: CSL-2985 Differential Revision: https://phabricator.twitter.biz/D274746 --- .../main/scala/com/twitter/util/Event.scala | 59 ++++++++++++++++--- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Event.scala b/util-core/src/main/scala/com/twitter/util/Event.scala index ca89f2824a..9f6890c527 100644 --- a/util-core/src/main/scala/com/twitter/util/Event.scala +++ b/util-core/src/main/scala/com/twitter/util/Event.scala @@ -8,14 +8,59 @@ import scala.collection.immutable.Queue import scala.language.higherKinds /** - * Events are instantaneous values, defined only at particular - * instants in time (cf. [[com.twitter.util.Var Vars]], which are - * defined at all times). It is possible to view Events as the - * discrete counterpart to [[com.twitter.util.Var Var]]'s continuous - * nature. + * Events are emitters of values. They can be processed as if they were + * streams. Here's how to create one that emits Ints. * - * Events are observed by registering [[com.twitter.util.Witness Witnesses]] - * to which the Event's values are notified. + * {{{ + * val ints = Event[Int]() + * }}} + * + * To listen to an Event stream for values, register a handler. + * + * {{{ + * val you = ints.respond { n => println(s"$n for you") } + * }}} + * + * Handlers in Event-speak are called [[com.twitter.util.Witness Witnesses]]. + * And, respond is equivalent to `register(Witness { ... })`, so we can also + * write: + * + * {{{ + * val me = ints.register(Witness { n => println(s"$n for me") }) + * }}} + * + * Registration is time-sensitive. Witnesses registered to an Event stream + * can't observe past values. In this way, Events are instantaneous. When the + * stream emits a value, it fans out to every Witness; if there are no + * Witnesses to that communication, whatever value it carried is lost. + * + * Events emit values, okay, but who decides what is emitted? The implementor + * of an Event decides! + * + * But there's another way: you can create an Event that is also a Witness. + * Remember, Witnesses are simply handlers that can receive a value. Their type + * is `T => Unit`. So an Event that you can send values to is also a Witness. + * + * The `ints` Event we created above is one of these. Watch: + * + * {{{ + * scala> ints.notify(1) + * 1 for you + * 1 for me + * }}} + * + * We can also stop listening. This is important for resource cleanup. + * + * {{{ + * scala> me.close() + * scala> ints.notify(2) + * 2 for you + * }}} + * + * In relation to [[com.twitter.util.Var Vars]], Events are instantaneous + * values, defined only at particular instants in time; whereas Vars are + * defined at all times. It is possible to view Events as the discrete + * counterpart to Var's continuous nature. * * Note: There is a Java-friendly API for this trait: [[com.twitter.util.AbstractEvent]]. */ From d6b34e626dfc0ffbbbd25c303febc430435b4bc0 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Wed, 20 Feb 2019 23:20:39 +0000 Subject: [PATCH 120/796] util-core: avoid creating a Promise if we can bypass the scheduler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem / Solution We can bypass the scheduler for transformTry, so we don't need to create a Promise in that case. Benchmark Before Benchmark Mode Cnt Score Error Units FutureBenchmark.fromConstMap avgt 10 57.302 ± 6.487 ns/op FutureBenchmark.fromConstMap:·gc.alloc.rate.norm avgt 10 104.000 ± 0.001 B/op After Benchmark Mode Cnt Score Error Units FutureBenchmark.fromConstMap avgt 10 7.130 ± 0.418 ns/op FutureBenchmark.fromConstMap:·gc.alloc.rate.norm avgt 10 32.000 ± 0.001 B/op JIRA Issues: CSL-7657 Differential Revision: https://phabricator.twitter.biz/D276146 --- .../src/main/scala/com/twitter/util/ConstFuture.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala index fee635c4fe..c49a6385d4 100644 --- a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala @@ -40,13 +40,13 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { def raise(interrupt: Throwable): Unit = () protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { - val p = new Promise[B] if (Future.BypassScheduler) { - try p.update(f(result)) + try Future.const(f(result)) catch { - case scala.util.control.NonFatal(e) => p.update(Throw(e)) + case scala.util.control.NonFatal(e) => Future.const(Throw(e)) } } else { + val p = new Promise[B] // see the note on `respond` for an explanation of why `Scheduler` is used. val saved = Local.save() Scheduler.submit(new Runnable { @@ -65,8 +65,8 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { p.update(computed) } }) + p } - p } def transform[B](f: Try[A] => Future[B]): Future[B] = { From 5cfea3c4e2c9c51a9479be3f3099ce4e52558720 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 21 Feb 2019 18:48:01 +0000 Subject: [PATCH 121/796] twitter-oss: Prepare OSS libraries for release 19.2.0 Problem We want to release the next versions of our Twitter OSS libraries 19.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D276537 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6a96a06f51..4b56412116 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.2.0 +------- + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 59016c4d42..fa5ecab24f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.1.0" +val collUtils = "com.twitter" %% "util-collection" % "19.2.0" ``` # Units diff --git a/build.sbt b/build.sbt index b83842414d..2c8567170c 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.2.0-SNAPSHOT" +val releaseVersion = "19.2.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From f367bc2f09e38857c16cca16a0f2f323d77f6c9a Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Fri, 22 Feb 2019 01:02:17 +0000 Subject: [PATCH 122/796] twitter-oss: Update OSS Libraries Post-Release to 19.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.3.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-7681 Differential Revision: https://phabricator.twitter.biz/D276994 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fa5ecab24f..d9ab402e6e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 2c8567170c..1d96fc60ef 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.2.0" +val releaseVersion = "19.3.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 898d7fe72e4373865a98816850de74026bf14a8f Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Mon, 25 Feb 2019 23:15:28 +0000 Subject: [PATCH 123/796] util-core: Revise Activity docs Problem / Solution Clarify and add examples to Activity. JIRA Issues: CSL-2985 Differential Revision: https://phabricator.twitter.biz/D277831 --- .../scala/com/twitter/util/Activity.scala | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Activity.scala b/util-core/src/main/scala/com/twitter/util/Activity.scala index 04eb57d687..7783af1083 100644 --- a/util-core/src/main/scala/com/twitter/util/Activity.scala +++ b/util-core/src/main/scala/com/twitter/util/Activity.scala @@ -10,16 +10,39 @@ import scala.reflect.ClassTag import scala.util.control.NonFatal /** - * An Activity is a handle to a concurrently running process, producing - * T-typed values. An activity is in one of three states: + * Activity, like [[com.twitter.util.Var Var]], is a value that varies over + * time; but, richer than Var, these values have error and pending states. + * They are useful for modeling a process that produces values, but also + * intermittent failures (e.g. address resolution, reloadable configuration). + * + * As such, an Activity can be in one of three states: * * - [[com.twitter.util.Activity.Pending Pending]]: output is pending; * - [[com.twitter.util.Activity.Ok Ok]]: an output is available; and * - [[com.twitter.util.Activity.Failed Failed]]: the process failed with an exception. * - * An activity may transition between any state at any time. + * An Activity may transition between any state at any time. They are typically + * constructed from [[com.twitter.util.Event Events]], but they can also be + * constructed without arguments. + * + * {{{ + * val (activity, witness) = Activity[Int]() + * println(activity.sample()) // throws java.lang.IllegalStateException: Still pending + * }}} + * + * As you can see, Activities start in a Pending state. Attempts to sample an + * Activity that is pending will fail. Curiously, the constructor for an + * Activity also returns a [[com.twitter.util.Witness Witness]]. + * + * Recall that a Witness is the interface to something that you can send values + * to; and in this case, that something is the Activity. + * + * {{{ + * witness.notify(Return(1)) + * println(activity.sample()) // prints 1 + * }}} * - * (The observant reader will notice that this is really a monad + * (For implementers, it may be useful to think of Activity as a monad * transformer for an ''Op'' monad over [[com.twitter.util.Var Var]] * where Op is like a [[com.twitter.util.Try Try]] with an additional * pending state.) From 9ac7830557b56cff5eb6d2b862407b5e3d3d53e9 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 27 Feb 2019 23:56:21 +0000 Subject: [PATCH 124/796] util-jvm: Various cleanups Problem util-jvm needs some tidying up. Solution Split out classes into top-level files, rename files to match class names, remove @RunWith annotation, and so on. Differential Revision: https://phabricator.twitter.biz/D279468 --- .../scala/com/twitter/jvm/CpuProfile.scala | 6 +- .../src/main/scala/com/twitter/jvm/Gc.scala | 5 ++ .../src/main/scala/com/twitter/jvm/Heap.scala | 15 ++++ .../src/main/scala/com/twitter/jvm/Jvm.scala | 78 +------------------ .../main/scala/com/twitter/jvm/NilJvm.scala | 2 +- .../com/twitter/jvm/{Opts.scala => Opt.scala} | 0 .../src/main/scala/com/twitter/jvm/Pool.scala | 29 +++++++ .../scala/com/twitter/jvm/PoolState.scala | 15 ++++ .../scala/com/twitter/jvm/Safepoint.scala | 15 ++++ .../main/scala/com/twitter/jvm/Snapshot.scala | 5 ++ .../twitter/jvm/{Cpu.scala => numProcs.scala} | 0 .../com/twitter/jvm/ContentionTest.scala | 7 +- .../com/twitter/jvm/CpuProfileTest.scala | 3 - .../test/scala/com/twitter/jvm/JvmTest.scala | 3 - .../jvm/MockScheduledExecutorService.scala | 1 - .../jvm/{OptsTest.scala => OptTest.scala} | 5 +- 16 files changed, 93 insertions(+), 96 deletions(-) create mode 100644 util-jvm/src/main/scala/com/twitter/jvm/Gc.scala create mode 100644 util-jvm/src/main/scala/com/twitter/jvm/Heap.scala rename util-jvm/src/main/scala/com/twitter/jvm/{Opts.scala => Opt.scala} (100%) create mode 100644 util-jvm/src/main/scala/com/twitter/jvm/Pool.scala create mode 100644 util-jvm/src/main/scala/com/twitter/jvm/PoolState.scala create mode 100644 util-jvm/src/main/scala/com/twitter/jvm/Safepoint.scala create mode 100644 util-jvm/src/main/scala/com/twitter/jvm/Snapshot.scala rename util-jvm/src/main/scala/com/twitter/jvm/{Cpu.scala => numProcs.scala} (100%) rename util-jvm/src/test/scala/com/twitter/jvm/{OptsTest.scala => OptTest.scala} (92%) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala index 142a86b36c..de5733f276 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala @@ -1,14 +1,12 @@ package com.twitter.jvm +import com.twitter.conversions.DurationOps._ +import com.twitter.util.{Duration, Future, Promise, Stopwatch, Time} import java.io.OutputStream import java.lang.management.ManagementFactory import java.nio.{ByteBuffer, ByteOrder} - import scala.collection.mutable -import com.twitter.conversions.DurationOps._ -import com.twitter.util.{Duration, Future, Promise, Stopwatch, Time} - /** * A CPU profile. */ diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Gc.scala b/util-jvm/src/main/scala/com/twitter/jvm/Gc.scala new file mode 100644 index 0000000000..bf96b655bd --- /dev/null +++ b/util-jvm/src/main/scala/com/twitter/jvm/Gc.scala @@ -0,0 +1,5 @@ +package com.twitter.jvm + +import com.twitter.util.{Duration, Time} + +case class Gc(count: Long, name: String, timestamp: Time, duration: Duration) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Heap.scala b/util-jvm/src/main/scala/com/twitter/jvm/Heap.scala new file mode 100644 index 0000000000..f75256695d --- /dev/null +++ b/util-jvm/src/main/scala/com/twitter/jvm/Heap.scala @@ -0,0 +1,15 @@ +package com.twitter.jvm + +/** + * Information about the Heap + * + * @param allocated Estimated number of bytes + * that have been allocated so far (into eden) + * + * @param tenuringThreshold How many times an Object + * needs to be copied before being tenured. + * + * @param ageHisto Histogram of the number of bytes that + * have been copied as many times. Note: 0-indexed. + */ +case class Heap(allocated: Long, tenuringThreshold: Long, ageHisto: Seq[Long]) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala index b0142466ec..d1c4e6be5e 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala @@ -9,76 +9,6 @@ import java.util.logging.{Level, Logger} import scala.collection.JavaConverters._ import scala.util.control.NonFatal -/** - * Information about the Heap - * - * @param allocated Estimated number of bytes - * that have been allocated so far (into eden) - * - * @param tenuringThreshold How many times an Object - * needs to be copied before being tenured. - * - * @param ageHisto Histogram of the number of bytes that - * have been copied as many times. Note: 0-indexed. - */ -case class Heap(allocated: Long, tenuringThreshold: Long, ageHisto: Seq[Long]) - -/** - * Information about the JVM's safepoint - * - * @param syncTimeMillis Cumulative time, in milliseconds, spent - * getting all threads to safepoint states - * - * @param totalTimeMillis Cumulative time, in milliseconds, that the - * application has been stopped for safepoint operations - * - * @param count The number of safepoints taken place since - * the JVM started - */ -case class Safepoint(syncTimeMillis: Long, totalTimeMillis: Long, count: Long) - -case class PoolState(numCollections: Long, capacity: StorageUnit, used: StorageUnit) { - def -(other: PoolState) = PoolState( - numCollections = this.numCollections - other.numCollections, - capacity = other.capacity, - used = this.used + other.capacity - other.used + - other.capacity * (this.numCollections - other.numCollections - 1) - ) - - override def toString = - "PoolState(n=%d,remaining=%s[%s of %s])".format(numCollections, capacity - used, used, capacity) -} - -/** - * A handle to a garbage collected memory pool. - */ -trait Pool { - - /** Get the current state of this memory pool. */ - def state(): PoolState - - /** - * Sample the allocation rate of this pool. Note that this is merely - * an estimation based on sampling the state of the pool initially - * and then again when the period elapses. - * - * @return Future of the samples rate (in bps). - */ - def estimateAllocRate(period: Duration, timer: Timer): Future[Long] = { - val elapsed = Stopwatch.start() - val begin = state() - timer.doLater(period) { - val end = state() - val interval = elapsed() - ((end - begin).used.inBytes * 1000) / interval.inMilliseconds - } - } -} - -case class Gc(count: Long, name: String, timestamp: Time, duration: Duration) - -case class Snapshot(timestamp: Time, heap: Heap, lastGcs: Seq[Gc]) - /** * Access JVM internal performance counters. We maintain a strict * interface so that we are decoupled from the actual underlying JVM. @@ -174,7 +104,7 @@ trait Jvm { } executor.scheduleAtFixedRate( - new Runnable { def run() = sample() }, + new Runnable { def run(): Unit = sample() }, 0 /*initial delay*/, Period.inMilliseconds, TimeUnit.MILLISECONDS @@ -198,7 +128,7 @@ trait Jvm { buffer = (gc :: buffer).takeWhile(_.timestamp > floor) } - (since: Time) => + since: Time => buffer.takeWhile(_.timestamp > since) } @@ -213,9 +143,9 @@ trait Jvm { */ def mainClassName: String = { val mainClass = for { - (_, stack) <- Thread.getAllStackTraces().asScala.find { case (t, s) => t.getName == "main" } + (_, stack) <- Thread.getAllStackTraces.asScala.find { case (t, s) => t.getName == "main" } frame <- stack.reverse.find { elem => - !(elem.getClassName.startsWith("scala.tools.nsc.MainGenericRunner")) + !elem.getClassName.startsWith("scala.tools.nsc.MainGenericRunner") } } yield frame.getClassName diff --git a/util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala index fa57bb40bb..a781fedd9e 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/NilJvm.scala @@ -5,7 +5,7 @@ import com.twitter.util.Time object NilJvm extends Jvm { val opts: Opts = new Opts { - def compileThresh = None + def compileThresh: Option[Int] = None } def forceGc(): Unit = System.gc() val snapCounters: Map[String, String] = Map() diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Opts.scala b/util-jvm/src/main/scala/com/twitter/jvm/Opt.scala similarity index 100% rename from util-jvm/src/main/scala/com/twitter/jvm/Opts.scala rename to util-jvm/src/main/scala/com/twitter/jvm/Opt.scala diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Pool.scala b/util-jvm/src/main/scala/com/twitter/jvm/Pool.scala new file mode 100644 index 0000000000..dff4649def --- /dev/null +++ b/util-jvm/src/main/scala/com/twitter/jvm/Pool.scala @@ -0,0 +1,29 @@ +package com.twitter.jvm + +import com.twitter.util.{Duration, Future, Stopwatch, Timer} + +/** + * A handle to a garbage collected memory pool. + */ +trait Pool { + + /** Get the current state of this memory pool. */ + def state(): PoolState + + /** + * Sample the allocation rate of this pool. Note that this is merely + * an estimation based on sampling the state of the pool initially + * and then again when the period elapses. + * + * @return Future of the samples rate (in bps). + */ + def estimateAllocRate(period: Duration, timer: Timer): Future[Long] = { + val elapsed = Stopwatch.start() + val begin = state() + timer.doLater(period) { + val end = state() + val interval = elapsed() + ((end - begin).used.inBytes * 1000) / interval.inMilliseconds + } + } +} diff --git a/util-jvm/src/main/scala/com/twitter/jvm/PoolState.scala b/util-jvm/src/main/scala/com/twitter/jvm/PoolState.scala new file mode 100644 index 0000000000..5d02289be8 --- /dev/null +++ b/util-jvm/src/main/scala/com/twitter/jvm/PoolState.scala @@ -0,0 +1,15 @@ +package com.twitter.jvm + +import com.twitter.util.StorageUnit + +case class PoolState(numCollections: Long, capacity: StorageUnit, used: StorageUnit) { + def -(other: PoolState): PoolState = PoolState( + numCollections = this.numCollections - other.numCollections, + capacity = other.capacity, + used = this.used + other.capacity - other.used + + other.capacity * (this.numCollections - other.numCollections - 1) + ) + + override def toString: String = + s"PoolState(n=$numCollections,remaining=${capacity - used}[$used of $capacity])" +} diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Safepoint.scala b/util-jvm/src/main/scala/com/twitter/jvm/Safepoint.scala new file mode 100644 index 0000000000..f08bed510e --- /dev/null +++ b/util-jvm/src/main/scala/com/twitter/jvm/Safepoint.scala @@ -0,0 +1,15 @@ +package com.twitter.jvm + +/** + * Information about the JVM's safepoint + * + * @param syncTimeMillis Cumulative time, in milliseconds, spent + * getting all threads to safepoint states + * + * @param totalTimeMillis Cumulative time, in milliseconds, that the + * application has been stopped for safepoint operations + * + * @param count The number of safepoints taken place since + * the JVM started + */ +case class Safepoint(syncTimeMillis: Long, totalTimeMillis: Long, count: Long) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Snapshot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Snapshot.scala new file mode 100644 index 0000000000..642d6e090b --- /dev/null +++ b/util-jvm/src/main/scala/com/twitter/jvm/Snapshot.scala @@ -0,0 +1,5 @@ +package com.twitter.jvm + +import com.twitter.util.Time + +case class Snapshot(timestamp: Time, heap: Heap, lastGcs: Seq[Gc]) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Cpu.scala b/util-jvm/src/main/scala/com/twitter/jvm/numProcs.scala similarity index 100% rename from util-jvm/src/main/scala/com/twitter/jvm/Cpu.scala rename to util-jvm/src/main/scala/com/twitter/jvm/numProcs.scala diff --git a/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala index f5cd2c8b7d..5de408fdcd 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala @@ -1,15 +1,11 @@ package com.twitter.jvm +import com.twitter.util.{Await, Promise} import java.util.concurrent.locks.ReentrantLock - -import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.concurrent.Eventually -import org.scalatest.junit.JUnitRunner import org.scalatest.time.{Millis, Seconds, Span} -import com.twitter.util.{Await, Promise} - class Philosopher { val ready = new Promise[Unit] private val lock = new ReentrantLock() @@ -22,7 +18,6 @@ class Philosopher { } } -@RunWith(classOf[JUnitRunner]) class ContentionTest extends FunSuite with Eventually { implicit override val patienceConfig = diff --git a/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala index 73fa5789dd..4e754ebe27 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala @@ -3,11 +3,8 @@ package com.twitter.jvm import com.twitter.conversions.DurationOps._ import com.twitter.util.Time import java.io.ByteArrayOutputStream -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class CpuProfileTest extends FunSuite { test("record") { diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala index 87c75e28f2..089a0449e7 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala @@ -3,12 +3,9 @@ package com.twitter.jvm import com.twitter.conversions.DurationOps._ import com.twitter.logging.{Level, TestLogging} import com.twitter.util.Time -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import scala.collection.mutable -@RunWith(classOf[JUnitRunner]) class JvmTest extends WordSpec with TestLogging { "Jvm" should { class JvmHelper { diff --git a/util-jvm/src/test/scala/com/twitter/jvm/MockScheduledExecutorService.scala b/util-jvm/src/test/scala/com/twitter/jvm/MockScheduledExecutorService.scala index 29d8bcf074..62cadac2b4 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/MockScheduledExecutorService.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/MockScheduledExecutorService.scala @@ -7,7 +7,6 @@ import java.util.concurrent.{ ScheduledFuture, TimeUnit } - import scala.collection.mutable // A mostly empty implementation so that we can successfully diff --git a/util-jvm/src/test/scala/com/twitter/jvm/OptsTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/OptTest.scala similarity index 92% rename from util-jvm/src/test/scala/com/twitter/jvm/OptsTest.scala rename to util-jvm/src/test/scala/com/twitter/jvm/OptTest.scala index 51296886fb..ba4933dde0 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/OptsTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/OptTest.scala @@ -3,12 +3,9 @@ package com.twitter.jvm import java.lang.{Boolean => JBool} import java.lang.management.ManagementFactory import javax.management.ObjectName -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) -class OptsTest extends FunSuite { +class OptTest extends FunSuite { test("Opts") { if (System.getProperty("java.vm.name").contains("HotSpot")) { val DiagnosticName = From b0a5d06269b9526b4408239ce1441b2a213dd0df Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 4 Mar 2019 23:06:16 +0000 Subject: [PATCH 125/796] util-core,inject-utils: Move String utilities from inject-utils to util-core Problem/Solution The `c.t.inject.utils.StringUtils` methods `snakify`, `camelify`, and `pascalify` are broadly useful outside of the finatra/inject/inject-utils package. Move them to corresponding methods in util/util-core `c.t.conversions.StringOps` along with associated tests. The methods in `c.t.conversions.StringOps` are `toSnakeCase`, `toCamelCase`, and `toPascalCase`. Mark the definitions in finatra/inject/inject-utils as deprecated and encourage users to move to respective `c.t.conversions.StringOps` implementation. JIRA Issues: CSL-7741 Differential Revision: https://phabricator.twitter.biz/D280886 --- CHANGELOG.rst | 10 +- .../com/twitter/conversions/StringOps.scala | 128 ++++++++++++++ .../twitter/util/StringConversionsTest.scala | 163 ++++++++++++++---- 3 files changed, 261 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b56412116..275d73d620 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,8 +7,14 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-core: Added `c.t.conversions.StringOps#toSnakeCase,toCamelCase,toPascalCase` implementations. + ``PHAB_ID=D280886`` + 19.2.0 -------- +------ New Features ~~~~~~~~~~~~ @@ -89,7 +95,7 @@ Breaking API Changes ``PHAB_ID=D270916`` 19.1.0 -------- +------ New Features ~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/conversions/StringOps.scala b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala index fd9e7dedc5..adef1f1edc 100644 --- a/util-core/src/main/scala/com/twitter/conversions/StringOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala @@ -118,6 +118,45 @@ object StringOps { } buffer } + + /** + * Converts foo_bar to fooBar (first letter lowercased) + * + * For example: + * + * {{{ + * "hello_world".toCamelCase + * }}} + * + * will return the String `"helloWorld"`. + */ + def toCamelCase: String = StringOps.toCamelCase(string) + + /** + * Converts foo_bar to FooBar (first letter uppercased) + * + * For example: + * + * {{{ + * "hello_world".toPascalCase + * }}} + * + * will return the String `"HelloWorld"`. + */ + def toPascalCase: String = StringOps.toPascalCase(string) + + /** + * Converts FooBar to foo_bar (all lowercase) + * + * For example: + * + * {{{ + * "HelloWorld".toSnakeCase + * }}} + * + * will return the String `"hello_world"`. + */ + def toSnakeCase: String = StringOps.toSnakeCase(string) } def hexlify(array: Array[Byte], from: Int, to: Int): String = { @@ -133,6 +172,95 @@ object StringOps { out.toString } + private[this] val SnakeCaseRegexFirstPass = """([A-Z]+)([A-Z][a-z])""".r + private[this] val SnakeCaseRegexSecondPass = """([a-z\d])([A-Z])""".r + + /** + * Turn a string of format "FooBar" into snake case "foo_bar" + * + * @note toSnakeCase is not reversible, ie. in general the following will _not_ be true: + * + * {{{ + * s == toCamelCase(toSnakeCase(s)) + * }}} + * + * Copied from the Lift Framework and RENAMED: + * https://github.com/lift/framework/blob/master/core/util/src/main/scala/net/liftweb/util/StringHelpers.scala + * Apache 2.0 License: https://github.com/lift/framework/blob/master/LICENSE.txt + * + * @return the underscored string + */ + def toSnakeCase(name: String): String = + SnakeCaseRegexSecondPass + .replaceAllIn( + SnakeCaseRegexFirstPass + .replaceAllIn(name, "$1_$2"), + "$1_$2" + ) + .toLowerCase + + /** + * Turns a string of format "foo_bar" into PascalCase "FooBar" + * + * @note Camel case may start with a capital letter (called "Pascal Case" or "Upper Camel Case") or, + * especially often in programming languages, with a lowercase letter. In this code's + * perspective, PascalCase means the first char is capitalized while camelCase means the first + * char is lowercased. In general both can be considered equivalent although by definition + * "CamelCase" is a valid camel-cased word. Hence, PascalCase can be considered to be a + * subset of camelCase. + * + * Copied from the "lift" framework and RENAMED: + * https://github.com/lift/framework/blob/master/core/util/src/main/scala/net/liftweb/util/StringHelpers.scala + * Apache 2.0 License: https://github.com/lift/framework/blob/master/LICENSE.txt + * Functional code courtesy of Jamie Webb (j@jmawebb.cjb.net) 2006/11/28 + * + * @param name the String to PascalCase + * @return the PascalCased string + */ + def toPascalCase(name: String): String = { + def loop(x: List[Char]): List[Char] = (x: @unchecked) match { + case '_' :: '_' :: rest => loop('_' :: rest) + case '_' :: c :: rest => Character.toUpperCase(c) :: loop(rest) + case '_' :: Nil => Nil + case c :: rest => c :: loop(rest) + case Nil => Nil + } + + if (name == null) { + "" + } else { + loop('_' :: name.toList).mkString + } + } + + /** + * Turn a string of format "foo_bar" into camelCase with the first letter in lower case: "fooBar" + * + * This function is especially used to camelCase method names. + * + * @note Camel case may start with a capital letter (called "Pascal Case" or "Upper Camel Case") or, + * especially often in programming languages, with a lowercase letter. In this code's + * perspective, PascalCase means the first char is capitalized while camelCase means the first + * char is lowercased. In general both can be considered equivalent although by definition + * "CamelCase" is a valid camel-cased word. Hence, PascalCase can be considered to be a + * subset of camelCase. + * + * Copied from the Lift Framework and RENAMED: + * https://github.com/lift/framework/blob/master/core/util/src/main/scala/net/liftweb/util/StringHelpers.scala + * Apache 2.0 License: https://github.com/lift/framework/blob/master/LICENSE.txt + * + * @param name the String to camelCase + * @return the camelCased string + */ + def toCamelCase(name: String): String = { + val tmp: String = toPascalCase(name) + if (tmp.length == 0) { + "" + } else { + tmp.substring(0, 1).toLowerCase + tmp.substring(1) + } + } + implicit final class RichByteArray(val bytes: Array[Byte]) extends AnyVal { /** diff --git a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala b/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala index 4ac9d3d6a5..800e696e5f 100644 --- a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala @@ -16,45 +16,132 @@ package com.twitter.util -import org.junit.runner.RunWith -import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner - import com.twitter.conversions.StringOps._ +import org.scalatest.FunSuite + +class StringConversionsTest extends FunSuite { + + test("string#quoteC") { + assert("nothing".quoteC == "nothing") + assert( + "name\tvalue\t\u20acb\u00fcllet?\u20ac".quoteC == "name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac" + ) + assert("she said \"hello\"".quoteC == "she said \\\"hello\\\"") + assert("\\backslash".quoteC == "\\\\backslash") + } + + test("string#unquoteC") { + assert("nothing".unquoteC == "nothing") + assert( + "name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac".unquoteC == "name\tvalue\t\u20acb\u00fcllet?\u20ac" + ) + assert("she said \\\"hello\\\"".unquoteC == "she said \"hello\"") + assert("\\\\backslash".unquoteC == "\\backslash") + assert("real\\$dollar".unquoteC == "real\\$dollar") + assert("silly\\/quote".unquoteC == "silly/quote") + } + + test("string#hexlify") { + assert("hello".getBytes.slice(1, 4).hexlify == "656c6c") + assert("hello".getBytes.hexlify == "68656c6c6f") + } + + test("string#unhexlify") { + assert("656c6c".unhexlify.toList == "hello".getBytes.slice(1, 4).toList) + assert("68656c6c6f".unhexlify.toList == "hello".getBytes.toList) + "5".unhexlify + assert("5".unhexlify.hexlify.toInt == 5) + } + + test("string#toCamelCase") { + assert("foo_bar".toCamelCase == "fooBar") + + assert(toCamelCase("foo_bar") == "fooBar") + assert(toPascalCase("foo_bar") == "FooBar") + + assert(toCamelCase("foo__bar") == "fooBar") + assert(toPascalCase("foo__bar") == "FooBar") + + assert(toCamelCase("foo___bar") == "fooBar") + assert(toPascalCase("foo___bar") == "FooBar") + + assert(toCamelCase("_foo_bar") == "fooBar") + assert(toPascalCase("_foo_bar") == "FooBar") + + assert(toCamelCase("foo_bar_") == "fooBar") + assert(toPascalCase("foo_bar_") == "FooBar") + + assert(toCamelCase("_foo_bar_") == "fooBar") + assert(toPascalCase("_foo_bar_") == "FooBar") + + assert(toCamelCase("a_b_c_d") == "aBCD") + assert(toPascalCase("a_b_c_d") == "ABCD") + } + + test("string#toPascalCase") { + assert("foo_bar".toPascalCase == "FooBar") + } + + test("string#toSnakeCase") { + assert("FooBar".toSnakeCase == "foo_bar") + } + + test("string#toPascalCase return an empty string if given null") { + assert(toPascalCase(null) == "") + } + + test("string#toPascalCase leave a CamelCased name untouched") { + assert(toPascalCase("GetTweets") == "GetTweets") + assert(toPascalCase("FooBar") == "FooBar") + assert(toPascalCase("HTML") == "HTML") + assert(toPascalCase("HTML5") == "HTML5") + assert(toPascalCase("Editor2TOC") == "Editor2TOC") + } + + test("string#toCamelCase Method") { + assert(toCamelCase("GetTweets") == "getTweets") + assert(toCamelCase("FooBar") == "fooBar") + assert(toCamelCase("HTML") == "hTML") + assert(toCamelCase("HTML5") == "hTML5") + assert(toCamelCase("Editor2TOC") == "editor2TOC") + } + + test("string#toPascalCase & toCamelCase Method function") { + // test both produce empty strings + toPascalCase(null).isEmpty && toCamelCase(null).isEmpty + + val name = "emperor_norton" + // test that the first letter for toCamelCase is lower-cased + assert(toCamelCase(name).toList.head.isLower) + // test that toPascalCase and toCamelCase.capitalize are the same + assert(toPascalCase(name) == toCamelCase(name).capitalize) + } + + test("string#toSnakeCase replace upper case with underscore") { + assert(toSnakeCase("MyCamelCase") == "my_camel_case") + assert(toSnakeCase("CamelCase") == "camel_case") + assert(toSnakeCase("Camel") == "camel") + assert(toSnakeCase("MyCamel12Case") == "my_camel12_case") + assert(toSnakeCase("CamelCase12") == "camel_case12") + assert(toSnakeCase("Camel12") == "camel12") + assert(toSnakeCase("Foobar") == "foobar") + } + + test("string#toSnakeCase not modify existing snake case strings") { + assert(toSnakeCase("my_snake_case") == "my_snake_case") + assert(toSnakeCase("snake") == "snake") + } + + test("string#toSnakeCase handle abbeviations") { + assert(toSnakeCase("ABCD") == "abcd") + assert(toSnakeCase("HTML") == "html") + assert(toSnakeCase("HTMLEditor") == "html_editor") + assert(toSnakeCase("EditorTOC") == "editor_toc") + assert(toSnakeCase("HTMLEditorTOC") == "html_editor_toc") -@RunWith(classOf[JUnitRunner]) -class StringConversionsTest extends WordSpec { - "string" should { - "quoteC" in { - assert("nothing".quoteC == "nothing") - assert( - "name\tvalue\t\u20acb\u00fcllet?\u20ac".quoteC == "name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac" - ) - assert("she said \"hello\"".quoteC == "she said \\\"hello\\\"") - assert("\\backslash".quoteC == "\\\\backslash") - } - - "unquoteC" in { - assert("nothing".unquoteC == "nothing") - assert( - "name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac".unquoteC == "name\tvalue\t\u20acb\u00fcllet?\u20ac" - ) - assert("she said \\\"hello\\\"".unquoteC == "she said \"hello\"") - assert("\\\\backslash".unquoteC == "\\backslash") - assert("real\\$dollar".unquoteC == "real\\$dollar") - assert("silly\\/quote".unquoteC == "silly/quote") - } - - "hexlify" in { - assert("hello".getBytes.slice(1, 4).hexlify == "656c6c") - assert("hello".getBytes.hexlify == "68656c6c6f") - } - - "unhexlify" in { - assert("656c6c".unhexlify.toList == "hello".getBytes.slice(1, 4).toList) - assert("68656c6c6f".unhexlify.toList == "hello".getBytes.toList) - "5".unhexlify - assert("5".unhexlify.hexlify.toInt == 5) - } + assert(toSnakeCase("HTML5") == "html5") + assert(toSnakeCase("HTML5Editor") == "html5_editor") + assert(toSnakeCase("Editor2TOC") == "editor2_toc") + assert(toSnakeCase("HTML5Editor2TOC") == "html5_editor2_toc") } } From 929d0d90ef319045b5cf7e4883cce0e8d51f8b3b Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 5 Mar 2019 23:34:24 +0000 Subject: [PATCH 126/796] Revert "util-core: Don't allow AsyncSemaphore Permits to be released several times" This reverts commit f0034171e8a00446bf0184918333fba367aab21e. Differential Revision: https://phabricator.twitter.biz/D281873 --- CHANGELOG.rst | 4 ---- .../twitter/concurrent/AsyncSemaphore.scala | 23 ++++++++----------- .../concurrent/AsyncSemaphoreTest.scala | 19 --------------- 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 275d73d620..caaf81cbaa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -57,10 +57,6 @@ Runtime Behavior Changes before the pipe would wait indefinitely for a read before transitioning to the Closed state. ``PHAB_ID=D229728`` -* util-core: Don't allow `AsyncSemaphore` `Permit`s to be released multiple times. Before it was - possible to release a permit more than once and incorrectly remove waiters from the queue. - With this change, the `release` permit operation is idempotent. ``PHAB_ID=D274017`` - Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala index a3efe46e29..75e64b59f4 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala @@ -59,33 +59,30 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { // Serves as our intrinsic lock. private[this] final def lock: Object = waitq - private[this] def semaphorePermit(): Permit = new Permit { - // access to `released` is synchronized by locking on the `lock` above. - private[this] var released = false + private[this] val semaphorePermit = new Permit { + private[this] val ReturnThis = Return(this) @tailrec override def release(): Unit = { val waiter = lock.synchronized { - if (released) null - else { - val next = waitq.pollFirst() - if (next == null) { - availablePermits += 1 - } - released = true - next + val next = waitq.pollFirst() + if (next == null) { + availablePermits += 1 } + next } if (waiter != null) { // If the waiter is already satisfied, it must // have been interrupted, so we can simply move on. - if (!waiter.updateIfEmpty(Return(semaphorePermit()))) { + if (!waiter.updateIfEmpty(ReturnThis)) { release() } } } } + private[this] val futurePermit = Future.value(semaphorePermit) + def numWaiters: Int = lock.synchronized(waitq.size) def numInitialPermits: Int = initialPermits def numPermitsAvailable: Int = lock.synchronized(availablePermits) @@ -125,7 +122,7 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { if (availablePermits > 0) { availablePermits -= 1 - Future.value(semaphorePermit()) + futurePermit } else { maxWaiters match { case Some(max) if waitq.size >= max => diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index e7ff1dda0b..63085ff026 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -390,24 +390,5 @@ class AsyncSemaphoreTest extends FunSpec { val msgs = results.collect { case Some(Throw(e)) => e.getMessage } assert(msgs.forall(_ == "woop")) } - - it("should be idempotent and not release a permit twice") { _ => - val semaphore = new AsyncSemaphore(1) - val (p1, p2, p3) = (semaphore.acquire(), semaphore.acquire(), semaphore.acquire()) - - assert(p1.isDefined) - assert(!p2.isDefined) - assert(!p3.isDefined) - assert(semaphore.numWaiters == 2) - - val permit1 = await(p1) - permit1.release() - permit1.release() - - assert(p1.isDefined) - assert(p2.isDefined) - assert(!p3.isDefined) - assert(semaphore.numWaiters == 1) - } } } From 86f6752c2d628381f66eda0920ecef9d44cee8a2 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Mon, 11 Mar 2019 18:19:46 +0000 Subject: [PATCH 127/796] util-jvm: Make Hotspot.foreachGc more robust Problem The implemenation of `Hotspot.foreachGc` relies on the default implementation from `Jvm`. This in turn relies on a mechanism that can (mostly silently) miss garbage collection events. Solution Have a Hotspot-specific implementation that relies on `GarbageCollectionNotificationInfo`, in a similar fashion to what is done for tracking allocations. Result `Hotspot.foreachGc` is more robust in that it should no longer miss garbage collection events. JIRA Issues: CSL-7663 Differential Revision: https://phabricator.twitter.biz/D281962 --- .../main/scala/com/twitter/jvm/Hotspot.scala | 94 ++++++++++++++++++- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index 556a276294..6afdbcfcac 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -5,14 +5,20 @@ import com.twitter.conversions.DurationOps._ import com.twitter.util.Time import java.lang.management.ManagementFactory import java.util.logging.{Level, Logger} -import javax.management.openmbean.CompositeDataSupport -import javax.management.{ObjectName, RuntimeMBeanException} +import javax.management.openmbean.{CompositeData, CompositeDataSupport} +import javax.management.{ + Notification, + NotificationEmitter, + NotificationListener, + ObjectName, + RuntimeMBeanException +} import scala.collection.JavaConverters._ import scala.language.reflectiveCalls class Hotspot extends Jvm { private[this] val epoch = - Time.fromMilliseconds(ManagementFactory.getRuntimeMXBean().getStartTime()) + Time.fromMilliseconds(ManagementFactory.getRuntimeMXBean.getStartTime) private[this] type Counter = { def getName(): String @@ -37,8 +43,7 @@ class Hotspot extends Jvm { private[this] def opt(name: String) = try Some { - val o = ManagementFactory - .getPlatformMBeanServer() + val o = ManagementFactory.getPlatformMBeanServer .invoke(DiagnosticBean, "getVMOption", Array(name), Array("java.lang.String")) o.asInstanceOf[CompositeDataSupport].get("value").asInstanceOf[String] } catch { @@ -82,6 +87,25 @@ class Hotspot extends Jvm { } yield Gc(invocations, kind, epoch + lastEntryTime, duration) } + private[this] def newGcListener(f: Gc => Unit): NotificationListener = + new NotificationListener { + def handleNotification(notification: Notification, handback: Any): Unit = { + if (Hotspot.isGcNotification(notification)) { + val gc = + Hotspot.gcFromNotificationInfo(notification.getUserData.asInstanceOf[CompositeData]) + f(gc) + } + } + } + + override def foreachGc(f: Gc => Unit): Unit = { + ManagementFactory.getGarbageCollectorMXBeans.asScala.foreach { + case bean: NotificationEmitter => + bean.addNotificationListener(newGcListener(f), null, null) + case _ => () + } + } + def snap: Snapshot = { val cs = counters("") val heap = for { @@ -186,3 +210,63 @@ class Hotspot extends Jvm { def forceGc(): Unit = System.gc() } + +private object Hotspot { + + // this inlines the value of GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION + // to avoid reflection + def isGcNotification(notification: Notification): Boolean = + notification.getType == "com.sun.management.gc.notification" + + private[this] val jvmStart = + Time.fromMilliseconds(ManagementFactory.getRuntimeMXBean.getStartTime) + + private[this] val gcNotifInfoFromMethod = + Class + .forName("com.sun.management.GarbageCollectionNotificationInfo") + .getMethod("from", classOf[CompositeData]) + + private[this] val gcNotifInfoGetGcInfoMethod = + Class + .forName("com.sun.management.GarbageCollectionNotificationInfo") + .getMethod("getGcInfo") + + private[this] val gcNotifInfoGetGcNameMethod = + Class + .forName("com.sun.management.GarbageCollectionNotificationInfo") + .getMethod("getGcName") + + private[this] val gcInfoGetIdMethod = + Class + .forName("com.sun.management.GcInfo") + .getMethod("getId") + + private[this] val gcInfoGetStartTimeMethod = + Class + .forName("com.sun.management.GcInfo") + .getMethod("getStartTime") + + private[this] val gcInfoGetDurationMethod = + Class + .forName("com.sun.management.GcInfo") + .getMethod("getDuration") + + /** + * This uses reflection because we the `com.sun` classes may not be available + * at compilation time. + */ + def gcFromNotificationInfo(compositeData: CompositeData): Gc = { + val gcNotifInfo /* com.sun.management.GarbageCollectionNotificationInfo */ = + gcNotifInfoFromMethod.invoke(compositeData) + val gcInfo /* com.sun.management.GcInfo */ = + gcNotifInfoGetGcInfoMethod.invoke(gcNotifInfo) + Gc( + count = gcInfoGetIdMethod.invoke(gcInfo).asInstanceOf[Long], + name = gcNotifInfoGetGcNameMethod.invoke(gcNotifInfo).asInstanceOf[String], + timestamp = jvmStart + gcInfoGetStartTimeMethod + .invoke(gcInfo).asInstanceOf[Long].milliseconds, + duration = gcInfoGetDurationMethod.invoke(gcInfo).asInstanceOf[Long].milliseconds + ) + } + +} From 0cc640ac8ff79218fd1d0fe165bbdb647f33a9f6 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Tue, 12 Mar 2019 23:41:08 +0000 Subject: [PATCH 128/796] util-core: Discard parent reader in `Reader.flatten` when child reader encounters an exception Problem Parent reader's status remains the same when child reader encounters an exception. Solution Discard parent reader when child reader encounters an exception. JIRA Issues: CSL-7595 Differential Revision: https://phabricator.twitter.biz/D281830 --- CHANGELOG.rst | 3 +++ util-core/src/main/scala/com/twitter/io/Reader.scala | 10 +++++++--- .../src/test/scala/com/twitter/io/ReaderTest.scala | 4 +++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index caaf81cbaa..bb8dcb435f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-core: Discard parent reader from `Reader.flatten` when child reader encounters an exception. + ``PHAB_ID=D281830`` + * util-core: Added `c.t.conversions.StringOps#toSnakeCase,toCamelCase,toPascalCase` implementations. ``PHAB_ID=D280886`` diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 03c17b776c..bda9519fb1 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -473,9 +473,13 @@ object Reader { currentReader.read().transform { case Return(None) => updateCurrentAndRead() case Return(sa) => Future.value(sa) - case Throw(t) => - closep.updateIfEmpty(Throw(t)) - Future.exception(t) + case t @ Throw(_) => + // update `closep` with the exception thrown before discarding readers, + // because discard will update `closep` to a `Discarded` StreamTermination + // and return a `ReaderDiscardedException` during reading. + closep.updateIfEmpty(t.cast[StreamTermination]) + readers.discard() + Future.const(t) } } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 6c6d588bf1..a16c4f6db5 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -257,8 +257,10 @@ class ReaderTest def discard(): Unit = p.setException(new ReaderDiscardedException) def onClose: Future[StreamTermination] = Future.never } - val reader = Reader.flatten(Reader.fromSeq(head +: undefinedReaders)) + val pReader = Reader.fromSeq(head +: undefinedReaders) + val reader = Reader.flatten(pReader) assertFailed(reader, p) + assert(await(pReader.onClose) == StreamTermination.Discarded) } test("Reader.flatten - onClose is satisfied when fully read") { From 967c2b5c726101a0a85f09f8c16e2a1ea5c083a2 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 13 Mar 2019 18:00:29 +0000 Subject: [PATCH 129/796] Twitter-oss: Prepare OSS libraries for release 19.3.0 Problem We want to release the next versions of our Twitter OSS libraries 19.3.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D285479 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bb8dcb435f..4ced63eb52 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.3.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index d9ab402e6e..2f217debec 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.2.0" +val collUtils = "com.twitter" %% "util-collection" % "19.3.0" ``` # Units diff --git a/build.sbt b/build.sbt index 1d96fc60ef..2a270eb589 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.3.0-SNAPSHOT" +val releaseVersion = "19.3.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From f06508e5dd097df47f255345f87621a62a9f8b09 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 13 Mar 2019 21:01:56 +0000 Subject: [PATCH 130/796] Twitter-oss: update OSS libraries post-release to 19.4.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.4.0. - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D285562 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2f217debec..f95ddbdfdd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 2a270eb589..0cdae68462 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.3.0" +val releaseVersion = "19.4.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From db8ad2a9de7ea4f80e89569ac7b957816efd9156 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Tue, 19 Mar 2019 00:34:16 +0000 Subject: [PATCH 131/796] util: Use a state machine in Reader.fromSeq Problem Some race conditions exist in `Reader.fromSeq` Solution Use a state machine to monitor the state and update `closep` only when state is updated Result No more race conditions in `Reader.fromSeq`, and keep one state machine for `Reader.fromFuture` and `Reader.fromSeq` JIRA Issues: CSL-7659 Differential Revision: https://phabricator.twitter.biz/D280723 --- .../scala/com/twitter/io/FutureReader.scala | 104 +++++++++++++++++ .../main/scala/com/twitter/io/Reader.scala | 110 +----------------- .../main/scala/com/twitter/io/SeqReader.scala | 78 +++++++++++++ 3 files changed, 186 insertions(+), 106 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/io/FutureReader.scala create mode 100644 util-core/src/main/scala/com/twitter/io/SeqReader.scala diff --git a/util-core/src/main/scala/com/twitter/io/FutureReader.scala b/util-core/src/main/scala/com/twitter/io/FutureReader.scala new file mode 100644 index 0000000000..ba7ea1f48e --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/FutureReader.scala @@ -0,0 +1,104 @@ +package com.twitter.io + +import com.twitter.util.{Future, Promise, Return, Throw} +import java.util.concurrent.atomic.AtomicReference + +/** + * We want to ensure that this reader always satisfies these invariants: + * 1. Satisfied closep is always aligned with the state + * 2. Reading from a discarded reader will always return ReaderDiscardedException + * 3. Reading from a fully read reader will always return None + * 4. Reading from a failed reader will always return the exception it has thrown + * + * We achieved this with a state machine where any access to the state is synchronized, + * and by ensuring that we always verify the update of state succeeded before setting closep, + * and by preventing changing the state when the reader is failed, fully read, or discarded + */ +private[io] final class FutureReader[A](fa: Future[A]) extends Reader[A] { + import FutureReader._ + + private[this] val closep = Promise[StreamTermination]() + private[this] val state = new AtomicReference[State](State.Idle) + + def read(): Future[Option[A]] = { + state.get() match { + case State.Idle => + if (state.compareAndSet(State.Idle, State.Reading)) { + fa.map(Some.apply).respond { + case t: Throw[_] => + if (state.compareAndSet(State.Reading, State.Failed)) { + closep.update(t.cast[StreamTermination]) + } + case Return(_) => + // it is safe to return the value even when the `state` is not updated, which could happen + // when the `state` is updated to `Discarded` in `discard()`, in that case, we still retain + // the execution order because `read()` happens before `discard()` + state.compareAndSet(State.Reading, State.Read) + } + } else { + // when `state` is updated to `Reading` or `Discarded` + read() + } + case State.Reading => + // multiple outstanding reads are not allowed + Future.exception(new IllegalStateException("read() while read is pending")) + case State.Read => + // update would fail if `state` is already updated to `Discarded` in `discard()` + if (state.compareAndSet(State.Read, State.FullyRead)) { + closep.update(StreamTermination.FullyRead.Return) + } + closep.flatMap { + case StreamTermination.FullyRead => Future.None + case StreamTermination.Discarded => Future.exception(new ReaderDiscardedException) + } + case State.Failed => + // closep is guaranteed to be an exception, flatMap should never be triggered but return the exception + closep.flatMap(_ => Future.None) + case State.FullyRead => + Future.None + case State.Discarded => + Future.exception(new ReaderDiscardedException) + } + } + + def discard(): Unit = { + if (state.compareAndSet(State.Idle, State.Discarded) || state + .compareAndSet(State.Read, State.Discarded) || state.compareAndSet( + State.Reading, + State.Discarded)) { + closep.update(StreamTermination.Discarded.Return) + fa.raise(new ReaderDiscardedException) + } + } + + def onClose: Future[StreamTermination] = closep +} + +object FutureReader { + + /** + * Indicates reader state when the reader is created via FutureReader + */ + sealed trait State + object State { + + /** Indicates the reader is ready to be read. */ + case object Idle extends State + + /** Indicates the reader has been read. */ + case object Read extends State + + /** Indicates a reading is in progress. */ + case object Reading extends State + + /** Indicates an exception occurred during reading */ + case object Failed extends State + + /** Indicates the EOS has been observed. */ + case object FullyRead extends State + + /** Indicates the reader has been discarded. */ + case object Discarded extends State + + } +} diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index bda9519fb1..02d6f4d03f 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -3,7 +3,6 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.util._ import java.io.{File, FileInputStream, InputStream} -import java.util.concurrent.atomic.AtomicReference import scala.annotation.tailrec import scala.collection.mutable.ListBuffer @@ -212,63 +211,9 @@ object Reader { /** * Construct a `Reader` from a `Future` * - * We want to ensure that this reader always satisfies these invariants: - * 1. Satisfied closep is always aligned with the state - * 2. Reading from a discarded reader will always return ReaderDiscardedException - * 3. Reading from a fully read reader will always return None - * 4. Reading from an exceptioned reader will always return the exception it has thrown - * - * We achieved this with a state machine where any access to the state is synchronized, - * and by ensuring that we always verify the update of state succeeded before setting closep, - * and by preventing changing the state when the reader is exceptioned, fully read, or discarded - * * @note Multiple outstanding reads are not allowed on this reader - * */ - def fromFuture[A](fa: Future[A]): Reader[A] = new Reader[A] { - - private[this] val closep = Promise[StreamTermination]() - private[this] val state = new AtomicReference[State](State.Idle) - - final def read(): Future[Option[A]] = { - state.get() match { - case State.Idle => - fa.map(Some.apply).respond { - case t: Throw[_] => - if (state.compareAndSet(State.Idle, State.Exception)) { - closep.update(t.cast[StreamTermination]) - } - case Return(_) => - state.compareAndSet(State.Idle, State.Read) - } - case State.Read => - if (state.compareAndSet(State.Read, State.FullyRead)) { - closep.update(StreamTermination.FullyRead.Return) - } - closep.flatMap { - case StreamTermination.FullyRead => Future.None - case StreamTermination.Discarded => Future.exception(new ReaderDiscardedException) - } - case State.Exception => - /** closep is guaranteed to be an exception, flatMap should never be triggered but return the exception */ - closep.flatMap(_ => Future.None) - case State.FullyRead => - Future.None - case State.Discard => - Future.exception(new ReaderDiscardedException) - } - } - - final def discard(): Unit = { - if (state.compareAndSet(State.Idle, State.Discard) || state - .compareAndSet(State.Read, State.Discard)) { - closep.update(StreamTermination.Discarded.Return) - fa.raise(new ReaderDiscardedException) - } - } - - final def onClose: Future[StreamTermination] = closep - } + def fromFuture[A](fa: Future[A]): Reader[A] = new FutureReader(fa) /** * Construct a `Reader` from a value `a` @@ -363,37 +308,10 @@ object Reader { * * The resources held by the returned [[Reader]] are released on reading of EOF and * [[Reader.discard()]]. + * + * @note Multiple outstanding reads are not allowed on this reader. */ - def fromSeq[A](seq: Seq[A]): Reader[A] = new Reader[A] { self => - private[this] val closep = Promise[StreamTermination]() - private[this] var state: Try[Seq[A]] = Return(seq) - - def read(): Future[Option[A]] = { - val result: Future[Option[A]] = self.synchronized { - state match { - case Return(Nil) => Future.None - case Return(head +: tail) => - state = Return(tail) - Future.value(Some(head)) - case t: Throw[_] => - Future.const(t.cast[Option[A]]) - } - } - if (result eq Future.None) { - closep.updateIfEmpty(StreamTermination.FullyRead.Return) - } - result - } - - def discard(): Unit = { - self.synchronized { - state = Throw(new ReaderDiscardedException) - } - closep.updateIfEmpty(StreamTermination.Discarded.Return) - } - - def onClose: Future[StreamTermination] = closep - } + def fromSeq[A](seq: Seq[A]): Reader[A] = new SeqReader(seq) /** * Allow [[AsyncStream]] to be consumed as a [[Reader]] @@ -540,24 +458,4 @@ object Reader { */ def framed(r: Reader[Buf], framer: Buf => Seq[Buf]): Reader[Buf] = new Framed(r, framer) - private sealed trait State - - private object State { - - /** Indicates no actions are taken on the Reader. */ - case object Idle extends State - - /** Indicates the reader has been read once. */ - case object Read extends State - - /** Indicates an exception occurred during reading */ - case object Exception extends State - - /** Indicates the reader is fully read. */ - case object FullyRead extends State - - /** Indicates the reader has been discarded. */ - case object Discard extends State - } - } diff --git a/util-core/src/main/scala/com/twitter/io/SeqReader.scala b/util-core/src/main/scala/com/twitter/io/SeqReader.scala new file mode 100644 index 0000000000..b19d1c4935 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/SeqReader.scala @@ -0,0 +1,78 @@ +package com.twitter.io + +import com.twitter.util.{Future, Promise} + +/** + * We want to ensure that this reader always satisfies these invariants: + * 1. Reading from a discarded reader will always return ReaderDiscardedException + * 2. Reading from a fully read reader will always return None + * + * We achieved this with a state machine where any access to the state is synchronized, + * and by preventing changing the state when the reader is fully read or discarded. + */ +private[io] final class SeqReader[A](seq: Seq[A]) extends Reader[A] { + import SeqReader._ + + private[this] val closep = Promise[StreamTermination]() + private[this] var value: Seq[A] = seq + private[this] var state: State = State.Idle + + def read(): Future[Option[A]] = { + val result = synchronized { + state match { + case State.Idle => + value match { + case head +: tail => + value = tail + Future.value(Some(head)) + case Nil => + state = State.FullyRead + Future.None + } + case State.FullyRead => + Future.None + case State.Discarded => + Future.exception(new ReaderDiscardedException) + } + } + + if (result.eq(Future.None)) + closep.updateIfEmpty(StreamTermination.FullyRead.Return) + + result + } + + def discard(): Unit = { + val discarded = synchronized { + state match { + case State.Idle => + state = State.Discarded + true + case _ => false + } + } + if (discarded) closep.updateIfEmpty(StreamTermination.Discarded.Return) + + } + + def onClose: Future[StreamTermination] = closep +} + +object SeqReader { + + /** + * Indicates reader state when the reader is created via SeqReader + */ + sealed trait State + object State { + + /** Indicates the reader is ready to be read. */ + case object Idle extends State + + /** Indicates the reader is fully read. */ + case object FullyRead extends State + + /** Indicates the reader has been discarded. */ + case object Discarded extends State + } +} From 0d9dded32b931d6fff7335b5f4c573c474b5b65a Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Thu, 21 Mar 2019 01:47:44 +0000 Subject: [PATCH 132/796] util-app: improve c.t.a.Flag usage by providing a 'letParse' method Problem If a `Flag` is used in a manner that results in type erasure (i.e. contained within a collection of `Flag`), there is no way of using a `Flag.let` without knowing the `Flag`'s `T` type. Solution Introduce a `letParse` method, which will use the underlying `flaggable` to parse and `let` using the correct type associated with the `Flag`. Result Users of Flags can now call `letParse` without needing to explicitly know or reference the Flag's `T: Flaggable` type. JIRA Issues: CSL-7824 Differential Revision: https://phabricator.twitter.biz/D288549 --- CHANGELOG.rst | 6 ++++ .../src/main/scala/com/twitter/app/Flag.scala | 12 +++++++ .../test/scala/com/twitter/app/FlagTest.scala | 32 +++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4ced63eb52..43e2cf078e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-app: Improve usage of `Flag.let` by providing a `Flag.letParse` method + ``PHAB_ID=D288549`` + 19.3.0 ------ diff --git a/util-app/src/main/scala/com/twitter/app/Flag.scala b/util-app/src/main/scala/com/twitter/app/Flag.scala index d497cff166..1a551c720f 100644 --- a/util-app/src/main/scala/com/twitter/app/Flag.scala +++ b/util-app/src/main/scala/com/twitter/app/Flag.scala @@ -146,17 +146,29 @@ class Flag[T: Flaggable] private[app] ( * Override the value of this flag with `t`, only for the scope of the current * [[com.twitter.util.Local]] for the given function `f`. * + * @see [[letParse]] * @see [[letClear]] */ def let[R](t: T)(f: => R): R = let(Some(t), f) + /** + * Override the value of this flag with `arg` [[String] parsed to this [[Flag]]'s [[T]] type, + * only for the scope of the current [[com.twitter.util.Local]] for the given function `f`. + * + * @see [[let]] + * @see [[letClear]] + */ + def letParse[R](arg: String)(f: => R): R = + let(flaggable.parse(arg))(f) + /** * Unset the value of this flag, such that [[isDefined]] will return `false`, * only for the scope of the current [[com.twitter.util.Local]] for the * given function `f`. * * @see [[let]] + * @see [[letParse]] */ def letClear[R](f: => R): R = let(None, f) diff --git a/util-app/src/test/scala/com/twitter/app/FlagTest.scala b/util-app/src/test/scala/com/twitter/app/FlagTest.scala index d92dfb04d5..4dcc3149c1 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagTest.scala @@ -86,6 +86,38 @@ class FlagTest extends FunSuite { assert(res2 == "false") } + test("Flag: letParse") { + def current: Boolean = MyGlobalBooleanFlag() + + // track the order the blocks execute and that they only execute once + var buf = mutable.Buffer[Int]() + + // make sure they stack properly + assert(!current) + MyGlobalBooleanFlag.letParse("true") { + buf += 1 + assert(current) + MyGlobalBooleanFlag.letParse("false") { + buf += 2 + assert(!current) + } + buf += 3 + assert(current) + } + assert(!current) + + assert(buf == Seq(1, 2, 3)) + } + + test("Flag: letParse return values reflect bindings") { + def current: Boolean = MyGlobalBooleanFlag() + + val res1 = MyGlobalBooleanFlag.letParse("true") { current.toString } + assert(res1 == "true") + val res2 = MyGlobalBooleanFlag.letParse("false") { current.toString } + assert(res2 == "false") + } + test("Flag: letClear") { // track the order the blocks execute and that they only execute once var buf = mutable.Buffer[Int]() From 11ef2104d02c42ee4f16e3231c1ae034592a7fb9 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 26 Mar 2019 16:13:09 +0000 Subject: [PATCH 133/796] util, scrooge, twitter-server: Update to Scala 2.12.8 in build.sbt and Travis CI Problem/Solution Update sbt default and cross building Scala version to 2.12.8 along with the Travis CI configuration. Differential Revision: https://phabricator.twitter.biz/D291345 --- .travis.yml | 2 +- build.sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 22e5acd724..0c8ac66297 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ before_cache: scala: - 2.11.12 - - 2.12.7 + - 2.12.8 jdk: - oraclejdk8 diff --git a/build.sbt b/build.sbt index 0cdae68462..9cec806ca5 100644 --- a/build.sbt +++ b/build.sbt @@ -21,8 +21,8 @@ val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.13.4" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion val defaultProjectSettings = Seq( - scalaVersion := "2.12.7", - crossScalaVersions := Seq("2.11.12", "2.12.7") + scalaVersion := "2.12.8", + crossScalaVersions := Seq("2.11.12", "2.12.8") ) val baseSettings = Seq( From 80ec6230ecfbc6b13db82ed31f84fd67c06e9532 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 26 Mar 2019 18:28:19 +0000 Subject: [PATCH 134/796] util,scrooge,fingle,twitter-server,finatra: Update sbt version from 1.1.4 to 1.2.8 Problem/Solution Update sbt version from 1.1.4 to 1.2.8 Differential Revision: https://phabricator.twitter.biz/D291366 --- project/build.properties | 2 +- sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project/build.properties b/project/build.properties index 23050492c6..1fc4b8093e 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.1.4 \ No newline at end of file +sbt.version=1.2.8 \ No newline at end of file diff --git a/sbt b/sbt index 9e21a00f08..f6a4bb6310 100755 --- a/sbt +++ b/sbt @@ -1,8 +1,8 @@ #!/bin/bash -sbtver=1.1.4 +sbtver=1.2.8 sbtjar=sbt-launch.jar -sbtsha128=ac6b13b709e3c0e3a8b2b3bf3031ec57660cb813 +sbtsha128=073c169c6e1a47b8ae78a7a718b907424dedab30 sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From 52d49ad98c3ec3b08218a60d6ede322c0f7edc5e Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 26 Mar 2019 13:44:06 -0700 Subject: [PATCH 135/796] Fix Hotspot.scala Problem / Solution The revert, and accompanying fix, for Hotspot.scala did not get pushed to Github due to a bug in the push script. This addresses that. Meanwhile, the script is being patched to fix that bug. --- util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index 6afdbcfcac..539a6b5296 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -257,7 +257,7 @@ private object Hotspot { */ def gcFromNotificationInfo(compositeData: CompositeData): Gc = { val gcNotifInfo /* com.sun.management.GarbageCollectionNotificationInfo */ = - gcNotifInfoFromMethod.invoke(compositeData) + gcNotifInfoFromMethod.invoke(null /* static method */, compositeData) val gcInfo /* com.sun.management.GcInfo */ = gcNotifInfoGetGcInfoMethod.invoke(gcNotifInfo) Gc( From ed6f6a73a41d1b7e8331687567e3191cd5ead19e Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 2 Apr 2019 16:55:33 +0000 Subject: [PATCH 136/796] twitter-oss: Revert sbt update to 1.2.8 Problem Using `sbt 1.2.8` does not allow our projects to build with Scala 2.11.x. Even following the advice to set the root project `crossScalaVersions` to `Nil` seems to still fail (see: https://www.scala-sbt.org/1.x/docs/Cross-Build.html). Solution Revert back to `sbt 1.1.4` until we can resolve the issues with building against Scala 2.11.x or we drop support for said Scala versions. Result Green TravisCI builds. Differential Revision: https://phabricator.twitter.biz/D294569 --- project/build.properties | 2 +- sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project/build.properties b/project/build.properties index 1fc4b8093e..23050492c6 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.8 \ No newline at end of file +sbt.version=1.1.4 \ No newline at end of file diff --git a/sbt b/sbt index f6a4bb6310..9e21a00f08 100755 --- a/sbt +++ b/sbt @@ -1,8 +1,8 @@ #!/bin/bash -sbtver=1.2.8 +sbtver=1.1.4 sbtjar=sbt-launch.jar -sbtsha128=073c169c6e1a47b8ae78a7a718b907424dedab30 +sbtsha128=ac6b13b709e3c0e3a8b2b3bf3031ec57660cb813 sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From 56867ff507dcb2ac99024fe2526f768bf22abf65 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 4 Apr 2019 01:50:35 +0000 Subject: [PATCH 137/796] util-core: Add AsyncMeter.perSecondUnbounded Problem / Solution There is no discoverable API to building unbounded `AsyncMeter` that allow N per second permits. This change adds `AsyncMeter.perSecondUnbounded(N)`. Differential Revision: https://phabricator.twitter.biz/D295868 --- .../scala/com/twitter/concurrent/AsyncMeter.scala | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala index 58e1e79799..b6a63104a3 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala @@ -31,10 +31,24 @@ object AsyncMeter { * second, and has a maximum burst size of `permits` over one second. * * This is equivalent to `AsyncMeter.newMeter(permits, 1.second, maxWaiters)`. + * + * @see [[perSecondUnbounded()]] for creating an async meter that allows unbounded waiters */ def perSecond(permits: Int, maxWaiters: Int)(implicit timer: Timer): AsyncMeter = newMeter(permits, Duration.fromSeconds(1), maxWaiters) + /** + * Creates an [[AsyncMeter]] that allows smoothed out `permits` per + * second, and has a maximum burst size of `permits` over one second. + * + * This meter is not bounded by the number of allowed waiters. This is equivalent to + * `AsyncMeter.newUnboundedMeter(permits, 1.second, maxWaiters)`. + * + * @see [[perSecond()]] for creating an async meter that only allows a certain number of waiters + */ + def perSecondUnbounded(permits: Int)(implicit timer: Timer): AsyncMeter = + newUnboundedMeter(permits, Duration.fromSeconds(1)) + /** * Creates an [[AsyncMeter]] that allows smoothed out `permits` per second, * and has a maximum burst size of 1 permit over `1.second / permits`. From b1c01238cc99ed4de7a96becbddcd2ff0348e58c Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Mon, 8 Apr 2019 21:53:03 +0000 Subject: [PATCH 138/796] util: Reader.flatten should listen to child reader's onClose before read Problem `Reader.flatten` listens to child reader's `closep` only after performing a `read()`. When child reader encounters an exception, we are only able to detect from parent reader after reading from it. Solution update `Reader.flatten`'s `closep` to be align with child reader's `closep` before `read()` JIRA Issues: CSL-7503 Differential Revision: https://phabricator.twitter.biz/D291050 --- .../main/scala/com/twitter/io/Reader.scala | 53 +++++++++++++------ .../scala/com/twitter/io/ReaderTest.scala | 37 +++++++++++++ 2 files changed, 74 insertions(+), 16 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 02d6f4d03f..bfff87b447 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -1,6 +1,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream +import com.twitter.util.Promise.Detachable import com.twitter.util._ import java.io.{File, FileInputStream, InputStream} import scala.annotation.tailrec @@ -371,26 +372,21 @@ object Reader { * @param readers A Reader holds a stream of Reader[A] */ def flatten[A](readers: Reader[Reader[A]]): Reader[A] = new Reader[A] { self => + // access currentReader and curReaderClosep are synchronized on `self` + private[this] var currentReader: Reader[A] = Reader.empty + private[this] var curReaderClosep: Promise[StreamTermination] with Detachable = + Promise.attached(currentReader.onClose) private[this] val closep = Promise[StreamTermination]() - // access this currentReader is synchronized on `self` - private[this] var currentReader: Reader[A] = Reader.empty readers.onClose.respond(closep.updateIfEmpty) - private def updateCurrentAndRead(): Future[Option[A]] = - readers.read().flatMap { - case Some(reader) => - self.synchronized { currentReader = reader } - read() - case None => - Future.None - } - - def read(): Future[Option[A]] = self.synchronized { - currentReader.read().transform { - case Return(None) => updateCurrentAndRead() - case Return(sa) => Future.value(sa) + def read(): Future[Option[A]] = { + self.synchronized(currentReader).read().transform { + case Return(None) => + updateCurrentAndRead() + case Return(sa) => + Future.value(sa) case t @ Throw(_) => // update `closep` with the exception thrown before discarding readers, // because discard will update `closep` to a `Discarded` StreamTermination @@ -402,11 +398,36 @@ object Reader { } def discard(): Unit = { - self.synchronized(currentReader).discard() + self + .synchronized { + curReaderClosep.detach() + currentReader + }.discard() readers.discard() } def onClose: Future[StreamTermination] = closep + + /** + * Update currentReader and the callback promise to be the `closep` of currentReader + */ + private def updateCurrentAndRead(): Future[Option[A]] = { + readers.read().flatMap { + case Some(reader) => + val curClosep = self.synchronized { + currentReader = reader + currentReader.onClose + } + curReaderClosep = Promise.attached(curClosep) + curReaderClosep.respond { + case Return(StreamTermination.FullyRead) => + case discardedOrFailed => closep.updateIfEmpty(discardedOrFailed) + } + read() + case _ => + Future.None + } + } } /** diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index a16c4f6db5..c62e6b5bff 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -290,6 +290,43 @@ class ReaderTest assert(ex3.getMessage == "stop") } + test("Reader.flatten - listen for onClose update from parent") { + val exceptionMsg = "boom" + val p = new Pipe[Int] + val reader = Reader.flatten(Reader.value(p)) + reader.read() + p.fail(new Exception(exceptionMsg)) + val exception = intercept[Exception] { + await(reader.onClose) + } + assert(exception.getMessage == exceptionMsg) + } + + test("Reader.flatten - re-register `curReaderClosep` to listen to the next reader in the stream") { + val exceptionMsg = "boom" + + val r1 = Reader.value(1) + val p2 = new Promise + val i2 = p2.interruptible + val r2 = Reader.fromFuture(i2) + + def rStreams: Stream[Reader[Any]] = r1 #:: r2 #:: rStreams + + val reader = Reader.fromSeq(rStreams) + val rFlat = reader.flatten + val e = new Exception(exceptionMsg) + + assert(await(rFlat.read()) == Some(1)) + // re-register `curReaderClosep` to listen to r2 + rFlat.read() + i2.raise(e) + val exception = intercept[Exception] { + await(rFlat.onClose) + } + + assert(exception.getMessage == exceptionMsg) + } + test("Reader.fromSeq works on infinite streams") { def ones: Stream[Int] = 1 #:: ones val reader = Reader.fromSeq(ones) From 6f92c31f75c07daeebc2909a7dfb6ec78bfd856b Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 10 Apr 2019 20:52:13 +0000 Subject: [PATCH 139/796] finatra/http: Initial draft of StreamingResponse and StreamingRequest Problem The current StreamingResponse (c.t.finatra.http.response.StreamingResponse) is tightly coupled with AsyncStream, make it impossible to be used with other primitive stream types like Reader. Solution Following the current functionality, we can have a polymorphic StreamingResponse (c.t.finatra.http.streaming.StreamingResponse) abstract over either a Reader or an AsyncStream. It also has the ability to add other primitive stream types support if needed. Symmetrically, a StreamingRequest that works with different streaming primitives can be constructed in a similar way. Result Eventually we would like something like: ``` post("/tweets/streaming") { ids: StreamingRequest[Reader, Long] => val tweets: Reader[Tweet] = tweetsRepository.getByIds(ids.stream) val resp: StreamingResponse[Reader, Tweet] = response.streaming(tweets) resp } ``` JIRA Issues: CSL-7162 Differential Revision: https://phabricator.twitter.biz/D290421 --- .../src/main/java/com/twitter/io/Readers.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index b8a669e3f9..89ba0c2543 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -4,8 +4,13 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.InputStream; +import java.util.List; +import java.util.stream.Stream; + import com.twitter.concurrent.AsyncStream; import com.twitter.util.Future; + +import scala.collection.JavaConverters; import scala.runtime.BoxedUnit; /** @@ -70,4 +75,26 @@ public static Reader newInputStreamReader(InputStream in, int chunkSize) { return Reader$.MODULE$.fromStream(in, chunkSize); } + /** + * See [[com.twitter.io.Reader$#fromSeq(scala.collection.Seq)]] + */ + public static Reader fromSeq(List list) { + return Reader$.MODULE$.fromSeq(JavaConverters.asScalaIteratorConverter(list.iterator()) + .asScala().toSeq()); + } + + /** + * See [[com.twitter.io.Reader$#fromSeq(scala.collection.Seq)]] + */ + public static Reader fromSeq(Stream stream) { + return Reader$.MODULE$.fromSeq(JavaConverters.asScalaIteratorConverter(stream.iterator()) + .asScala().toSeq()); + } + + /** + * See [[com.twitter.io.Reader#toAsyncStream]] + */ + public static AsyncStream toAsyncStream(Reader reader) { + return Reader$.MODULE$.toAsyncStream(reader); + } } From b5915a24a201d565adc3995ea4664f64ed48a6fc Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Wed, 10 Apr 2019 21:47:32 +0000 Subject: [PATCH 140/796] util-app: `GlobalFlag.getAll()` fails to validate flags defined within a package object Problem GlobalFlag.getAll() fails to validate flags defined within a package object. This is caused by Scala compiler not creating static forwarders for objects defined within a package object, so the necessary method can't be called via reflection. Solution Instead of relying on static forwarders, now we're getting Flag object through MODULE$ field Result You can now define global flags within package objects. Signed-off-by: Kevin Oliver Differential Revision: https://phabricator.twitter.biz/D297652 --- .gitignore | 1 - .../scala/com/twitter/app/GlobalFlag.scala | 41 +++++++++++-------- .../com/twitter/app/JavaGlobalFlag.scala | 22 +++++----- .../com/twitter/app/GlobalFlagTest.scala | 19 +++++++++ .../test/scala/com/twitter/app/package.scala | 5 +++ 5 files changed, 58 insertions(+), 30 deletions(-) create mode 100644 util-app/src/test/scala/com/twitter/app/package.scala diff --git a/.gitignore b/.gitignore index a8dfda5e8c..42cfbcc9cc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,5 @@ lib_managed/ .DS_Store .ensime .ivyjars -sbt out/ sbt-launch.jar diff --git a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala index 382aba2ebc..dfa1bad083 100644 --- a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala +++ b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala @@ -1,6 +1,6 @@ package com.twitter.app -import java.lang.reflect.{Method, Modifier} +import java.lang.reflect.Modifier import scala.collection.mutable.ArrayBuffer import scala.util.control.NonFatal @@ -116,29 +116,36 @@ abstract class GlobalFlag[T] private[app] ( object GlobalFlag { - private[app] def get(f: String): Option[Flag[_]] = { - def validMethod(m: Method): Boolean = - m != null && - Modifier.isStatic(m.getModifiers) && - m.getReturnType == classOf[Flag[_]] && - m.getParameterCount == 0 - - def tryMethod(clsName: String, methodName: String): Option[Flag[_]] = + private[app] def get(flagName: String): Option[Flag[_]] = { + def tryMethod(className: String, methodName: String): Option[Flag[_]] = try { - val cls = Class.forName(clsName) + val cls = Class.forName(className) val m = cls.getMethod(methodName) - if (validMethod(m)) - Some(m.invoke(null).asInstanceOf[Flag[_]]) - else - None + val isValid = Modifier.isStatic(m.getModifiers) && + m.getReturnType == classOf[Flag[_]] && + m.getParameterCount == 0 + if (isValid) Some(m.invoke(null).asInstanceOf[Flag[_]]) else None } catch { case _: ClassNotFoundException | _: NoSuchMethodException | _: IllegalArgumentException => None } - tryMethod(f, "getGlobalFlag").orElse { + def tryModuleField(className: String): Option[Flag[_]] = + try { + val cls = Class.forName(className) + val f = cls.getField("MODULE$") + val isValid = Modifier.isStatic(f.getModifiers) && classOf[Flag[_]] + .isAssignableFrom(f.getType) + if (isValid) Some(f.get(null).asInstanceOf[Flag[_]]) else None + } catch { + case _: ClassNotFoundException | _: NoSuchFieldException | _: IllegalArgumentException => + None + } + + val className = if (!flagName.endsWith("$")) flagName + "$" else flagName + tryModuleField(className).orElse { // fallback for GlobalFlags declared in Java - tryMethod(f + "$", "globalFlagInstance") + tryMethod(className, "globalFlagInstance") } } @@ -173,7 +180,7 @@ object GlobalFlag { try { val cls: Class[_] = Class.forName(info.className, false, loader) if (cls.isAnnotationPresent(markerClass)) { - get(info.className.dropRight(1)) match { + get(info.className) match { case Some(f) => flags += f case None => println("failed for " + info.className) } diff --git a/util-app/src/main/scala/com/twitter/app/JavaGlobalFlag.scala b/util-app/src/main/scala/com/twitter/app/JavaGlobalFlag.scala index 97670c2d42..28b89102d8 100644 --- a/util-app/src/main/scala/com/twitter/app/JavaGlobalFlag.scala +++ b/util-app/src/main/scala/com/twitter/app/JavaGlobalFlag.scala @@ -1,21 +1,19 @@ package com.twitter.app /** - * @define java_declaration - * * In order to declare a new [[GlobalFlag]] in Java, it can be done with a bit * of ceremony by following a few steps: * - * 1. the flag's class name must end with "$", e.g. `javaGlobalWithDefault$`. - * This is done to mimic the behavior of Scala's singleton `object`. - * 1. the class must have a method, `public static Flag globalFlagInstance()`, - * that returns the singleton instance of the [[Flag]]. - * 1. the class should have a `static final` field holding the singleton - * instance of the flag. - * 1. the class should extend `JavaGlobalFlag`. - * 1. to avoid extraneous creation of instances, thus forcing users to only - * access the singleton, the constructor should be `private` and the class - * should be `public final`. + * 1. the flag's class name must end with "$", e.g. `javaGlobalWithDefault$`. + * This is done to mimic the behavior of Scala's singleton `object`. + * 1. the class must have a method, `public static Flag globalFlagInstance()`, + * that returns the singleton instance of the [[Flag]]. + * 1. the class should have a `static final` field holding the singleton + * instance of the flag. + * 1. the class should extend `JavaGlobalFlag`. + * 1. to avoid extraneous creation of instances, thus forcing users to only + * access the singleton, the constructor should be `private` and the class + * should be `public final`. * * An example of a [[GlobalFlag]] declared in Java: * {{{ diff --git a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala index 101a3f78f9..e76e7ad8b5 100644 --- a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala @@ -1,5 +1,6 @@ package com.twitter.app +import java.net.URLClassLoader import org.scalatest.FunSuite object MyGlobalFlag extends GlobalFlag[String]("a test flag", "a global test flag") @@ -63,4 +64,22 @@ class GlobalFlagTest extends FunSuite { } } + test("GlobalFlag.getAll") { + val mockClassLoader = new MockClassLoader(getClass.getClassLoader.asInstanceOf[URLClassLoader]) + val flags = GlobalFlag.getAll(mockClassLoader) + assert(flags.length == 4) + assert(flags.exists(_.help.equals("a package object test flag"))) + } + + private class MockClassLoader(realClassLoader: URLClassLoader) + extends URLClassLoader(realClassLoader.getURLs) { + private val isValidClassName = (className: String) => + List(MyGlobalFlag, MyGlobalBooleanFlag, MyGlobalFlagNoDefault, PackageObjectTest) + .map(_.getClass.getName) + .contains(className) + + override def loadClass(name: String): Class[_] = + if (isValidClassName(name)) realClassLoader.loadClass(name) else null + } + } diff --git a/util-app/src/test/scala/com/twitter/app/package.scala b/util-app/src/test/scala/com/twitter/app/package.scala new file mode 100644 index 0000000000..d25cfd23dc --- /dev/null +++ b/util-app/src/test/scala/com/twitter/app/package.scala @@ -0,0 +1,5 @@ +package com.twitter + +package object app { + object PackageObjectTest extends GlobalFlag[Boolean](true, "a package object test flag") +} From 36875a72adff0edc32a742c741231a9b6968262b Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 12 Apr 2019 17:51:58 +0000 Subject: [PATCH 141/796] util: Use HTTPS URLs Problem/Solution Update to use `https://` where possible. JIRA Issues: OSS-3912 Differential Revision: https://phabricator.twitter.biz/D299929 --- CHANGELOG.rst | 2 +- CONFIG.ini | 2 -- CONTRIBUTING.md | 12 ++++++------ README.md | 2 +- build.sbt | 4 ++-- doc/src/sphinx/_templates/sidebarintro.html | 4 ++-- doc/src/sphinx/util-cookbook/futures.rst | 2 +- doc/src/sphinx/util-stats/user_guide.rst | 2 +- site/index.html | 4 ++-- .../src/main/scala/com/twitter/app/App.scala | 2 +- .../main/scala/com/twitter/app/Flaggable.scala | 2 +- .../com/twitter/concurrent/OfferBenchmark.scala | 2 +- .../com/twitter/util/StringEncoderTest.scala | 2 +- .../scala/com/twitter/concurrent/Scheduler.scala | 2 +- .../com/twitter/concurrent/AsyncStream.scala | 4 ++-- .../src/main/scala/com/twitter/io/TempFile.scala | 6 +++--- .../main/scala/com/twitter/io/TempFolder.scala | 2 +- .../main/scala/com/twitter/util/Awaitable.scala | 2 +- .../src/main/scala/com/twitter/util/Config.scala | 2 +- .../scala/com/twitter/util/Credentials.scala | 2 +- .../src/main/scala/com/twitter/util/Future.scala | 2 +- .../src/main/scala/com/twitter/util/Signal.scala | 2 +- .../scala/com/twitter/util/StorageUnit.scala | 2 +- .../src/main/scala/com/twitter/util/Time.scala | 2 +- .../scala/com/twitter/util/CredentialsTest.scala | 2 +- .../scala/com/twitter/util/DurationTest.scala | 2 +- .../com/twitter/util/StringConversionsTest.scala | 2 +- .../scala/com/twitter/hashing/Hashable.scala | 12 ++++++------ .../com/twitter/jvm/ContentionSnapshot.scala | 2 +- .../main/scala/com/twitter/jvm/CpuProfile.scala | 4 ++-- .../main/scala/com/twitter/jvm/Estimator.scala | 2 +- .../main/scala/com/twitter/jvm/Heapster.scala | 2 +- .../scala/com/twitter/logging/FileHandler.scala | 2 +- .../scala/com/twitter/logging/Formatter.scala | 2 +- .../main/scala/com/twitter/logging/Handler.scala | 2 +- .../com/twitter/logging/LazyLogRecord.scala | 2 +- .../scala/com/twitter/logging/LogRecord.scala | 2 +- .../main/scala/com/twitter/logging/Logger.scala | 2 +- .../com/twitter/logging/LoggerFactory.scala | 2 +- .../com/twitter/logging/QueueingHandler.scala | 2 +- .../com/twitter/logging/ScribeHandler.scala | 2 +- .../com/twitter/logging/SyslogHandler.scala | 2 +- .../com/twitter/logging/ThrottledHandler.scala | 2 +- .../twitter/logging/config/LoggerConfig.scala | 2 +- .../com/twitter/logging/config/package.scala | 2 +- .../main/scala/com/twitter/logging/package.scala | 2 +- .../com/twitter/logging/FileHandlerTest.scala | 2 +- .../com/twitter/logging/FormatterTest.scala | 2 +- .../scala/com/twitter/logging/LoggerTest.scala | 2 +- .../scala/com/twitter/logging/LoggingTest.scala | 2 +- .../twitter/logging/QueueingHandlerTest.scala | 2 +- .../com/twitter/logging/ScribeHandlerTest.scala | 2 +- .../com/twitter/logging/SyslogHandlerTest.scala | 2 +- .../twitter/logging/ThrottledHandlerTest.scala | 2 +- util-slf4j-api/README.md | 2 +- .../scala/com/twitter/util/logging/Logger.scala | 4 ++-- .../util/logging/Slf4jBridgeUtility.scala | 16 ++++++++-------- .../twitter/finagle/stats/StatsReceiver.scala | 4 ++-- .../scala/com/twitter/logging/TestLogging.scala | 2 +- .../com/twitter/util/ThriftSerializerTest.scala | 2 +- 60 files changed, 85 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 43e2cf078e..9e63fd1cb1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,5 @@ .. Author notes: this file is formatted with restructured text - (http://docutils.sourceforge.net/docs/user/rst/quickstart.html). + (https://docutils.sourceforge.net/docs/user/rst/quickstart.html). The changelog style is adapted from Apache Lucene. Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in commits. diff --git a/CONFIG.ini b/CONFIG.ini index d9780e87f6..0b96e72684 100644 --- a/CONFIG.ini +++ b/CONFIG.ini @@ -1,5 +1,3 @@ -; See http://go/CONFIG.ini - ; owned by Core System Libraries [jira] project: CSL diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed1abc0822..b9a3783de7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -172,13 +172,13 @@ Please let us know if you have any questions about this process! [0]: https://github.com/twitter/util/pull/109 [1]: https://github.com/twitter/util/blob/master/util-core/src/test/java/com/twitter/util/VarCompilationTest.java -[scaladoc]: http://docs.scala-lang.org/style/scaladoc.html +[scaladoc]: https://docs.scala-lang.org/style/scaladoc.html [es]: https://twitter.github.io/effectivescala/ -[sd]: http://docs.scala-lang.org/style/scaladoc.html -[funsuite]: http://www.scalatest.org/getting_started_with_fun_suite -[scalatest]: http://www.scalatest.org/ -[ssg]: http://docs.scala-lang.org/style/scaladoc.html +[sd]: https://docs.scala-lang.org/style/scaladoc.html +[funsuite]: https://www.scalatest.org/getting_started_with_fun_suite +[scalatest]: https://www.scalatest.org/ +[ssg]: https://docs.scala-lang.org/style/scaladoc.html [travis-ci]: https://travis-ci.org/twitter/util [changes]: https://github.com/twitter/util/blob/develop/CHANGELOG.rst [scalacheck]: https://www.scalacheck.org/ -[gendrivenprop]: http://www.scalatest.org/user_guide/generator_driven_property_checks +[gendrivenprop]: https://www.scalatest.org/user_guide/generator_driven_property_checks diff --git a/README.md b/README.md index f95ddbdfdd..2d094f8c73 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ and is being actively developed and maintained. ## Releases [Releases](https://maven-badges.herokuapp.com/maven-central/com.twitter/util_2.12) -are done on an approximately monthly schedule. While [semver](http://semver.org/) +are done on an approximately monthly schedule. While [semver](https://semver.org/) is not followed, the [changelogs](CHANGELOG.rst) are detailed and include sections on public API breaks and changes in runtime behavior. diff --git a/build.sbt b/build.sbt index 9cec806ca5..61c4e2b9e4 100644 --- a/build.sbt +++ b/build.sbt @@ -31,7 +31,7 @@ val baseSettings = Seq( // Workaround for a scaladoc bug which causes it to choke on empty classpaths. unmanagedClasspath in Compile += Attributed.blank(new java.io.File("doesnotexist")), libraryDependencies ++= Seq( - // See http://www.scala-sbt.org/0.13/docs/Testing.html#JUnit + // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.10.19" % "test", "org.scalatest" %% "scalatest" % "3.0.0" % "test" @@ -73,7 +73,7 @@ val baseSettings = Seq( Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 diff --git a/doc/src/sphinx/_templates/sidebarintro.html b/doc/src/sphinx/_templates/sidebarintro.html index aa6e686cbb..586120c874 100644 --- a/doc/src/sphinx/_templates/sidebarintro.html +++ b/doc/src/sphinx/_templates/sidebarintro.html @@ -5,6 +5,6 @@

Useful Links

 diff --git a/doc/src/sphinx/util-cookbook/futures.rst b/doc/src/sphinx/util-cookbook/futures.rst index a46f43a334..6b07f206bf 100644 --- a/doc/src/sphinx/util-cookbook/futures.rst +++ b/doc/src/sphinx/util-cookbook/futures.rst @@ -11,7 +11,7 @@ Conversions between Twitter's Future and Scala's Future Twitter's ``com.twitter.util.Future`` is similar to, but `predates `_, -`Scala's `_ +`Scala's `_ ``scala.concurrent.Future`` and as such they are not directly compatible (e.g. support for continuation-local variables and interruptibility). diff --git a/doc/src/sphinx/util-stats/user_guide.rst b/doc/src/sphinx/util-stats/user_guide.rst index c53bd5d12c..37deb280fb 100644 --- a/doc/src/sphinx/util-stats/user_guide.rst +++ b/doc/src/sphinx/util-stats/user_guide.rst @@ -164,7 +164,7 @@ Via TwitterServer/finagle-stats — the `HTTP admin interface`_ responds with js .. _Futures: https://twitter.github.io/finagle/guide/Futures.html .. _Stat.time: https://github.com/twitter/util/blob/develop/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala .. _Stat.timeFuture: https://github.com/twitter/util/blob/develop/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala -.. _java.lang.ref.WeakReference: http://docs.oracle.com/javase/8/docs/api/java/lang/ref/WeakReference.html +.. _java.lang.ref.WeakReference: https://docs.oracle.com/javase/8/docs/api/java/lang/ref/WeakReference.html .. _InMemoryStatsReceiver: https://github.com/twitter/util/blob/master/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala .. _HTTP admin interface: https://twitter.github.io/twitter-server/Features.html#http-admin-interface .. _DenylistStatsReceiver: https://github.com/twitter/util/blob/develop/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala diff --git a/site/index.html b/site/index.html index cd4e154e43..bc308a8316 100644 --- a/site/index.html +++ b/site/index.html @@ -69,7 +69,7 @@

Util

  • User’s guides
  • API documentation
  • Finaglers Gitter channel
    • -
  • Finaglers Google Group
    • +
  • Finaglers Google Group

    • Contributing

    @@ -85,7 +85,7 @@

    Contributing

    Before endeavoring on large changes, please discuss them with the - Finagle Google Group + Finagle Google Group to receive feedback and suggestions.

    For all patches, please review our diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index b560b9c359..b2ec4dcac5 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -43,7 +43,7 @@ trait App extends Closable with CloseAwaitably { /** The [[com.twitter.app.Flags]] instance associated with this application */ //failfastOnFlagsNotParsed is called in the ctor of App.scala here which is a bad idea - //as things like this can happen http://stackoverflow.com/questions/18138397/calling-method-from-constructor + //as things like this can happen https://stackoverflow.com/questions/18138397/calling-method-from-constructor val flag: Flags = new Flags(name, includeGlobal = true, failfastOnFlagsNotParsed) private var _args = Array[String]() diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index 1b8ab25833..9275b10cc9 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -48,7 +48,7 @@ import scala.collection.JavaConverters._ * } * }}} * - * [1] http://en.wikipedia.org/wiki/Type_class + * [1] https://en.wikipedia.org/wiki/Type_class */ abstract class Flaggable[T] { diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala index cc0c7eb6e6..8fcd1871bf 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala @@ -42,7 +42,7 @@ object OfferBenchmark { // N.B. Traditionally you'd put this type of setup inside of a benchmark setup method per invocation // but because the runtime of each invocation is so small, the overhead is high JMH advises against // using Invocation setup methods: - // http://hg.openjdk.java.net/code-tools/jmh/file/bdfc7d3a6ebf/jmh-core/src/main/java/org/openjdk/jmh/annotations/Level.java#l50 + // https://hg.openjdk.java.net/code-tools/jmh/file/bdfc7d3a6ebf/jmh-core/src/main/java/org/openjdk/jmh/annotations/Level.java#l50 def toChooseFrom(): Seq[Offer[Int]] = { val ps = new ArrayBuffer[Promise[Tx[Int]]](numToChooseFrom) val ofs = new ArrayBuffer[Offer[Int]](numToChooseFrom) diff --git a/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala b/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala index 89f8c8b415..5520a5f90c 100644 --- a/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala +++ b/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala @@ -7,7 +7,7 @@ package com.twitter.util * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala index 1b3f716d9f..38d7b4cbf9 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala @@ -28,7 +28,7 @@ trait Scheduler { // // On OSX, the Mach thread_info call is used. // - // [1] http://linux.die.net/man/3/clock_gettime + // [1] https://linux.die.net/man/3/clock_gettime // // Note, these clock stats were decommissioned as they only make // sense relative to `wallTime` and the tracking error we have diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 04b68ddaa0..3a112597a8 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -492,7 +492,7 @@ sealed abstract class AsyncStream[+A] { * Buffer the specified number of items from the stream, or all * remaining items if the end of the stream is reached before finding * that many items. In all cases, this method should act like - * + * * and not cause evaluation of the remainder of the stream. */ private[concurrent] def buffer(n: Int): Future[(Seq[A], () => AsyncStream[A])] = { @@ -531,7 +531,7 @@ sealed abstract class AsyncStream[+A] { * Convert the stream into a stream of groups of items. This * facilitates batch processing of the items in the stream. In all * cases, this method should act like - * + * * The resulting stream will cause this original stream to be * evaluated group-wise, so calling this method will cause the first * `groupSize` cells to be evaluated (even without examining the diff --git a/util-core/src/main/scala/com/twitter/io/TempFile.scala b/util-core/src/main/scala/com/twitter/io/TempFile.scala index 0bc6908f76..b206261837 100644 --- a/util-core/src/main/scala/com/twitter/io/TempFile.scala +++ b/util-core/src/main/scala/com/twitter/io/TempFile.scala @@ -10,7 +10,7 @@ object TempFile { * * Note, due to the usage of `File.deleteOnExit()` callers should * be careful using this as it can leak memory. - * See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4513817 for + * See https://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4513817 for * example. * * @param path the resource-relative path to make a temp file from @@ -24,7 +24,7 @@ object TempFile { * * Note, due to the usage of `File.deleteOnExit()` callers should * be careful using this as it can leak memory. - * See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4513817 for + * See https://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4513817 for * example. * * @param klass the `Class` to use for getting the resource. @@ -41,7 +41,7 @@ object TempFile { * * Note, due to the usage of `File.deleteOnExit()` callers should * be careful using this as it can leak memory. - * See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4513817 for + * See https://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4513817 for * example. * * @param path the path to make a temp file from diff --git a/util-core/src/main/scala/com/twitter/io/TempFolder.scala b/util-core/src/main/scala/com/twitter/io/TempFolder.scala index 56aecde891..d45067d14d 100644 --- a/util-core/src/main/scala/com/twitter/io/TempFolder.scala +++ b/util-core/src/main/scala/com/twitter/io/TempFolder.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-core/src/main/scala/com/twitter/util/Awaitable.scala b/util-core/src/main/scala/com/twitter/util/Awaitable.scala index c6deb99642..48a5dffb85 100644 --- a/util-core/src/main/scala/com/twitter/util/Awaitable.scala +++ b/util-core/src/main/scala/com/twitter/util/Awaitable.scala @@ -186,7 +186,7 @@ object Await { all(awaitables.asScala.toSeq, timeout) } -// See http://stackoverflow.com/questions/26643045/java-interoperability-woes-with-scala-generics-and-boxing +// See https://stackoverflow.com/questions/26643045/java-interoperability-woes-with-scala-generics-and-boxing private[util] trait CloseAwaitably0[U <: Unit] extends Awaitable[U] { private[this] val onClose: Promise[U] = new Promise[U] private[this] val closed: AtomicBoolean = new AtomicBoolean(false) diff --git a/util-core/src/main/scala/com/twitter/util/Config.scala b/util-core/src/main/scala/com/twitter/util/Config.scala index 911cc548bc..e9d2a888bb 100644 --- a/util-core/src/main/scala/com/twitter/util/Config.scala +++ b/util-core/src/main/scala/com/twitter/util/Config.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-core/src/main/scala/com/twitter/util/Credentials.scala b/util-core/src/main/scala/com/twitter/util/Credentials.scala index 437a5c7c85..b780076f3c 100644 --- a/util-core/src/main/scala/com/twitter/util/Credentials.scala +++ b/util-core/src/main/scala/com/twitter/util/Credentials.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 2713638a17..0ac48515f6 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -2143,7 +2143,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * match the semantics of a Java Future as closely as possible to * avoid issues with the way another API might use them. See: * - * http://download.oracle.com/javase/6/docs/api/java/util/concurrent/Future.html#cancel(boolean) + * https://download.oracle.com/javase/6/docs/api/java/util/concurrent/Future.html#cancel(boolean) */ def toJavaFuture: JavaFuture[_ <: A] = { new JavaFuture[A] { diff --git a/util-core/src/main/scala/com/twitter/util/Signal.scala b/util-core/src/main/scala/com/twitter/util/Signal.scala index a0657df184..96de468b46 100644 --- a/util-core/src/main/scala/com/twitter/util/Signal.scala +++ b/util-core/src/main/scala/com/twitter/util/Signal.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala index 668643e0d9..58f51b18e9 100644 --- a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala +++ b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index bb2b54bb63..f4f89420ee 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala b/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala index 35f95e3c70..5149232002 100644 --- a/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index fa35290017..3e0f299a04 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala b/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala index 800e696e5f..09f90c6776 100644 --- a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala b/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala index 24e6184618..1eada3cdfc 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala @@ -47,7 +47,7 @@ object Hashable extends LowPriorityHashable { /** * FNV fast hashing algorithm in 32 bits. - * @see http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash + * @see https://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash */ val FNV1_32: Hashable[Array[Byte], Int] = new Hashable[Array[Byte], Int] { def apply(key: Array[Byte]) = { @@ -67,7 +67,7 @@ object Hashable extends LowPriorityHashable { /** * FNV fast hashing algorithm in 32 bits, variant with operations reversed. - * @see http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash + * @see https://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash */ val FNV1A_32: Hashable[Array[Byte], Int] = new Hashable[Array[Byte], Int] { def apply(key: Array[Byte]): Int = { @@ -87,7 +87,7 @@ object Hashable extends LowPriorityHashable { /** * FNV fast hashing algorithm in 64 bits. - * @see http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash + * @see https://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash */ val FNV1_64: Hashable[Array[Byte], Long] = new Hashable[Array[Byte], Long] { def apply(key: Array[Byte]): Long = { @@ -107,7 +107,7 @@ object Hashable extends LowPriorityHashable { /** * FNV fast hashing algorithm in 64 bits, variant with operations reversed. - * @see http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash + * @see https://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash */ val FNV1A_64: Hashable[Array[Byte], Long] = new Hashable[Array[Byte], Long] { def apply(key: Array[Byte]): Long = { @@ -179,7 +179,7 @@ object Hashable extends LowPriorityHashable { /** * Paul Hsieh's hash function. - * http://www.azillionmonkeys.com/qed/hash.html + * https://www.azillionmonkeys.com/qed/hash.html */ val HSIEH: Hashable[Array[Byte], Int] = new Hashable[Array[Byte], Int] { def apply(key: Array[Byte]): Int = { @@ -244,7 +244,7 @@ object Hashable extends LowPriorityHashable { /** * Jenkins Hash Function - * http://en.wikipedia.org/wiki/Jenkins_hash_function + * https://en.wikipedia.org/wiki/Jenkins_hash_function */ val JENKINS: Hashable[Array[Byte], Long] = new Hashable[Array[Byte], Long] { def apply(key: Array[Byte]): Long = { diff --git a/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala b/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala index fe2de5980a..5f1095876b 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala @@ -5,7 +5,7 @@ import java.lang.management.{ManagementFactory, ThreadInfo} /** * A thread contention summary providing a brief overview of threads - * that are [[http://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#BLOCKED BLOCKED]], [[http://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#WAITING WAITING]], or [[http://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#TIMED_WAITING TIMED_WAITING]] + * that are [[https://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#BLOCKED BLOCKED]], [[https://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#WAITING WAITING]], or [[https://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#TIMED_WAITING TIMED_WAITING]] * * While this could be an object, we use instantiation as a signal of * intent and enable contention monitoring. diff --git a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala index de5733f276..f06d944f7c 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala @@ -24,7 +24,7 @@ case class CpuProfile( * Write a Google pprof-compatible profile to `out`. The format is * documented here: * - * http://google-perftools.googlecode.com/svn/trunk/doc/cpuprofile-fileformat.html + * https://google-perftools.googlecode.com/svn/trunk/doc/cpuprofile-fileformat.html */ def writeGoogleProfile(out: OutputStream): Unit = { var next = 1 @@ -80,7 +80,7 @@ object CpuProfile { * When looking for RUNNABLEs, the JVM's notion of runnable differs from the * from kernel's definition and for some well known cases, we can filter * out threads that are actually asleep. - * See http://www.brendangregg.com/blog/2014-06-09/java-cpu-sampling-using-hprof.html + * See https://www.brendangregg.com/blog/2014-06-09/java-cpu-sampling-using-hprof.html */ private[jvm] def isRunnable(stackElem: StackTraceElement): Boolean = !IdleClassAndMethod.contains((stackElem.getClassName, stackElem.getMethodName)) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala b/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala index 2e64022f67..de10956e17 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala @@ -134,7 +134,7 @@ class WindowedMeans(N: Int, windows: Seq[(Int, Int)]) extends Estimator[Double] * Unix-like load average, an exponentially weighted moving average, * smoothed to the given interval (counted in number of * measurements). - * See: http://web.mit.edu/saltzer/www/publications/instrumentation.html + * See: https://web.mit.edu/saltzer/www/publications/instrumentation.html */ class LoadAverage(interval: Double) extends Estimator[Double] { private[this] val a = math.exp(-1D / interval) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala b/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala index ebca1f7fa7..32b7dcca00 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala index 4ce7ce7f2e..fb52e411d7 100644 --- a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala index 36009f07ae..432e7a7ceb 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/Handler.scala b/util-logging/src/main/scala/com/twitter/logging/Handler.scala index e92f5f384f..939a6b8799 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Handler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Handler.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala b/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala index 7f55f2e1b1..63b2d26135 100644 --- a/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala +++ b/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/LogRecord.scala b/util-logging/src/main/scala/com/twitter/logging/LogRecord.scala index 935e0c2bc3..0f25e890d4 100644 --- a/util-logging/src/main/scala/com/twitter/logging/LogRecord.scala +++ b/util-logging/src/main/scala/com/twitter/logging/LogRecord.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/Logger.scala b/util-logging/src/main/scala/com/twitter/logging/Logger.scala index 1916e586e7..7d310cd7e6 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Logger.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Logger.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala b/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala index ca423a77e4..0d95a7747c 100644 --- a/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala +++ b/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala b/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala index 1035c1a05f..e617f71a97 100644 --- a/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala index 26e8e204b1..59669fedf0 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala b/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala index eeed3266fa..7f97478949 100644 --- a/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala index f194ace493..f649545779 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala b/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala index 3f646e5343..494b592335 100644 --- a/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala +++ b/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/config/package.scala b/util-logging/src/main/scala/com/twitter/logging/config/package.scala index 0d14da5e81..a2117d664f 100644 --- a/util-logging/src/main/scala/com/twitter/logging/config/package.scala +++ b/util-logging/src/main/scala/com/twitter/logging/config/package.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/main/scala/com/twitter/logging/package.scala b/util-logging/src/main/scala/com/twitter/logging/package.scala index a9df0daab2..80d214e09b 100644 --- a/util-logging/src/main/scala/com/twitter/logging/package.scala +++ b/util-logging/src/main/scala/com/twitter/logging/package.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index c64322a85c..36aa95554c 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala index 617a4f055d..be51e09564 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index 95c582c17b..a8fd342c0c 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggingTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggingTest.scala index 394e034641..d020deb837 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggingTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggingTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala index f4e8ae86ac..7a1597ad26 100644 --- a/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala index 92ac5957b6..34174ecfbd 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala index 920379c553..992967e773 100644 --- a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala index c3573375af..18d09eea38 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-slf4j-api/README.md b/util-slf4j-api/README.md index 543c104995..1e45827a87 100644 --- a/util-slf4j-api/README.md +++ b/util-slf4j-api/README.md @@ -4,7 +4,7 @@ This library provides a Scala-friendly logging wrapper around the [slf4j-api](ht ## Differences with `util-logging` -The code in `util-logging` is a scala wrapper over `java.util.logging` (JUL) which is a logging API **and** implementation. `util-slf4j-api` is an API-only and adds a thin scala wrapper over the [slf4j-api](http://www.slf4j.org/) which allows +The code in `util-logging` is a scala wrapper over `java.util.logging` (JUL) which is a logging API **and** implementation. `util-slf4j-api` is an API-only and adds a thin scala wrapper over the [slf4j-api](https://www.slf4j.org/) which allows users to choose their logging implementation. It is recommended that users move to using `util-slf4j-api` over `util-logging`. Since the [slf4j-api](https://www.slf4j.org/manual.html) is only an interface it requires an actual logging implementation. You should ensure that you do not end-up with *multiple* logging implementations on your classpath, e.g., you should not have multiple SLF4J bindings (`slf4j-nop`, `slf4j-log4j12`, `slf4j-jdk14`, etc.) and/or a `java.util.logging` implementation, etc. on your classpath as these are all competing implementations and since classpath order is non-deterministic, can lead to unexpected logging behavior. diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala index f78ed8c8c4..fa783f7231 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala @@ -91,13 +91,13 @@ object Logger { * @define logWith * * Log the given parameterized message at the named log level using the given - * args. See [[http://www.slf4j.org/faq.html#logging_performance Parameterized Message Logging]] + * args. See [[https://www.slf4j.org/faq.html#logging_performance Parameterized Message Logging]] * * @define logWithMarker * * Log the given parameterized message at the named log level using the given * args taking into consideration the given [[Marker]] data. - * See [[http://www.slf4j.org/faq.html#logging_performance Parameterized Message Logging]] + * See [[https://www.slf4j.org/faq.html#logging_performance Parameterized Message Logging]] */ @SerialVersionUID(1L) final class Logger private (underlying: slf4j.Logger) extends Serializable { diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala index d6968d39f7..aa846e736e 100644 --- a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala @@ -5,25 +5,25 @@ import org.slf4j.LoggerFactory import org.slf4j.bridge.SLF4JBridgeHandler /** - * A utility to safely install the [[http://www.slf4j.org/ slf4j-api]] [[http://www.slf4j.org/api/org/slf4j/bridge/SLF4JBridgeHandler.html SLF4JBridgeHandler]]. + * A utility to safely install the [[https://www.slf4j.org/ slf4j-api]] [[https://www.slf4j.org/api/org/slf4j/bridge/SLF4JBridgeHandler.html SLF4JBridgeHandler]]. * The utility attempts to detect if the `slf4j-jdk14` dependency is present on * the classpath as it is unwise to install the `jul-to-slf4j` bridge with the `slf4j-jdk14` - * dependency present as it may cause an [[http://www.slf4j.org/legacy.html#julRecursion infinite loop]]. + * dependency present as it may cause an [[https://www.slf4j.org/legacy.html#julRecursion infinite loop]]. * - * If the [[http://www.slf4j.org/api/org/slf4j/bridge/SLF4JBridgeHandler.html SLF4JBridgeHandler]] is already installed we do not try to install + * If the [[https://www.slf4j.org/api/org/slf4j/bridge/SLF4JBridgeHandler.html SLF4JBridgeHandler]] is already installed we do not try to install * it again. * * Additionally note there is a performance impact for bridging jul log messages in this - * manner. However, using the [[http://logback.qos.ch/manual/configuration.html#LevelChangePropagator Logback LevelChangePropagator]] + * manner. However, using the [[https://logback.qos.ch/manual/configuration.html#LevelChangePropagator Logback LevelChangePropagator]] * (> 0.9.25) can eliminate this performance impact. * - * @see [[http://www.slf4j.org/legacy.html#jul-to-slf4j SLF4J: jul-to-slf4j bridge]] + * @see [[https://www.slf4j.org/legacy.html#jul-to-slf4j SLF4J: jul-to-slf4j bridge]] */ object Slf4jBridgeUtility extends Logging { /** - * Attempt installation of the [[http://www.slf4j.org/ slf4j-api]] [[http://www.slf4j.org/api/org/slf4j/bridge/SLF4JBridgeHandler.html SLF4JBridgeHandler]]. - * The [[http://www.slf4j.org/api/org/slf4j/bridge/SLF4JBridgeHandler.html SLF4JBridgeHandler]] will not be installed if already installed or if the + * Attempt installation of the [[https://www.slf4j.org/ slf4j-api]] [[https://www.slf4j.org/api/org/slf4j/bridge/SLF4JBridgeHandler.html SLF4JBridgeHandler]]. + * The [[https://www.slf4j.org/api/org/slf4j/bridge/SLF4JBridgeHandler.html SLF4JBridgeHandler]] will not be installed if already installed or if the * `slf4j-jdk14` dependency is detected on the classpath. */ def attemptSlf4jBridgeHandlerInstallation(): Unit = install() @@ -49,7 +49,7 @@ object Slf4jBridgeUtility extends Logging { .getLogger(this.getClass) .warn( "Detected [org.slf4j.impl.JDK14LoggerFactory] on classpath. " + - "SLF4JBridgeHandler will not be installed, see: http://www.slf4j.org/legacy.html#julRecursion" + "SLF4JBridgeHandler will not be installed, see: https://www.slf4j.org/legacy.html#julRecursion" ) false } catch { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 02429f090f..906a221ac3 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -172,7 +172,7 @@ trait StatsReceiver { * @see [[StatsReceiver.provideGauge]] when there is not a good location * to store the returned [[Gauge gauge]] that can give the desired lifecycle. * - * @see [[http://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] + * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ def addGauge(name: String*)(f: => Float): Gauge = addGauge(Verbosity.Default, name: _*)(f) @@ -191,7 +191,7 @@ trait StatsReceiver { * @see [[StatsReceiver.provideGauge]] when there is not a good location * to store the returned [[Gauge gauge]] that can give the desired lifecycle. * - * @see [[http://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] + * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge diff --git a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala index b877e8e66e..e01d82d33a 100644 --- a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala +++ b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala b/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala index e5d50ec307..136a845cf6 100644 --- a/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala +++ b/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala @@ -5,7 +5,7 @@ * not use this file except in compliance with the License. You may obtain * a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From 78a03102f502eb3aa9d1483994f2b6b10193bda9 Mon Sep 17 00:00:00 2001 From: Soojee Chung Date: Wed, 17 Apr 2019 17:18:01 +0000 Subject: [PATCH 142/796] doc: Remove unused parameter in Future conversion example Problem The asScala method in the first example of the Futures documentation passes in an unused parameter, implicit e: ExecutionContext. Solution Removed unused parameter. Github issue #246 Signed-off-by: Yufan Gong Differential Revision: https://phabricator.twitter.biz/D301848 --- doc/src/sphinx/util-cookbook/futures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sphinx/util-cookbook/futures.rst b/doc/src/sphinx/util-cookbook/futures.rst index 6b07f206bf..8f1e394acb 100644 --- a/doc/src/sphinx/util-cookbook/futures.rst +++ b/doc/src/sphinx/util-cookbook/futures.rst @@ -29,7 +29,7 @@ Scala: /** Convert from a Twitter Future to a Scala Future */ implicit class RichTwitterFuture[A](val tf: TwitterFuture[A]) extends AnyVal { - def asScala(implicit e: ExecutionContext): ScalaFuture[A] = { + def asScala: ScalaFuture[A] = { val promise: ScalaPromise[A] = ScalaPromise() tf.respond { case Return(value) => promise.success(value) From 885707e4271dfb9a4b70554fb4eee9b4f13cfdae Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Wed, 17 Apr 2019 23:10:08 +0000 Subject: [PATCH 143/796] Twitter-oss: Prepare OSS libraries for release 19.4.0 Problem We want to release the next versions of our Twitter OSS libraries 19.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D302314 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9e63fd1cb1..744fa97fd9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.4.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 2d094f8c73..4662cdf434 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.3.0" +val collUtils = "com.twitter" %% "util-collection" % "19.4.0" ``` # Units diff --git a/build.sbt b/build.sbt index 61c4e2b9e4..aa86ecb2b6 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.4.0-SNAPSHOT" +val releaseVersion = "19.4.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 20a96aa4872d9220ae9cdbb8b0726c87c171eaf4 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Fri, 19 Apr 2019 16:17:57 +0000 Subject: [PATCH 144/796] Twitter-oss: update OSS libraries post-release to 19.5.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D302995 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4662cdf434..88f7a0dcd7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index aa86ecb2b6..d8b61d2634 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.4.0" +val releaseVersion = "19.5.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 4004556dfe6f94e718874dde3c3fe7ceed753475 Mon Sep 17 00:00:00 2001 From: Shane Delmore Date: Wed, 1 May 2019 10:20:42 +0000 Subject: [PATCH 145/796] Switch from Scala 2.11 to Scala 2.12 Differential Revision: https://phabricator.twitter.biz/D303705 --- .../scala/com/twitter/cache/caffeine/BUILD | 1 + .../TwitterFuturesCapturePoints.xml | 84 ------------------- 2 files changed, 1 insertion(+), 84 deletions(-) diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index c56d0d4896..eac4bf2f69 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -3,6 +3,7 @@ junit_tests( compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", + "3rdparty/jvm/com/google/code/findbugs:jsr305", "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", diff --git a/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml b/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml index 3100e4daa1..9fc9fb702c 100644 --- a/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml +++ b/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml @@ -51,88 +51,4 @@ insert-class-name="com.twitter.util.ConstFuture$$anon$2" insert-method-name="run" insert-key-expression="f" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - From e7762013853f830386be5da5190094058f0fb644 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 8 May 2019 21:55:14 +0000 Subject: [PATCH 146/796] all: Audit gitignore Contents Problem / Solution Our open source project gitignore files have accumulated quite a bit of cruft. Let's unify them, remove no longer necessary entries, and call out project specific items. JIRA Issues: CSL-7911 Differential Revision: https://phabricator.twitter.biz/D311962 --- .gitignore | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 42cfbcc9cc..59f053bcff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,16 @@ -target/ -dist/ -project/boot/ -project/plugins/project/ -project/plugins/src_managed/ -*.log -*.tmproj -lib_managed/ -*.swp +*.class *.iml -.idea/ -.idea/* +*.ipr +*.log .DS_Store -.ensime -.ivyjars +.cache +.history +.idea/ +.lib +dist/ +lib_managed/ out/ +project/boot/ +project/plugins/project/ sbt-launch.jar +target/ From 5732c58148c269d6a9de4ad6e2f4931ff824ba4e Mon Sep 17 00:00:00 2001 From: Jordan Bull Date: Tue, 14 May 2019 19:16:42 +0000 Subject: [PATCH 147/796] util-logging: Add histogram stats of batch size to scribe handler Problem The `c.t.logging.ScribeHandler` does not provide stats on the size of the batches sent to Scribe despite this being critical information for debugging issues in scribe servers. Solution Add histogram stats for the size of batches sent to Scribe servers (both number of events and total byte count). Result The size of batches sent to Scribe is visible. JIRA Issues: PUBSUB-35936 Differential Revision: https://phabricator.twitter.biz/D310874 --- .../com/twitter/logging/ScribeHandler.scala | 14 ++++-- .../twitter/logging/ScribeHandlerTest.scala | 45 ++++++++++++++++++- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala index 59669fedf0..4a9708f203 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala @@ -21,9 +21,8 @@ import java.net._ import java.nio.{ByteBuffer, ByteOrder} import java.nio.charset.StandardCharsets.ISO_8859_1 import java.util.concurrent.atomic.AtomicLong -import java.util.concurrent.{ArrayBlockingQueue, LinkedBlockingQueue, TimeUnit, ThreadPoolExecutor} +import java.util.concurrent.{ArrayBlockingQueue, LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit} import java.util.{Arrays, logging => javalog} - import com.twitter.concurrent.NamedPoolThreadFactory import com.twitter.conversions.StringOps._ import com.twitter.conversions.DurationOps._ @@ -262,7 +261,7 @@ class ScribeHandler( while (remaining > 0 && socket.isDefined) { val count = maxMessagesPerTransaction min remaining val buffer = makeBuffer(count) - var offset = 0 + stats.statBatchSize(count, buffer.capacity) try { outStream.write(buffer.array) @@ -319,7 +318,6 @@ class ScribeHandler( // should be private, make it visible to tests private[logging] def makeBuffer(count: Int): ByteBuffer = { val texts = for (i <- 0 until count) yield queue.poll() - val recordHeader = ByteBuffer.wrap(new Array[Byte](10 + category.length)) recordHeader.order(ByteOrder.BIG_ENDIAN) recordHeader.put(11: Byte) @@ -511,6 +509,9 @@ class ScribeHandler( val instances = statsReceiver.addGauge("instances") { 1 } val unsentQueue = statsReceiver.addGauge("unsent_queue") { queueSize } + val batchSizeBytesStat = statsReceiver.stat("batch_size_bytes") + val batchSizeCountStat = statsReceiver.stat("batch_size_messages") + def incrSentRecords(count: Int): Unit = { sentRecords.addAndGet(count) totalSentRecords.incr(count) @@ -537,6 +538,11 @@ class ScribeHandler( def incrCloses(): Unit = totalCloses.incr() + def statBatchSize(numMessages: Int, bytes: Int): Unit = { + batchSizeCountStat.add(numMessages) + batchSizeBytesStat.add(bytes) + } + def log(): Unit = { synchronized { val period = Time.now.since(_lastLogStats) diff --git a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala index 34174ecfbd..bb1c56f833 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala @@ -21,13 +21,14 @@ import com.twitter.conversions.StringOps._ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.InMemoryStatsReceiver import com.twitter.util.{RandomSocket, Time} +import java.io.ByteArrayOutputStream import java.net.Socket import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.{ ArrayBlockingQueue, RejectedExecutionHandler, - TimeUnit, - ThreadPoolExecutor + ThreadPoolExecutor, + TimeUnit } import java.util.{logging => javalog} import org.junit.runner.RunWith @@ -224,6 +225,46 @@ class ScribeHandlerTest extends WordSpec with BeforeAndAfter with Eventually wit assert(rejected.get() == 94) } + "report batch sizes" in { + val statsReceiver = new InMemoryStatsReceiver + val mockSocket = mock[Socket] + when(mockSocket.getOutputStream).thenReturn(new ByteArrayOutputStream()) + + val scribe = ScribeHandler( + port = portWithoutListener, + category = "test", + bufferTime = 5.seconds, + formatter = BareFormatter, + level = None, + statsReceiver = statsReceiver + ).apply() + + scribe.setSocket(Some(mockSocket)) + + // Get the buffer first to know the size before flushing + Time.withCurrentTimeFrozen { _ => + scribe.updateLastTransmission() + scribe.publish(record2) + scribe.publish(record1) + } + val buffer = scribe.makeBuffer(2) + + // Make sure multiple records get flushed together + Time.withCurrentTimeFrozen { _ => + scribe.updateLastTransmission() + scribe.publish(record2) + scribe.publish(record1) + } + + scribe.flush() + + scribe.flusher.shutdown() + scribe.flusher.awaitTermination(15, TimeUnit.SECONDS) + + assert(statsReceiver.stat("batch_size_bytes")() == Seq(buffer.capacity())) + assert(statsReceiver.stat("batch_size_messages")() == Seq(2)) + } + "handle batch write errors" in { val statsReceiver = new InMemoryStatsReceiver val mockSocket = mock[Socket] From 4875552e3bf5752a389b9af524831d6bb3507c57 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 15 May 2019 00:09:07 +0000 Subject: [PATCH 148/796] util-app|twitter-server: Track duplicated flags and add a lint rule Problem It is possible to register the same Flag name in a `c.t.app.App` multiple times. `c.t.app.App#flag` is an instance of `c.t.app.Flags` which stores Flags in a `Map[String, Flag[_]]` keyed by Flag name where the last registered name wins. The current logic prints a notice to System.err when a duplicated Flag name is registered, however, this can lead to unintended behavior for users with little visibility into what is happening. Solution Track the registration of Flags with the same name and add a lint rule to TwitterServer, `DuplicateFlagDefinitions` which will give visibility in the form of a lint rule violation when multiple Flags with the same name are registered in the app. Result Better visibility for users when multiple Flags with the same name are registered in the app. JIRA Issues: CSL-7829 Differential Revision: https://phabricator.twitter.biz/D314410 --- CHANGELOG.rst | 7 ++++++ .../main/scala/com/twitter/app/Flags.scala | 25 +++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 744fa97fd9..51df8aa501 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-app: Track the registration of duplicated Flag names. Currently, we print a warning to + `stderr` but do not track the duplicated Flag names. Tracking them allows us to inspect and + warn over the entire set. ``PHAB_ID=D314410`` + 19.4.0 ------ diff --git a/util-app/src/main/scala/com/twitter/app/Flags.scala b/util-app/src/main/scala/com/twitter/app/Flags.scala index c1db75327d..be748f74ba 100644 --- a/util-app/src/main/scala/com/twitter/app/Flags.scala +++ b/util-app/src/main/scala/com/twitter/app/Flags.scala @@ -1,5 +1,6 @@ package com.twitter.app +import scala.collection.JavaConverters._ import scala.collection.immutable.TreeSet import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -87,6 +88,8 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) // thread-safety is provided by synchronization on `this` private[this] val flags = new mutable.HashMap[String, Flag[_]] + // when a flag with the same name as a previously added flag is added, track it here + private[this] val duplicates = new java.util.concurrent.ConcurrentSkipListSet[String]() @volatile private[this] var cmdUsage = "" @@ -248,8 +251,10 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) * @param f A concrete Flag to add */ def add(f: Flag[_]): Unit = synchronized { - if (flags contains f.name) - System.err.printf("Flag %s already defined!\n", f.name) + if (flags.contains(f.name)) { + System.err.printf("Flag: \"%s\" already defined!\n", f.name) + duplicates.add(f.name) + } flags(f.name) = f.withFailFast(failFastUntilParsed) } @@ -286,7 +291,7 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) * @param help The help string of the flag. */ def create[T](name: String, default: T, help: String, flaggable: Flaggable[T]): Flag[T] = { - implicit val impl = flaggable + implicit val impl: Flaggable[T] = flaggable apply(name, default, help) } @@ -331,13 +336,9 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) cmd + argv0 + " [...]\n" + "flags:\n" + - (lines mkString "\n") + ( - if (globalLines.isEmpty) "" - else { - "\nglobal flags:\n" + - (globalLines mkString "\n") - } - ) + lines.mkString("\n") + + (if (globalLines.isEmpty) "" + else "\nglobal flags:\n" + globalLines.mkString("\n")) } /** @@ -401,4 +402,8 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) lines.mkString("\n") } + + /** Return the set of any registered duplicated flag names. */ + protected[twitter] def registeredDuplicates: Set[String] = + duplicates.asScala.toSet } From 8840055958ad9adcc176ff6d0e4914f1f61730eb Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 16 May 2019 17:10:26 +0000 Subject: [PATCH 149/796] Update to Jackson 2.9.8 Problem/Solution Update Jackson dependencies to version `2.9.8`. JIRA Issues: CSL-7986 Differential Revision: https://phabricator.twitter.biz/D304193 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index d8b61d2634..41e9b41534 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll( ExclusionRule("javax.jms", "jms") ) val slf4jVersion = "1.7.21" -val jacksonVersion = "2.9.6" +val jacksonVersion = "2.9.8" val guavaLib = "com.google.guava" % "guava" % "19.0" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.3.4" From a1f161f6d05dc8cca9f8b945e9f86ec5eaad4dc9 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 17 May 2019 18:23:01 +0000 Subject: [PATCH 150/796] csl: Release CSL libraries 19.5 It's a release day. Let's cut a May release. JIRA Issues: CSL-8130 Differential Revision: https://phabricator.twitter.biz/D315909 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 51df8aa501..b74f373cbe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.5.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 88f7a0dcd7..e107c1773a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.4.0" +val collUtils = "com.twitter" %% "util-collection" % "19.5.0" ``` # Units diff --git a/build.sbt b/build.sbt index 41e9b41534..f84e93621f 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.5.0-SNAPSHOT" +val releaseVersion = "19.5.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From f02affd94535991a957df48ceea471dca8570396 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 20 May 2019 18:29:17 +0000 Subject: [PATCH 151/796] csl: Post-release CSL libraries 19.6.0-SNAPSHOT The release is done. Move to the next version. JIRA Issues: CSL-8130 Differential Revision: https://phabricator.twitter.biz/D316720 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e107c1773a..1d6355d56a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index f84e93621f..a9f1d84ec1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.5.0" +val releaseVersion = "19.6.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From c6cdd230d2e9e108967ac49a1549a51b1f173235 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 21 May 2019 00:30:23 +0000 Subject: [PATCH 152/796] twitter-oss: Prepare OSS libraries for point release 19.5.1 Problem We want to release a point release version of our Twitter OSS libraries 19.5.1 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libaries for their point release. Differential Revision: https://phabricator.twitter.biz/D317491 --- CHANGELOG.rst | 5 +++++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b74f373cbe..f2055721c1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.5.1 +------ + +No Changes + 19.5.0 ------ diff --git a/README.md b/README.md index 1d6355d56a..f1c58c9ea6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.5.0" +val collUtils = "com.twitter" %% "util-collection" % "19.5.1" ``` # Units diff --git a/build.sbt b/build.sbt index a9f1d84ec1..2f92d05c86 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.6.0-SNAPSHOT" +val releaseVersion = "19.5.1" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 2278eabe12d3b780bf237bc6f2ff4752a19a91c0 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 21 May 2019 19:00:36 +0000 Subject: [PATCH 153/796] twitter-oss: Update OSS libraries post-release to 19.6.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.6.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D317825 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f1c58c9ea6..1b33c6f215 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 2f92d05c86..a9f1d84ec1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.5.1" +val releaseVersion = "19.6.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From f5aa144d459e74f0b4dedfa416d2ff87b9e60c74 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 28 May 2019 16:34:27 +0000 Subject: [PATCH 154/796] util: Add ryano to Top Level Owners Problem / Solution I am already an owner for a good portion of util, but have worked with, or helped develop some of the other code for which I am not. Time to add myself to the top level owners file. Differential Revision: https://phabricator.twitter.biz/D319794 --- OWNERS | 1 + util-benchmark/OWNERS | 1 + util-codec/OWNERS | 1 + util-intellij/OWNERS | 1 + util-logging/OWNERS | 1 + util-stats/OWNERS | 1 + util-zk/OWNERS | 1 + 7 files changed, 7 insertions(+) diff --git a/OWNERS b/OWNERS index a044bdad0f..1c2f0835fc 100644 --- a/OWNERS +++ b/OWNERS @@ -2,3 +2,4 @@ banderson koliver mnakamura roanta +ryano diff --git a/util-benchmark/OWNERS b/util-benchmark/OWNERS index a044bdad0f..1c2f0835fc 100644 --- a/util-benchmark/OWNERS +++ b/util-benchmark/OWNERS @@ -2,3 +2,4 @@ banderson koliver mnakamura roanta +ryano diff --git a/util-codec/OWNERS b/util-codec/OWNERS index a044bdad0f..1c2f0835fc 100644 --- a/util-codec/OWNERS +++ b/util-codec/OWNERS @@ -2,3 +2,4 @@ banderson koliver mnakamura roanta +ryano diff --git a/util-intellij/OWNERS b/util-intellij/OWNERS index a044bdad0f..1c2f0835fc 100644 --- a/util-intellij/OWNERS +++ b/util-intellij/OWNERS @@ -2,3 +2,4 @@ banderson koliver mnakamura roanta +ryano diff --git a/util-logging/OWNERS b/util-logging/OWNERS index 983c0ab37e..d9ce0ff511 100644 --- a/util-logging/OWNERS +++ b/util-logging/OWNERS @@ -3,3 +3,4 @@ banderson koliver mnakamura roanta +ryano diff --git a/util-stats/OWNERS b/util-stats/OWNERS index ad4779b7a0..b749f10796 100644 --- a/util-stats/OWNERS +++ b/util-stats/OWNERS @@ -2,4 +2,5 @@ banderson koliver mnakamura roanta +ryano vkostyukov diff --git a/util-zk/OWNERS b/util-zk/OWNERS index d893509652..5008e21b63 100644 --- a/util-zk/OWNERS +++ b/util-zk/OWNERS @@ -2,4 +2,5 @@ banderson koliver mnakamura roanta +ryano stuhood From 6fcca2dbdf24c2300bcafe5e83cc37b20188bf3b Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 28 May 2019 18:04:11 +0000 Subject: [PATCH 155/796] util: Add ccoco to some OWNERS files Problem/Solution Add `ccoco` to OWNERS. Differential Revision: https://phabricator.twitter.biz/D320245 --- util-app/OWNERS | 1 + util-intellij/OWNERS | 1 + util-logging/OWNERS | 2 +- util-slf4j-api/OWNERS | 6 ++++++ util-slf4j-jul-bridge/OWNERS | 6 ++++++ 5 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 util-slf4j-api/OWNERS create mode 100644 util-slf4j-jul-bridge/OWNERS diff --git a/util-app/OWNERS b/util-app/OWNERS index 1c2f0835fc..dc165413a2 100644 --- a/util-app/OWNERS +++ b/util-app/OWNERS @@ -1,4 +1,5 @@ banderson +ccoco koliver mnakamura roanta diff --git a/util-intellij/OWNERS b/util-intellij/OWNERS index 1c2f0835fc..dc165413a2 100644 --- a/util-intellij/OWNERS +++ b/util-intellij/OWNERS @@ -1,4 +1,5 @@ banderson +ccoco koliver mnakamura roanta diff --git a/util-logging/OWNERS b/util-logging/OWNERS index d9ce0ff511..dc165413a2 100644 --- a/util-logging/OWNERS +++ b/util-logging/OWNERS @@ -1,5 +1,5 @@ -# hzhang is owner for ScribeHandler only banderson +ccoco koliver mnakamura roanta diff --git a/util-slf4j-api/OWNERS b/util-slf4j-api/OWNERS new file mode 100644 index 0000000000..dc165413a2 --- /dev/null +++ b/util-slf4j-api/OWNERS @@ -0,0 +1,6 @@ +banderson +ccoco +koliver +mnakamura +roanta +ryano diff --git a/util-slf4j-jul-bridge/OWNERS b/util-slf4j-jul-bridge/OWNERS new file mode 100644 index 0000000000..dc165413a2 --- /dev/null +++ b/util-slf4j-jul-bridge/OWNERS @@ -0,0 +1,6 @@ +banderson +ccoco +koliver +mnakamura +roanta +ryano From 1d4bba539efaa893e527ce814c818925816c085c Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 28 May 2019 21:12:10 +0000 Subject: [PATCH 156/796] finagle|util: Add vkostyukov as an owner I gained some understanding of a couple of things in Finagle/Util in the last ~five years. I feel comfortable to be a top-level owner for those two libraries and want to continue learning them while helping with review burden. Differential Revision: https://phabricator.twitter.biz/D319772 --- OWNERS | 1 + util-app/OWNERS | 1 + util-benchmark/OWNERS | 1 + util-codec/OWNERS | 1 + util-intellij/OWNERS | 1 + util-logging/OWNERS | 1 + util-security/OWNERS | 1 + util-zk/OWNERS | 1 + 8 files changed, 8 insertions(+) diff --git a/OWNERS b/OWNERS index 1c2f0835fc..477af14021 100644 --- a/OWNERS +++ b/OWNERS @@ -3,3 +3,4 @@ koliver mnakamura roanta ryano +vkostyukov \ No newline at end of file diff --git a/util-app/OWNERS b/util-app/OWNERS index dc165413a2..05d43901dd 100644 --- a/util-app/OWNERS +++ b/util-app/OWNERS @@ -4,3 +4,4 @@ koliver mnakamura roanta ryano +vkostyukov diff --git a/util-benchmark/OWNERS b/util-benchmark/OWNERS index 1c2f0835fc..b749f10796 100644 --- a/util-benchmark/OWNERS +++ b/util-benchmark/OWNERS @@ -3,3 +3,4 @@ koliver mnakamura roanta ryano +vkostyukov diff --git a/util-codec/OWNERS b/util-codec/OWNERS index 1c2f0835fc..b749f10796 100644 --- a/util-codec/OWNERS +++ b/util-codec/OWNERS @@ -3,3 +3,4 @@ koliver mnakamura roanta ryano +vkostyukov diff --git a/util-intellij/OWNERS b/util-intellij/OWNERS index dc165413a2..05d43901dd 100644 --- a/util-intellij/OWNERS +++ b/util-intellij/OWNERS @@ -4,3 +4,4 @@ koliver mnakamura roanta ryano +vkostyukov diff --git a/util-logging/OWNERS b/util-logging/OWNERS index dc165413a2..05d43901dd 100644 --- a/util-logging/OWNERS +++ b/util-logging/OWNERS @@ -4,3 +4,4 @@ koliver mnakamura roanta ryano +vkostyukov diff --git a/util-security/OWNERS b/util-security/OWNERS index fa6d114e20..f5d09e0dd3 100644 --- a/util-security/OWNERS +++ b/util-security/OWNERS @@ -4,3 +4,4 @@ koliver mnakamura roanta ryano +vkostyukov diff --git a/util-zk/OWNERS b/util-zk/OWNERS index 5008e21b63..40598c8896 100644 --- a/util-zk/OWNERS +++ b/util-zk/OWNERS @@ -4,3 +4,4 @@ mnakamura roanta ryano stuhood +vkostyukov From d176d11e5e28e8b3823ad28db1954eb1e7e91ee4 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 3 Jun 2019 21:08:14 +0000 Subject: [PATCH 157/796] twitter-oss: Update CONTRIBUTING docs to include section on sbt script Problem/Solution We should add a section about the local `sbt` script that we expect users to use when building the project. Differential Revision: https://phabricator.twitter.biz/D322805 --- CONTRIBUTING.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b9a3783de7..e1d6880a9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,13 @@ We'd love to get patches from you! +## Building + +This library is built using [sbt][sbt]. When building please use the included +[`./sbt`](https://github.com/twitter/util/blob/develop/sbt) script which +provides a thin wrapper over [sbt][sbt] and correctly sets memory and other +settings. + If you have any questions or run into any problems, please create an issue here, chat with us in [gitter](https://gitter.im/twitter/finagle), or email the Finaglers [mailing list](https://groups.google.com/forum/#!forum/finaglers). @@ -172,13 +179,14 @@ Please let us know if you have any questions about this process! [0]: https://github.com/twitter/util/pull/109 [1]: https://github.com/twitter/util/blob/master/util-core/src/test/java/com/twitter/util/VarCompilationTest.java -[scaladoc]: https://docs.scala-lang.org/style/scaladoc.html +[changes]: https://github.com/twitter/util/blob/develop/CHANGELOG.rst [es]: https://twitter.github.io/effectivescala/ -[sd]: https://docs.scala-lang.org/style/scaladoc.html [funsuite]: https://www.scalatest.org/getting_started_with_fun_suite +[gendrivenprop]: https://www.scalatest.org/user_guide/generator_driven_property_checks +[sbt]: https://www.scala-sbt.org/ +[scalacheck]: https://www.scalacheck.org/ +[scaladoc]: https://docs.scala-lang.org/style/scaladoc.html [scalatest]: https://www.scalatest.org/ +[sd]: https://docs.scala-lang.org/style/scaladoc.html [ssg]: https://docs.scala-lang.org/style/scaladoc.html -[travis-ci]: https://travis-ci.org/twitter/util -[changes]: https://github.com/twitter/util/blob/develop/CHANGELOG.rst -[scalacheck]: https://www.scalacheck.org/ -[gendrivenprop]: https://www.scalatest.org/user_guide/generator_driven_property_checks +[travis-ci]: https://travis-ci.org/twitter/util \ No newline at end of file From aaa3d6e2bf37a3bd565a8c51187fd9b6db8a0b25 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Thu, 6 Jun 2019 16:32:49 +0000 Subject: [PATCH 158/796] twitter-server: metric metadata endpoint prototype Problem Users would like to know more information about the metrics emitted by a service. The most basic info they'd like is the metric type. Solution Add an endpoint (/admin/exp/metric_metadata) which will return all the metrics and their types as well as whether or not counters are latched. Additionally users may pass in an explicit set of metrics they are curious about via a query param titled m (just like the /admin/metrics endpoint). Result Users can now query their service for type information about their metrics. JIRA Issues: CSL-8125 Differential Revision: https://phabricator.twitter.biz/D313672 --- .../scala/com/twitter/finagle/stats/StatsRegistry.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsRegistry.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsRegistry.scala index b31788f29c..d0aadc223b 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsRegistry.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsRegistry.scala @@ -5,6 +5,10 @@ package com.twitter.finagle.stats * efficient mechanism to sample stats. */ private[twitter] trait StatsRegistry { + + /** Whether or not the counters are latched. */ + val latched: Boolean + def apply(): Map[String, StatEntry] } @@ -19,4 +23,7 @@ private[twitter] trait StatEntry { /** The instantaneous value of the entry. */ val value: Double + + /** The type of the metric backing this StatEntry ("counter", "gauge", or "histogram"). */ + val metricType: String } From 6ac4c6017daf111313b80cee2b658f73458e78f6 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Mon, 10 Jun 2019 20:01:39 +0000 Subject: [PATCH 159/796] [util-core] Reader.empty fails to return `ReaderDiscardedException` after it's discarded Problem Reading from `Reader.empty` returns `None` after the reader is discarded, it should return `ReaderDiscardedException` Solution Check `Reader.empty`'s `closep` in `read()`, return `ReaderDiscardedException` if it's discarded. Result Reading from `Reader.empty` should return `ReaderDiscardedException` if the reader has been discarded. JIRA Issues: CSL-7995 Differential Revision: https://phabricator.twitter.biz/D325465 --- CHANGELOG.rst | 6 ++++++ .../main/scala/com/twitter/io/Reader.scala | 9 +++++++-- .../scala/com/twitter/io/ReaderTest.scala | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f2055721c1..290e1ae6af 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Bug Fixes +~~~~~~~~~ + +* util-core: Fixed the behavior in `c.t.io.Reader` where reading from `Reader#empty` fails to return + a `ReaderDiscardedException` after it's discarded. ``PHAB_ID=D325465`` + 19.5.1 ------ diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index bfff87b447..e489f198b2 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -142,8 +142,13 @@ object Reader { def empty[A]: Reader[A] = new Reader[A] { private[this] val closep = Promise[StreamTermination]() def read(): Future[Option[Nothing]] = { - closep.updateIfEmpty(StreamTermination.FullyRead.Return) - Future.None + if (closep.updateIfEmpty(StreamTermination.FullyRead.Return)) + Future.None + else + closep.flatMap { + case StreamTermination.FullyRead => Future.None + case StreamTermination.Discarded => Future.exception(new ReaderDiscardedException) + } } def discard(): Unit = { closep.updateIfEmpty(StreamTermination.Discarded.Return) diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index c62e6b5bff..eaf5d9f392 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -562,6 +562,25 @@ class ReaderTest assert(await(readAllString(reader2)) == l.mkString) } } + + test( + "Reader.empty " + + "- return Future.None and update closep to be FullyRead after first read") { + val reader = Reader.empty + assert(await(reader.read()) == None) + assert(await(reader.onClose) == StreamTermination.FullyRead) + } + + test( + "Reader.empty " + + "- return ReaderDiscardedException when reading from a discarded reader") { + val reader = Reader.empty + reader.discard() + intercept[ReaderDiscardedException] { + await(reader.read()) + } + assert(await(reader.onClose) == StreamTermination.Discarded) + } } object ReaderTest { From 8ba4c6f2f6261489394a0167f5d274534babbf12 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Tue, 11 Jun 2019 22:36:16 +0000 Subject: [PATCH 160/796] util-core: Optimize Buf.ByteBuffer.equals Problem Checking equality of two Buf.ByteBuffer's is not as fast as it could be. Solution Avoid `Buf.ByteBuffer.unapply` which creates a copy of the underlying `java.nio.ByteBuffer`. Result JMH results are 99.8% faster and allocations are removed. JIRA Issues: CSL-7968 Differential Revision: https://phabricator.twitter.biz/D326177 --- .../src/main/scala/com/twitter/io/Buf.scala | 4 +++- .../src/test/scala/com/twitter/io/BufTest.scala | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 153accb7c8..092119b788 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -864,7 +864,9 @@ object Buf { } override def equals(other: Any): Boolean = other match { - case ByteBuffer(otherBB) => underlying.equals(otherBB) + // while normally this might use `ByteBuffer.unapply` this + // approach avoids the unnecessary `java.nio.ByteBuffer` copies. + case bb: ByteBuffer => underlying.equals(bb.underlying) case buf: Buf => Buf.equals(this, buf) case _ => false } diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index f83b4e01e1..80a5b7122e 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -103,6 +103,22 @@ class BufTest } } + test("Buf.equals") { + val bytes = Array[Byte](0, 1, 2) + val byteBuffer = Buf.ByteBuffer.Owned(java.nio.ByteBuffer.wrap(bytes)) + val byteArray = Buf.ByteArray(bytes: _*) + val composite = Buf.ByteArray(0).concat(Buf.ByteArray(1)).concat(Buf.ByteArray(2)) + val bufs = Seq(byteBuffer, byteArray, composite) + + bufs.foreach { b0 => + bufs.foreach { b1 => + withClue(s"${b0.getClass.getName} and ${b1.getClass.getName}") { + assert(b0 == b1) + } + } + } + } + test("Buf.write(Array[Byte], offset) validates output array is large enough") { // we skip Empty because we cannot create an array // too small for this test. From 50983409c657fa82467726b1dc1072bf7025c1ba Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 13 Jun 2019 20:56:09 +0000 Subject: [PATCH 161/796] util-core: Refactor `FutureReader` to use synchronized block instead of AtomicReference Problem FutureReader is using an AtomicReference to ensure synchronized access to its state, while the logic is complex and the code looks tedious because we need to constantly check if the atomic value is updated or not. Solution Use a synchronized block instead of AtomicReference Result We are able to get rid of the check conditions and `Reading` state, the code looks cleaner and logic is clearer JIRA Issues: CSL-7809 Differential Revision: https://phabricator.twitter.biz/D326466 --- .../scala/com/twitter/io/FutureReader.scala | 95 +++++++++---------- .../main/scala/com/twitter/io/SeqReader.scala | 7 +- 2 files changed, 48 insertions(+), 54 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/FutureReader.scala b/util-core/src/main/scala/com/twitter/io/FutureReader.scala index ba7ea1f48e..806ba77288 100644 --- a/util-core/src/main/scala/com/twitter/io/FutureReader.scala +++ b/util-core/src/main/scala/com/twitter/io/FutureReader.scala @@ -1,7 +1,7 @@ package com.twitter.io +import com.twitter.io.FutureReader._ import com.twitter.util.{Future, Promise, Return, Throw} -import java.util.concurrent.atomic.AtomicReference /** * We want to ensure that this reader always satisfies these invariants: @@ -11,62 +11,61 @@ import java.util.concurrent.atomic.AtomicReference * 4. Reading from a failed reader will always return the exception it has thrown * * We achieved this with a state machine where any access to the state is synchronized, - * and by ensuring that we always verify the update of state succeeded before setting closep, * and by preventing changing the state when the reader is failed, fully read, or discarded */ private[io] final class FutureReader[A](fa: Future[A]) extends Reader[A] { - import FutureReader._ - private[this] val closep = Promise[StreamTermination]() - private[this] val state = new AtomicReference[State](State.Idle) + private[this] var state: State = State.Idle def read(): Future[Option[A]] = { - state.get() match { - case State.Idle => - if (state.compareAndSet(State.Idle, State.Reading)) { + val (updatedState, result) = synchronized { + val result = state match { + case State.Idle => fa.map(Some.apply).respond { - case t: Throw[_] => - if (state.compareAndSet(State.Reading, State.Failed)) { - closep.update(t.cast[StreamTermination]) - } + case Throw(t) => + state = State.Failed(t) case Return(_) => - // it is safe to return the value even when the `state` is not updated, which could happen - // when the `state` is updated to `Discarded` in `discard()`, in that case, we still retain - // the execution order because `read()` happens before `discard()` - state.compareAndSet(State.Reading, State.Read) + state = State.Read } - } else { - // when `state` is updated to `Reading` or `Discarded` - read() - } - case State.Reading => - // multiple outstanding reads are not allowed - Future.exception(new IllegalStateException("read() while read is pending")) - case State.Read => - // update would fail if `state` is already updated to `Discarded` in `discard()` - if (state.compareAndSet(State.Read, State.FullyRead)) { - closep.update(StreamTermination.FullyRead.Return) - } - closep.flatMap { - case StreamTermination.FullyRead => Future.None - case StreamTermination.Discarded => Future.exception(new ReaderDiscardedException) - } - case State.Failed => - // closep is guaranteed to be an exception, flatMap should never be triggered but return the exception - closep.flatMap(_ => Future.None) + case State.Read => + state = State.FullyRead + Future.None + case State.Failed(_) => + // closep is guaranteed to be an exception, flatMap + // should never be triggered but return the exception + closep.flatMap(_ => Future.None) + case State.FullyRead => + Future.None + case State.Discarded => + Future.exception(new ReaderDiscardedException) + } + (state, result) + } + + updatedState match { + case State.Failed(t) => + // use `updateIfEmpty` instead of `update` here because we don't want to throw + // `ImmutableResult` exception when reading from a failed `FutureReader` multiple times + closep.updateIfEmpty(Throw(t)) case State.FullyRead => - Future.None - case State.Discarded => - Future.exception(new ReaderDiscardedException) + closep.updateIfEmpty(StreamTermination.FullyRead.Return) + case _ => } + + result } def discard(): Unit = { - if (state.compareAndSet(State.Idle, State.Discarded) || state - .compareAndSet(State.Read, State.Discarded) || state.compareAndSet( - State.Reading, - State.Discarded)) { - closep.update(StreamTermination.Discarded.Return) + val discarded = synchronized { + state match { + case State.Idle | State.Read => + state = State.Discarded + true + case _ => false + } + } + if (discarded) { + closep.updateIfEmpty(StreamTermination.Discarded.Return) fa.raise(new ReaderDiscardedException) } } @@ -79,8 +78,8 @@ object FutureReader { /** * Indicates reader state when the reader is created via FutureReader */ - sealed trait State - object State { + private sealed trait State + private object State { /** Indicates the reader is ready to be read. */ case object Idle extends State @@ -88,17 +87,13 @@ object FutureReader { /** Indicates the reader has been read. */ case object Read extends State - /** Indicates a reading is in progress. */ - case object Reading extends State - /** Indicates an exception occurred during reading */ - case object Failed extends State + case class Failed(t: Throwable) extends State /** Indicates the EOS has been observed. */ case object FullyRead extends State /** Indicates the reader has been discarded. */ case object Discarded extends State - } } diff --git a/util-core/src/main/scala/com/twitter/io/SeqReader.scala b/util-core/src/main/scala/com/twitter/io/SeqReader.scala index b19d1c4935..bbfe3b4f09 100644 --- a/util-core/src/main/scala/com/twitter/io/SeqReader.scala +++ b/util-core/src/main/scala/com/twitter/io/SeqReader.scala @@ -1,5 +1,6 @@ package com.twitter.io +import com.twitter.io.SeqReader._ import com.twitter.util.{Future, Promise} /** @@ -11,8 +12,6 @@ import com.twitter.util.{Future, Promise} * and by preventing changing the state when the reader is fully read or discarded. */ private[io] final class SeqReader[A](seq: Seq[A]) extends Reader[A] { - import SeqReader._ - private[this] val closep = Promise[StreamTermination]() private[this] var value: Seq[A] = seq private[this] var state: State = State.Idle @@ -63,8 +62,8 @@ object SeqReader { /** * Indicates reader state when the reader is created via SeqReader */ - sealed trait State - object State { + private sealed trait State + private object State { /** Indicates the reader is ready to be read. */ case object Idle extends State From 04cee8fb8e56e2566318abec74eefa59dfd8ad83 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Thu, 13 Jun 2019 23:55:08 +0000 Subject: [PATCH 162/796] util-core: Use locals at callback creation for Future's interrupt handler Problem Future's interrupt handler was using the raiser's locals which is inconsistent with other callbacks. Solution Add the locals to `Interruptible` and use the captured locals in the handler. Hide it behind a toggle that is turned off, and eventually have it on by default. Result When toggled on with `com.twitter.util.UseLocalInInterruptible`, the interrupt handler uses the locals at the time of the callback creation rather than the raiser's locals. JIRA Issues: CSL-8234 Differential Revision: https://phabricator.twitter.biz/D324315 --- CHANGELOG.rst | 8 + .../main/scala/com/twitter/util/Promise.scala | 35 ++++- .../scala/com/twitter/util/PromiseTest.scala | 140 ++++++++++++++++++ 3 files changed, 175 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 290e1ae6af..d1d221ca0c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,14 @@ Bug Fixes * util-core: Fixed the behavior in `c.t.io.Reader` where reading from `Reader#empty` fails to return a `ReaderDiscardedException` after it's discarded. ``PHAB_ID=D325465`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-core: Use Local at callback creation for Future's interrupt handler rather than + raiser's locals so that it is consistent with other callbacks. This functionality is + currently disabled and can be enabled by a toggle (com.twitter.util.UseLocalInInterruptible) + by setting it to 1.0 if you would like to try it out. ``PHAB_ID=D324315`` + 19.5.1 ------ diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 44fced4ecc..bd193dbf5c 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -7,6 +7,16 @@ import scala.util.control.NonFatal object Promise { + // An experimental property that enables the interrupt handler + // to use the state at the time of callback creation. + @volatile private var UseLocalInInterruptible: Boolean = + System.getProperty("com.twitter.util.UseLocalInInterruptible", "false").toBoolean + + // Switches to enabling the interrupt handler to use + // the state at the time of callback creation. + private[twitter] def useLocalInInterruptible(useLocal: Boolean): Unit = + UseLocalInInterruptible = useLocal + /** * Embeds an "interrupt handler" into a [[Promise]]. * @@ -279,7 +289,8 @@ object Promise { */ private class Interruptible[A]( val waitq: WaitQueue[A], - val handler: PartialFunction[Throwable, Unit]) + val handler: PartialFunction[Throwable, Unit], + val saved: Local.Context) /** * An unsatisfied [[Promise]] which forwards interrupts to `other`. @@ -466,7 +477,7 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ def this(handleInterrupt: PartialFunction[Throwable, Unit]) { this() - this.state = new Interruptible[A](WaitQueue.empty, handleInterrupt) + this.state = new Interruptible[A](WaitQueue.empty, handleInterrupt, Local.save()) } def this(result: Try[A]) { @@ -498,15 +509,15 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ @tailrec final def setInterruptHandler(f: PartialFunction[Throwable, Unit]): Unit = state match { case waitq: WaitQueue[A] => - if (!cas(waitq, new Interruptible(waitq, f))) + if (!cas(waitq, new Interruptible(waitq, f, Local.save()))) setInterruptHandler(f) case s: Interruptible[A] => - if (!cas(s, new Interruptible(s.waitq, f))) + if (!cas(s, new Interruptible(s.waitq, f, Local.save()))) setInterruptHandler(f) case s: Transforming[A] => - if (!cas(s, new Interruptible(s.waitq, f))) + if (!cas(s, new Interruptible(s.waitq, f, Local.save()))) setInterruptHandler(f) case s: Interrupted[A] => @@ -568,7 +579,15 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ case s: Interruptible[A] => if (!cas(s, new Interrupted(s.waitq, intr))) raise(intr) else { - s.handler.applyOrElse(intr, Promise.AlwaysUnit) + if (!UseLocalInInterruptible) + s.handler.applyOrElse(intr, Promise.AlwaysUnit) + else { + val current = Local.save() + if (current ne s.saved) + Local.restore(s.saved) + try s.handler.applyOrElse(intr, Promise.AlwaysUnit) + finally Local.restore(current) + } } case s: Transforming[A] => @@ -594,7 +613,7 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ waitq.contains(k) case s: Interruptible[A] => - if (!cas(s, new Interruptible(s.waitq.remove(k), s.handler))) + if (!cas(s, new Interruptible(s.waitq.remove(k), s.handler, s.saved))) detach(k) else s.waitq.contains(k) @@ -802,7 +821,7 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ if (!cas(waitq, WaitQueue(k, waitq))) continue(k) case s: Interruptible[A] => - if (!cas(s, new Interruptible(WaitQueue(k, s.waitq), s.handler))) + if (!cas(s, new Interruptible(WaitQueue(k, s.waitq), s.handler, s.saved))) continue(k) case s: Transforming[A] => if (!cas(s, new Transforming(WaitQueue(k, s.waitq), s.other))) diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index 310dabc4d3..8b167afbf4 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -203,4 +203,144 @@ class PromiseTest extends FunSuite { } } + test("maps use local state at the time of callback creation") { + val local = new Local[String] + local.set(Some("main thread")) + val p = new Promise[String]() + val f = p.map { _ => + local().getOrElse("") + } + + val thread = new Thread(new Runnable { + def run(): Unit = { + local.set(Some("worker thread")) + p.setValue("go") + } + }) + thread.start() + + assert("main thread" == Await.result(f, 3.seconds)) + } + + test("interrupts use local state at the time of callback creation") { + Promise.useLocalInInterruptible(true) + + val local = new Local[String] + implicit val timer: Timer = new JavaTimer(true) + + local.set(Some("main thread")) + val p = new Promise[String] + + p.setInterruptHandler({ + case _ => + val localVal = local().getOrElse("") + p.updateIfEmpty(Return(localVal)) + }) + val f = Future.sleep(1.second).before(p) + + val thread = new Thread(new Runnable { + def run(): Unit = { + local.set(Some("worker thread")) + p.raise(new Exception) + } + }) + thread.start() + + assert("main thread" == Await.result(f, 3.seconds)) + } + + test("interrupts use raisers state") { + Promise.useLocalInInterruptible(false) + + val local = new Local[String] + implicit val timer: Timer = new JavaTimer(true) + + local.set(Some("main thread")) + val p = new Promise[String] + + p.setInterruptHandler({ + case _ => + val localVal = local().getOrElse("") + p.updateIfEmpty(Return(localVal)) + }) + val f = Future.sleep(1.second).before(p) + + val thread = new Thread(new Runnable { + def run(): Unit = { + local.set(Some("worker thread")) + p.raise(new Exception) + } + }) + thread.start() + + assert("worker thread" == Await.result(f, 3.seconds)) + } + + test("interrupt then map uses local state at the time of callback creation") { + val local = new Local[String] + implicit val timer: Timer = new JavaTimer(true) + + local.set(Some("main thread")) + val p = new Promise[String]() + p.setInterruptHandler({ + case _ => + val localVal = local().getOrElse("") + p.updateIfEmpty(Return(localVal)) + }) + val fRes = p.transform { _ => + val localVal = local().getOrElse("") + Future.value(localVal) + } + val f = Future.sleep(1.second).before(fRes) + + val thread = new Thread(new Runnable { + def run(): Unit = { + local.set(Some("worker thread")) + p.raise(new Exception) + } + }) + thread.start() + + assert("main thread" == Await.result(f, 3.seconds)) + } + + test("locals in Interruptible after detatch") { + Promise.useLocalInInterruptible(true) + + val local = new Local[String] + implicit val timer: Timer = new JavaTimer(true) + + local.set(Some("main thread")) + val p = new Promise[String] + + val p2 = Promise.attached(p) // p2 is the detachable promise + + p2.setInterruptHandler({ + case _ => + val localVal = local().getOrElse("") + p2.updateIfEmpty(Return(localVal)) + }) + + p.setInterruptHandler({ + case _ => + val localVal = local().getOrElse("") + p.updateIfEmpty(Return(localVal)) + }) + + val thread = new Thread(new Runnable { + def run(): Unit = { + local.set(Some("worker thread")) + + // We need to call detach explicitly + assert(p2.detach()) + + p2.raise(new Exception) + p.raise(new Exception) + } + }) + thread.start() + + assert("main thread" == Await.result(p2, 3.seconds)) + assert("main thread" == Await.result(p, 3.seconds)) + } } From 5a0458972f63e375f0a8d8d6d626c1f331d763d0 Mon Sep 17 00:00:00 2001 From: Jordan Bull Date: Tue, 18 Jun 2019 21:35:01 +0000 Subject: [PATCH 163/796] util-logging: Add latency stats to scribe logging Problem There is no way to tell if the scribe server is responding in a timely manner causing buffer growth. Solution Add latency metrics to both success and failure calls to the scribe handler. Result Latency when sending to scribe will be visible in stats. Differential Revision: https://phabricator.twitter.biz/D319385 --- .../com/twitter/logging/ScribeHandler.scala | 432 ++++++++++-------- .../twitter/logging/ScribeHandlerTest.scala | 114 +++-- 2 files changed, 299 insertions(+), 247 deletions(-) diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala index 4a9708f203..3e6ce9e203 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala @@ -27,7 +27,7 @@ import com.twitter.concurrent.NamedPoolThreadFactory import com.twitter.conversions.StringOps._ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.{NullStatsReceiver, StatsReceiver} -import com.twitter.util.{Duration, Time} +import com.twitter.util.{Duration, Stopwatch, Time} import scala.util.control.NonFatal object ScribeHandler { @@ -114,6 +114,106 @@ object ScribeHandler { level, NullStatsReceiver ) + + private[logging] val ScribePrefix: Array[Byte] = Array[Byte]( + // version 1, call, "Log", reqid=0 + 0x80.toByte, + 1, + 0, + 1, + 0, + 0, + 0, + 3, + 'L'.toByte, + 'o'.toByte, + 'g'.toByte, + 0, + 0, + 0, + 0, + // list of structs + 15, + 0, + 1, + 12 + ) + private[logging] val OldScribePrefix: Array[Byte] = Array[Byte]( + // (no version), "Log", reply, reqid=0 + 0, + 0, + 0, + 3, + 'L'.toByte, + 'o'.toByte, + 'g'.toByte, + 1, + 0, + 0, + 0, + 0, + // list of structs + 15, + 0, + 1, + 12 + ) + + private[logging] val ScribeReply: Array[Byte] = Array[Byte]( + // version 1, reply, "Log", reqid=0 + 0x80.toByte, + 1, + 0, + 2, + 0, + 0, + 0, + 3, + 'L'.toByte, + 'o'.toByte, + 'g'.toByte, + 0, + 0, + 0, + 0, + // int, fid 0, 0=ok + 8, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ) + private[logging] val OldScribeReply: Array[Byte] = Array[Byte]( + 0, + 0, + 0, + 20, + // (no version), "Log", reply, reqid=0 + 0, + 0, + 0, + 3, + 'L'.toByte, + 'o'.toByte, + 'g'.toByte, + 2, + 0, + 0, + 0, + 0, + // int, fid 0, 0=ok + 8, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ) } /** @@ -193,126 +293,127 @@ class ScribeHandler( def queueSize: Int = queue.size() override def flush(): Unit = { + flusher.execute(new Runnable { + def run(): Unit = { sendBatch() } + }) + } - def connect(): Unit = { - if (!socket.isDefined) { - if (Time.now.since(lastConnectAttempt) > connectBackoff) { - try { - lastConnectAttempt = Time.now - socket = Some(new Socket(hostname, port)) + private def connect(): Unit = { + if (!socket.isDefined) { + if (Time.now.since(lastConnectAttempt) > connectBackoff) { + try { + lastConnectAttempt = Time.now + socket = Some(new Socket(hostname, port)) - stats.incrConnection() - serverType = Unknown - } catch { - case e: Exception => - log.error("Unable to open socket to scribe server at %s:%d: %s", hostname, port, e) - stats.incrConnectionFailure() - } - } else { - stats.incrConnectionSkipped() + stats.incrConnection() + serverType = Unknown + } catch { + case e: Exception => + log.error("Unable to open socket to scribe server at %s:%d: %s", hostname, port, e) + stats.incrConnectionFailure() } + } else { + stats.incrConnectionSkipped() } } + } - def detectArchaicServer(): Unit = { - if (serverType != Unknown) return + private def detectArchaicServer(): Unit = { + if (serverType != Unknown) return - serverType = socket match { - case None => Unknown - case Some(s) => { - val outStream = s.getOutputStream() + serverType = socket match { + case None => Unknown + case Some(s) => { + val outStream = s.getOutputStream() - try { - val fakeMessageWithOldScribePrefix: Array[Byte] = { - val prefix = OLD_SCRIBE_PREFIX - val messageSize = prefix.length + 5 - val buffer = ByteBuffer.wrap(new Array[Byte](messageSize + 4)) - buffer.order(ByteOrder.BIG_ENDIAN) - buffer.putInt(messageSize) - buffer.put(prefix) - buffer.putInt(0) - buffer.put(0: Byte) - buffer.array - } - - outStream.write(fakeMessageWithOldScribePrefix) - readResponseExpecting(s, OLD_SCRIBE_REPLY) - - // Didn't get exception, so the server must be archaic. - log.debug("Scribe server is archaic; changing to old protocol for future requests.") - Archaic - } catch { - case NonFatal(_) => Modern + try { + val fakeMessageWithOldScribePrefix: Array[Byte] = { + val prefix = OldScribePrefix + val messageSize = prefix.length + 5 + val buffer = ByteBuffer.wrap(new Array[Byte](messageSize + 4)) + buffer.order(ByteOrder.BIG_ENDIAN) + buffer.putInt(messageSize) + buffer.put(prefix) + buffer.putInt(0) + buffer.put(0: Byte) + buffer.array } + + outStream.write(fakeMessageWithOldScribePrefix) + readResponseExpecting(s, OldScribeReply) + + // Didn't get exception, so the server must be archaic. + log.debug("Scribe server is archaic; changing to old protocol for future requests.") + Archaic + } catch { + case NonFatal(_) => Modern } } } + } - // TODO we should send any remaining messages before the app terminates - def sendBatch(): Unit = { - synchronized { - connect() - detectArchaicServer() - socket.foreach { s => - val outStream = s.getOutputStream() - var remaining = queue.size - // try to send the log records in batch - // need to check if socket is closed due to exception - while (remaining > 0 && socket.isDefined) { - val count = maxMessagesPerTransaction min remaining - val buffer = makeBuffer(count) - stats.statBatchSize(count, buffer.capacity) - - try { - outStream.write(buffer.array) - val expectedReply = if (isArchaicServer()) OLD_SCRIBE_REPLY else SCRIBE_REPLY - - readResponseExpecting(s, expectedReply) - - stats.incrSentRecords(count) - remaining -= count - } catch { - case e: Exception => - stats.incrDroppedRecords(count) - log.error( - e, - "Failed to send %s %d log entries to scribe server at %s:%d", - category, - count, - hostname, - port - ) - closeSocket() - } + // TODO we should send any remaining messages before the app terminates + // visible for tests + private[logging] def sendBatch(): Unit = { + synchronized { + connect() + detectArchaicServer() + socket.foreach { s => + val outStream = s.getOutputStream() + var remaining = queue.size + // try to send the log records in batch + // need to check if socket is closed due to exception + while (remaining > 0 && socket.isDefined) { + val count = maxMessagesPerTransaction min remaining + val buffer = makeBuffer(count) + stats.statBatchSize(count, buffer.capacity) + + val elapsed = Stopwatch.start() + try { + outStream.write(buffer.array) + val expectedReply = if (isArchaicServer()) OldScribeReply else ScribeReply + + readResponseExpecting(s, expectedReply) + + stats.sendRecordSuccess(count, elapsed().inMillis) + remaining -= count + } catch { + case e: Exception => + stats.sendRecordFailure(count, elapsed().inMillis) + log.error( + e, + "Failed to send %s %d log entries to scribe server at %s:%d", + category, + count, + hostname, + port + ) + closeSocket() } - updateLastTransmission() } - stats.log() + updateLastTransmission() } + stats.log() } + } - def readResponseExpecting(socket: Socket, expectedReply: Array[Byte]): Unit = { - var offset = 0 + private def readResponseExpecting(socket: Socket, expectedReply: Array[Byte]): Unit = { + var offset = 0 - val inStream = socket.getInputStream() + val inStream = socket.getInputStream() - // read response: - val response = new Array[Byte](expectedReply.length) - while (offset < response.length) { - val n = inStream.read(response, offset, response.length - offset) - if (n < 0) { - throw new IOException("End of stream") - } - offset += n - } - if (!Arrays.equals(response, expectedReply)) { - throw new IOException("Error response from scribe server: " + response.hexlify) + // read response: + val response = new Array[Byte](expectedReply.length) + while (offset < response.length) { + val n = inStream.read(response, offset, response.length - offset) + if (n < 0) { + throw new IOException("End of stream") } + offset += n + } + if (!Arrays.equals(response, expectedReply)) { + throw new IOException("Error response from scribe server: " + response.hexlify) } - - flusher.execute(new Runnable { - def run(): Unit = { sendBatch() } - }) } // should be private, make it visible to tests @@ -327,7 +428,7 @@ class ScribeHandler( recordHeader.put(11: Byte) recordHeader.putShort(2) - val prefix = if (isArchaicServer()) OLD_SCRIBE_PREFIX else SCRIBE_PREFIX + val prefix = if (isArchaicServer()) OldScribePrefix else ScribePrefix val messageSize = (count * (recordHeader.capacity + 5)) + texts.foldLeft(0) { _ + _.length } + prefix.length + 5 val buffer = ByteBuffer.wrap(new Array[Byte](messageSize + 4)) buffer.order(ByteOrder.BIG_ENDIAN) @@ -391,106 +492,6 @@ class ScribeHandler( ) } - private[this] val SCRIBE_PREFIX: Array[Byte] = Array[Byte]( - // version 1, call, "Log", reqid=0 - 0x80.toByte, - 1, - 0, - 1, - 0, - 0, - 0, - 3, - 'L'.toByte, - 'o'.toByte, - 'g'.toByte, - 0, - 0, - 0, - 0, - // list of structs - 15, - 0, - 1, - 12 - ) - private[this] val OLD_SCRIBE_PREFIX: Array[Byte] = Array[Byte]( - // (no version), "Log", reply, reqid=0 - 0, - 0, - 0, - 3, - 'L'.toByte, - 'o'.toByte, - 'g'.toByte, - 1, - 0, - 0, - 0, - 0, - // list of structs - 15, - 0, - 1, - 12 - ) - - private[this] val SCRIBE_REPLY: Array[Byte] = Array[Byte]( - // version 1, reply, "Log", reqid=0 - 0x80.toByte, - 1, - 0, - 2, - 0, - 0, - 0, - 3, - 'L'.toByte, - 'o'.toByte, - 'g'.toByte, - 0, - 0, - 0, - 0, - // int, fid 0, 0=ok - 8, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ) - private[this] val OLD_SCRIBE_REPLY: Array[Byte] = Array[Byte]( - 0, - 0, - 0, - 20, - // (no version), "Log", reply, reqid=0 - 0, - 0, - 0, - 3, - 'L'.toByte, - 'o'.toByte, - 'g'.toByte, - 2, - 0, - 0, - 0, - 0, - // int, fid 0, 0=ok - 8, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ) - private class ScribeHandlerStats(statsReceiver: StatsReceiver) { private var _lastLogStats = Time.epoch @@ -512,11 +513,40 @@ class ScribeHandler( val batchSizeBytesStat = statsReceiver.stat("batch_size_bytes") val batchSizeCountStat = statsReceiver.stat("batch_size_messages") - def incrSentRecords(count: Int): Unit = { + val successLatencyStat = statsReceiver.stat("success_latency_ms") + val failureLatencyStat = statsReceiver.stat("failure_latency_ms") + + /** + * Update stats for the number of records sent and the histogram of latency to scribe. + * @param count + * The number of new messages sent to scribe. + * @param latencyMs + * The latency of the successful send to scribe. + */ + def sendRecordSuccess(count: Int, latencyMs: Long): Unit = { sentRecords.addAndGet(count) totalSentRecords.incr(count) + successLatencyStat.add(latencyMs) + } + + /** + * Update stats for the number of records dropped and the latency histogram for failed requests + * to the scribe server. + * @param count + * The number of messages that failed to send to scribe and were dropped. + * @param latencyMs + * The latency of the failed request to scribe. + */ + def sendRecordFailure(count: Int, latencyMs: Long): Unit = { + incrDroppedRecords(count) + failureLatencyStat.add(latencyMs) } + /** + * Update stats for the number of records dropped. + * @param count + * The number of messages that were dropped. + */ def incrDroppedRecords(count: Int = 1): Unit = { droppedRecords.addAndGet(count) totalDroppedRecords.incr(count) diff --git a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala index bb1c56f833..b68af502a4 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala @@ -21,7 +21,7 @@ import com.twitter.conversions.StringOps._ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.InMemoryStatsReceiver import com.twitter.util.{RandomSocket, Time} -import java.io.ByteArrayOutputStream +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} import java.net.Socket import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.{ @@ -122,7 +122,10 @@ class ScribeHandlerTest extends WordSpec with BeforeAndAfter with Eventually wit scribe.publish(record2) assert(statsReceiver.counter("dropped_records")() == 1l) + // Failures that never attempt to write to scribe do not have a latency + assert(statsReceiver.stat("failure_latency_ms")().size == 0) assert(statsReceiver.counter("sent_records")() == 0l) + assert(statsReceiver.stat("failure_latency_ms")().size == 0) } "have backoff on connection errors" in { @@ -225,76 +228,95 @@ class ScribeHandlerTest extends WordSpec with BeforeAndAfter with Eventually wit assert(rejected.get() == 94) } - "report batch sizes" in { - val statsReceiver = new InMemoryStatsReceiver - val mockSocket = mock[Socket] - when(mockSocket.getOutputStream).thenReturn(new ByteArrayOutputStream()) + "report scribe send stats" in { + Time.withCurrentTimeFrozen { timeControl => + val statsReceiver = new InMemoryStatsReceiver + val mockSocket = mock[Socket] + when(mockSocket.getOutputStream).thenReturn(new ByteArrayOutputStream()) + when(mockSocket.getInputStream) + .thenReturn(null) // First do not respond to attempts to use the old protocol + .thenAnswer(_ => { + timeControl.advance(707.millis) + new ByteArrayInputStream(ScribeHandler.ScribeReply) + }) - val scribe = ScribeHandler( - port = portWithoutListener, - category = "test", - bufferTime = 5.seconds, - formatter = BareFormatter, - level = None, - statsReceiver = statsReceiver - ).apply() + val scribe = ScribeHandler( + port = portWithoutListener, + category = "test", + bufferTime = 5.seconds, + formatter = BareFormatter, + level = None, + statsReceiver = statsReceiver + ).apply() - scribe.setSocket(Some(mockSocket)) + scribe.setSocket(Some(mockSocket)) - // Get the buffer first to know the size before flushing - Time.withCurrentTimeFrozen { _ => + // Get the buffer first to know the size before flushing scribe.updateLastTransmission() scribe.publish(record2) scribe.publish(record1) - } - val buffer = scribe.makeBuffer(2) + val buffer = scribe.makeBuffer(2) - // Make sure multiple records get flushed together - Time.withCurrentTimeFrozen { _ => + // Make sure multiple records get flushed together scribe.updateLastTransmission() scribe.publish(record2) scribe.publish(record1) - } - scribe.flush() + scribe.sendBatch() - scribe.flusher.shutdown() - scribe.flusher.awaitTermination(15, TimeUnit.SECONDS) + scribe.flusher.shutdown() + scribe.flusher.awaitTermination(15, TimeUnit.SECONDS) + + assert(statsReceiver.stat("batch_size_bytes")() == Seq(buffer.capacity())) + assert(statsReceiver.stat("batch_size_messages")() == Seq(2)) - assert(statsReceiver.stat("batch_size_bytes")() == Seq(buffer.capacity())) - assert(statsReceiver.stat("batch_size_messages")() == Seq(2)) + assert(statsReceiver.counter("sent_records")() == 2L) + assert(statsReceiver.stat("success_latency_ms")() == Seq(707)) + assert(statsReceiver.counter("dropped_records")() == 0L) + assert(statsReceiver.stat("failure_latency_ms")() == Seq.empty) + } } "handle batch write errors" in { - val statsReceiver = new InMemoryStatsReceiver - val mockSocket = mock[Socket] - when(mockSocket.getOutputStream).thenReturn(null) + Time.withCurrentTimeFrozen { timeControl => + val statsReceiver = new InMemoryStatsReceiver + val mockSocket = mock[Socket] + when(mockSocket.getOutputStream).thenReturn(new ByteArrayOutputStream()) + when(mockSocket.getInputStream) + .thenReturn(null) // First do not respond to attempts to use the old protocol + .thenAnswer(_ => { + timeControl.advance(1738.millis) + // An empty byte array ensures failure + new ByteArrayInputStream(Array.emptyByteArray) + }) - val scribe = ScribeHandler( - port = portWithoutListener, - category = "test", - bufferTime = 5.seconds, - formatter = BareFormatter, - level = None, - statsReceiver = statsReceiver - ).apply() + val scribe = ScribeHandler( + port = portWithoutListener, + category = "test", + bufferTime = 5.seconds, + formatter = BareFormatter, + level = None, + statsReceiver = statsReceiver + ).apply() - scribe.setSocket(Some(mockSocket)) + scribe.setSocket(Some(mockSocket)) - // Make sure multiple records get flushed together - Time.withCurrentTimeFrozen { _ => + // Make sure multiple records get flushed together scribe.updateLastTransmission() scribe.publish(record2) scribe.publish(record1) - } - // Throws exception writing to null output stream - scribe.flush() + // Throws exception reading an empty response + scribe.sendBatch() - scribe.flusher.shutdown() - scribe.flusher.awaitTermination(15, TimeUnit.SECONDS) + scribe.flusher.shutdown() + scribe.flusher.awaitTermination(15, TimeUnit.SECONDS) - assert(statsReceiver.counter("dropped_records")() == 2L) + assert(statsReceiver.counter("dropped_records")() == 2L) + assert(statsReceiver.stat("failure_latency_ms")() == Seq(1738)) + assert(statsReceiver.counter("sent_records")() == 0L) + assert(statsReceiver.stat("success_latency_ms")() == Seq.empty) + } } } } From a576c4eacfddd6ce0f48796831c273637ea63e45 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 18 Jun 2019 23:55:09 +0000 Subject: [PATCH 164/796] util-logging: Switch away from lambda in ScribeHandlerTest Problem/Solution Sbt complaining missing parameter type for creating Answer[?] by lambda, switch away from it. JIRA Issues: CSL-8346 Differential Revision: https://phabricator.twitter.biz/D330090 --- .../twitter/logging/ScribeHandlerTest.scala | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala index b68af502a4..4756b161f0 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala @@ -31,15 +31,14 @@ import java.util.concurrent.{ TimeUnit } import java.util.{logging => javalog} -import org.junit.runner.RunWith import org.mockito.Mockito._ -import org.scalatest.junit.JUnitRunner +import org.mockito.invocation.InvocationOnMock +import org.mockito.stubbing.Answer import org.scalatest.{BeforeAndAfter, WordSpec} import org.scalatest.concurrent.Eventually import org.scalatest.mockito.MockitoSugar import org.scalatest.time.{Millis, Minutes, Span} -@RunWith(classOf[JUnitRunner]) class ScribeHandlerTest extends WordSpec with BeforeAndAfter with Eventually with MockitoSugar { val record1 = new javalog.LogRecord(Level.INFO, "This is a message.") record1.setMillis(1206769996722L) @@ -235,9 +234,11 @@ class ScribeHandlerTest extends WordSpec with BeforeAndAfter with Eventually wit when(mockSocket.getOutputStream).thenReturn(new ByteArrayOutputStream()) when(mockSocket.getInputStream) .thenReturn(null) // First do not respond to attempts to use the old protocol - .thenAnswer(_ => { - timeControl.advance(707.millis) - new ByteArrayInputStream(ScribeHandler.ScribeReply) + .thenAnswer(new Answer[ByteArrayInputStream] { + def answer(invocation: InvocationOnMock): ByteArrayInputStream = { + timeControl.advance(707.millis) + new ByteArrayInputStream(ScribeHandler.ScribeReply) + } }) val scribe = ScribeHandler( @@ -284,10 +285,12 @@ class ScribeHandlerTest extends WordSpec with BeforeAndAfter with Eventually wit when(mockSocket.getOutputStream).thenReturn(new ByteArrayOutputStream()) when(mockSocket.getInputStream) .thenReturn(null) // First do not respond to attempts to use the old protocol - .thenAnswer(_ => { - timeControl.advance(1738.millis) - // An empty byte array ensures failure - new ByteArrayInputStream(Array.emptyByteArray) + .thenAnswer(new Answer[ByteArrayInputStream] { + def answer(invocation: InvocationOnMock): ByteArrayInputStream = { + timeControl.advance(1738.millis) + // An empty byte array ensures failure + new ByteArrayInputStream(Array.emptyByteArray) + } }) val scribe = ScribeHandler( From 54213cb3e55e55c429bd2087744f0db5aa837f12 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 19 Jun 2019 03:48:20 +0000 Subject: [PATCH 165/796] twitter-oss: Prepare OSS Libraries for Release 19.6.0 Problem We want to release the next version of our Twitter OSS libraries 19.6.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next release. JIRA Issues: CSL-8329 Differential Revision: https://phabricator.twitter.biz/D329712 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d1d221ca0c..06c6ae82b3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.6.0 +------ + Bug Fixes ~~~~~~~~~ diff --git a/README.md b/README.md index 1b33c6f215..1f2cb13dee 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.5.1" +val collUtils = "com.twitter" %% "util-collection" % "19.6.0" ``` # Units diff --git a/build.sbt b/build.sbt index a9f1d84ec1..60479e8b32 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.6.0-SNAPSHOT" +val releaseVersion = "19.6.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From ac4ae17041bc8ed3b8abdc06e8d671baac4cc7d2 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 19 Jun 2019 18:48:06 +0000 Subject: [PATCH 166/796] util: Remove Logging section from README.md Problem We don't want users to be relying on util-logging anymore but it is still discussed in the README.md. Solution Remove the sections describing util-logging. JIRA Issues: CSL-8208 Differential Revision: https://phabricator.twitter.biz/D330429 --- README.md | 222 +++++------------------------------------------------- 1 file changed, 19 insertions(+), 203 deletions(-) diff --git a/README.md b/README.md index 1f2cb13dee..95ec07f723 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,24 @@ for { } ``` +## Future interrupts + +Method `raise` on `Future` (`def raise(cause: Throwable)`) raises the interrupt described by +`cause` to the producer of this `Future`. Interrupt handlers are installed on a `Promise` +using `setInterruptHandler`, which takes a partial function: + +```scala +val p = new Promise[T] +p.setInterruptHandler { + case exc: MyException => + // deal with interrupt.. +} +``` + +Interrupts differ in semantics from cancellation in important ways: there can only be one +interrupt handler per promise, and interrupts are only delivered if the promise is not yet +complete. + # Object Pool The pool order is FIFO. @@ -159,209 +177,7 @@ val distributor = new KetamaDistributor(nodes, 1 /* num reps */) distributor.nodeForHash("abc".##) // => client ``` -# Logging - -`util-logging` is a small wrapper around Java's built-in logging to make it more -Scala-friendly. - -## Using - -To access logging, you can usually just use: - -```scala -import com.twitter.logging.Logger -private val log = Logger.get(getClass) -``` - -This creates a `Logger` object that uses the current class or object's -package name as the logging node, so class "com.example.foo.Lamp" will log to -node `com.example.foo` (generally showing "foo" as the name in the logfile). -You can also get a logger explicitly by name: - -```scala -private val log = Logger.get("com.example.foo") -``` - -Logger objects wrap everything useful from `java.util.logging.Logger`, as well -as adding some convenience methods: - -```scala -// log a string with sprintf conversion: -log.info("Starting compaction on zone %d...", zoneId) - -try { - ... -} catch { - // log an exception backtrace with the message: - case e: IOException => - log.error(e, "I/O exception: %s", e.getMessage) -} -``` - -Each of the log levels (from "fatal" to "trace") has these two convenience -methods. You may also use `log` directly: - -```scala -import com.twitter.logging.Level -log(Level.DEBUG, "Logging %s at debug level.", name) -``` - -An advantage to using sprintf ("%s", etc) conversion, as opposed to: - -```scala -log(Level.DEBUG, s"Logging $name at debug level.") -``` - -is that Java & Scala perform string concatenation at runtime, even if nothing -will be logged because the log file isn't writing debug messages right now. -With `sprintf` parameters, the arguments are just bundled up and passed directly -to the logging level before formatting. If no log message would be written to -any file or device, then no formatting is done and the arguments are thrown -away. That makes it very inexpensive to include verbose debug logging which -can be turned off without recompiling and re-deploying. - -If you prefer, there are also variants that take lazily evaluated parameters, -and only evaluate them if logging is active at that level: - -```scala -log.ifDebug(s"Login from $name at $date.") -``` - -The logging classes are done as an extension to the `java.util.logging` API, -and so you can use the Java interface directly, if you want to. Each of the -Java classes (Logger, Handler, Formatter) is just wrapped by a Scala class. - - -## Configuring - -In the Java style, log nodes are in a tree, with the root node being "" (the -empty string). If a node has a filter level set, only log messages of that -priority or higher are passed up to the parent. Handlers are attached to nodes -for sending log messages to files or logging services, and may have formatters -attached to them. - -Logging levels are, in priority order of highest to lowest: - -- `FATAL` - the server is about to exit -- `CRITICAL` - an event occurred that is bad enough to warrant paging someone -- `ERROR` - a user-visible error occurred (though it may be limited in scope) -- `WARNING` - a coder may want to be notified, but the error was probably not - user-visible -- `INFO` - normal informational messages -- `DEBUG` - coder-level debugging information -- `TRACE` - intensive debugging information - -Each node may also optionally choose to *not* pass messages up to the parent -node. - -The `LoggerFactory` builder is used to configure individual log nodes, by -filling in fields and calling the `apply` method. For example, to configure -the root logger to filter at `INFO` level and write to a file: - -```scala -import com.twitter.logging._ - -val factory = LoggerFactory( - node = "", - level = Some(Level.INFO), - handlers = List( - FileHandler( - filename = "/var/log/example/example.log", - rollPolicy = Policy.SigHup - ) - ) -) - -val logger = factory() -``` - -As many `LoggerFactory`s can be configured as you want, so you can attach to -several nodes if you like. To remove all previous configurations, use: - -```scala -Logger.clearHandlers() -``` - -## Handlers - -- `QueueingHandler` - - Queues log records and publishes them in another thread thereby enabling "async logging". - -- `ConsoleHandler` - - Logs to the console. - -- `FileHandler` - - Logs to a file, with an optional file rotation policy. The policies are: - - - `Policy.Never` - always use the same logfile (default) - - `Policy.Hourly` - roll to a new logfile at the top of every hour - - `Policy.Daily` - roll to a new logfile at midnight every night - - `Policy.Weekly(n)` - roll to a new logfile at midnight on day N (0 = Sunday) - - `Policy.SigHup` - reopen the logfile on SIGHUP (for logrotate and similar services) - - When a logfile is rolled, the current logfile is renamed to have the date - (and hour, if rolling hourly) attached, and a new one is started. So, for - example, `test.log` may become `test-20080425.log`, and `test.log` will be - reopened as a new file. - -- `SyslogHandler` - - Log to a syslog server, by host and port. - -- `ScribeHandler` - - Log to a scribe server, by host, port, and category. Buffering and backoff - can also be configured: You can specify how long to collect log lines - before sending them in a single burst, the maximum burst size, and how long - to backoff if the server seems to be offline. - -- `ThrottledHandler` - - Wraps another handler, tracking (and squelching) duplicate messages. If you - use a format string like `"Error %d at %s"`, the log messages will be - de-duped based on the format string, even if they have different parameters. - - -## Formatters - -Handlers usually have a formatter attached to them, and these formatters -generally just add a prefix containing the date, log level, and logger name. - -- `Formatter` - - A standard log prefix like `"ERR [20080315-18:39:05.033] jobs: "`, which - can be configured to truncate log lines to a certain length, limit the lines - of an exception stack trace, and use a special time zone. - - You can override the format string used to generate the prefix, also. - -- `BareFormatterConfig` - - No prefix at all. May be useful for logging info destined for scripts. - -- `SyslogFormatterConfig` - - A formatter required by the syslog protocol, with configurable syslog - priority and date format. - -## Future interrupts - -Method `raise` on `Future` (`def raise(cause: Throwable)`) raises the interrupt described by `cause` to the producer of this `Future`. Interrupt handlers are installed on a `Promise` using `setInterruptHandler`, which takes a partial function: - -```scala -val p = new Promise[T] -p.setInterruptHandler { - case exc: MyException => - // deal with interrupt.. -} -``` - -Interrupts differ in semantics from cancellation in important ways: there can only be one interrupt handler per promise, and interrupts are only delivered if the promise is not yet complete. - -## Time and Duration +# Time and Duration Like arithmetic on doubles, `Time` and `Duration` arithmetic is now free of overflows. Instead, they overflow to `Top` and `Bottom` values, which are analogous to positive and negative infinity. From 365ecf01382cbce6304c870ac8941018bcaf8558 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 20 Jun 2019 00:12:15 +0000 Subject: [PATCH 167/796] twitter-oss: Update OSS Libraries Post-Release to 19.7.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries: 19.7.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-8329 Differential Revision: https://phabricator.twitter.biz/D330715 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 95ec07f723..0b2d779b0a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 60479e8b32..b33f7eab6f 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.6.0" +val releaseVersion = "19.7.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 0d77572c76c7c54c0b10a1d25856af16148fe3c4 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Thu, 20 Jun 2019 18:20:48 +0000 Subject: [PATCH 168/796] util-core: Remove deprecated methods from Scheduler Problem A few methods on `c.t.concurrent.Scheduler` were deprecated back in 2015 and were returning -1 as their implementation. Solution Remove the `usrTime`, `cpuTime`, and `wallTime` methods. Differential Revision: https://phabricator.twitter.biz/D330386 --- CHANGELOG.rst | 7 +++++ .../com/twitter/concurrent/Scheduler.scala | 31 ------------------- 2 files changed, 7 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 06c6ae82b3..0df8fe05aa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: Removed deprecated `c.t.concurrent.Scheduler` methods `usrTime`, + `cpuTime`, and `wallTime`. These were deprecated in 2015 and have no + replacement. ``PHAB_ID=D330386`` + 19.6.0 ------ diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala index 38d7b4cbf9..00a00838b5 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala @@ -23,37 +23,6 @@ trait Scheduler { */ def flush(): Unit - // A note on Hotspot's ThreadMXBean's CPU time. On Linux, this - // uses clock_gettime[1] which should both be fast and accurate. - // - // On OSX, the Mach thread_info call is used. - // - // [1] https://linux.die.net/man/3/clock_gettime - // - // Note, these clock stats were decommissioned as they only make - // sense relative to `wallTime` and the tracking error we have - // experienced `wallTime` and `*Time` make it impossible to - // use these reliably. It is not worth the performance and - // code complexity to support them. - - /** - * The amount of User time that's been scheduled as per ThreadMXBean. - */ - @deprecated("schedulers no longer export this", "2015-01-10") - def usrTime: Long = -1L - - /** - * The amount of CPU time that's been scheduled as per ThreadMXBean. - */ - @deprecated("schedulers no longer export this", "2015-01-10") - def cpuTime: Long = -1L - - /** - * Total walltime spent in the scheduler. - */ - @deprecated("schedulers no longer export this", "2015-01-10") - def wallTime: Long = -1L - /** * The number of dispatches performed by this scheduler. */ From ad9ccd08935e9516926c2ff86890c6783727b0e2 Mon Sep 17 00:00:00 2001 From: Martijn Hoekstra Date: Mon, 24 Jun 2019 21:58:27 +0000 Subject: [PATCH 169/796] util: Remove all RunWith Problem Superfluous calls to RunWith will require changes to the import for JUnitRunner. Solution Remove all RunWith and associated changes Result Code that won't need to be changed in the future to address deprecated alias for JUnitRunner Signed-off-by: Yufan Gong Differential Revision: https://phabricator.twitter.biz/D332290 --- .../test/scala/com/twitter/cache/guava/GuavaCacheTest.scala | 3 --- .../com/twitter/cache/guava/LoadingFutureCacheTest.scala | 3 --- .../scala/com/twitter/cache/ConcurrentMapCacheTest.scala | 3 --- .../src/test/scala/com/twitter/cache/EvictingCacheTest.scala | 3 --- .../test/scala/com/twitter/cache/KeyEncodingCacheTest.scala | 3 --- .../scala/com/twitter/cache/caffeine/CaffeineCacheTest.scala | 3 --- .../com/twitter/cache/caffeine/LoadingFutureCacheTest.scala | 3 --- .../src/test/scala/com/twitter/util/StringEncoderTest.scala | 5 ----- .../test/scala/com/twitter/concurrent/AsyncMeterTest.scala | 3 --- .../test/scala/com/twitter/concurrent/AsyncMutexTest.scala | 4 ---- .../scala/com/twitter/concurrent/AsyncSemaphoreTest.scala | 3 --- .../test/scala/com/twitter/concurrent/AsyncStreamTest.scala | 3 --- .../src/test/scala/com/twitter/concurrent/BrokerTest.scala | 3 --- .../src/test/scala/com/twitter/concurrent/OfferTest.scala | 3 --- .../src/test/scala/com/twitter/concurrent/OnceTest.scala | 3 --- .../src/test/scala/com/twitter/concurrent/PeriodTest.scala | 3 --- .../test/scala/com/twitter/concurrent/SchedulerTest.scala | 5 ----- .../test/scala/com/twitter/concurrent/SerializedTest.scala | 5 +---- .../test/scala/com/twitter/concurrent/SpoolSourceTest.scala | 3 --- .../src/test/scala/com/twitter/concurrent/SpoolTest.scala | 3 --- util-core/src/test/scala/com/twitter/concurrent/TxTest.scala | 4 ---- .../src/test/scala/com/twitter/io/BufInputStreamTest.scala | 3 --- util-core/src/test/scala/com/twitter/io/BufReaderTest.scala | 3 --- util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala | 3 --- util-core/src/test/scala/com/twitter/io/FilesTest.scala | 3 --- .../test/scala/com/twitter/io/InputStreamReaderTest.scala | 3 --- util-core/src/test/scala/com/twitter/io/StreamIOTest.scala | 3 --- .../src/test/scala/com/twitter/io/TempDirectoryTest.scala | 3 --- util-core/src/test/scala/com/twitter/io/TempFileTest.scala | 4 ---- .../test/scala/com/twitter/io/exp/ActivitySourceTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/ActivityTest.scala | 3 --- .../src/test/scala/com/twitter/util/BoundedStackTest.scala | 3 --- .../src/test/scala/com/twitter/util/CancellableTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/ClosableTest.scala | 3 --- .../src/test/scala/com/twitter/util/CloseAwaitablyTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/ConfigTest.scala | 3 --- .../src/test/scala/com/twitter/util/CredentialsTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/DiffTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/DurationTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/EventTest.scala | 3 --- .../src/test/scala/com/twitter/util/FutureOfferTest.scala | 4 ---- .../src/test/scala/com/twitter/util/FuturePoolTest.scala | 3 --- .../test/scala/com/twitter/util/LastWriteWinsQueueTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/MemoizeTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/NetUtilTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/PoolTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/PromiseTest.scala | 3 --- .../src/test/scala/com/twitter/util/StateMachineTest.scala | 3 --- .../src/test/scala/com/twitter/util/StorageUnitTest.scala | 3 --- .../src/test/scala/com/twitter/util/ThrowablesTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/TimeTest.scala | 4 ---- util-core/src/test/scala/com/twitter/util/TimerTest.scala | 3 --- .../src/test/scala/com/twitter/util/TokenBucketTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/TryTest.scala | 3 --- .../test/scala/com/twitter/util/TwitterDateFormatTest.scala | 3 --- util-core/src/test/scala/com/twitter/util/VarTest.scala | 3 --- .../src/test/scala/com/twitter/util/WindowedAdderTest.scala | 3 --- .../src/test/scala/com/twitter/hashing/DiagnosticsTest.scala | 3 --- .../src/test/scala/com/twitter/hashing/HashableTest.scala | 3 --- .../scala/com/twitter/hashing/KetamaDistributorTest.scala | 3 --- .../src/test/scala/com/twitter/hashing/KeyHasherTest.scala | 3 --- .../src/test/scala/com/twitter/util/lint/RuleTest.scala | 3 --- .../src/test/scala/com/twitter/util/lint/RulesTest.scala | 3 --- .../src/test/scala/com/twitter/logging/FileHandlerTest.scala | 3 --- .../src/test/scala/com/twitter/logging/FormatterTest.scala | 3 --- .../src/test/scala/com/twitter/logging/HasLogLevelTest.scala | 3 --- .../src/test/scala/com/twitter/logging/LogRecordTest.scala | 3 --- .../src/test/scala/com/twitter/logging/LoggerTest.scala | 3 --- .../src/test/scala/com/twitter/logging/PolicyTest.scala | 3 --- .../test/scala/com/twitter/logging/QueueingHandlerTest.scala | 3 --- .../test/scala/com/twitter/logging/SyslogHandlerTest.scala | 3 --- .../src/test/scala/com/twitter/util/reflect/ProxyTest.scala | 3 --- .../test/scala/com/twitter/util/registry/FormatterTest.scala | 3 --- .../scala/com/twitter/util/registry/GlobalRegistryTest.scala | 3 --- .../test/scala/com/twitter/util/registry/LibraryTest.scala | 3 --- .../test/scala/com/twitter/util/registry/RosterTest.scala | 3 --- .../scala/com/twitter/util/registry/SimpleRegistryTest.scala | 3 --- .../test/scala/com/twitter/util/security/PemFileTest.scala | 3 --- .../twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala | 3 --- .../com/twitter/util/security/X500PrincipalInfoTest.scala | 3 --- .../com/twitter/util/security/X509CertificateFileTest.scala | 3 --- .../twitter/util/security/X509TrustManagerFactoryTest.scala | 3 --- .../twitter/finagle/stats/BroadcastStatsReceiverTest.scala | 3 --- .../stats/CategorizingExceptionStatsHandlerTest.scala | 3 --- .../twitter/finagle/stats/DelegatingStatsReceiverTest.scala | 3 --- .../twitter/finagle/stats/InMemoryStatsReceiverTest.scala | 3 --- .../stats/MultiCategorizingExceptionStatsHandlerTest.scala | 3 --- .../scala/com/twitter/finagle/stats/StatsReceiverTest.scala | 3 --- .../com/twitter/finagle/stats/WithHistogramDetailsTest.scala | 3 --- .../scala/com/twitter/util/testing/ArgumentCaptureTest.scala | 3 --- .../src/test/scala/com/twitter/util/ThriftCodecTest.scala | 3 --- .../test/scala/com/twitter/util/ThriftSerializerTest.scala | 3 --- .../test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala | 3 --- util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala | 3 --- util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala | 3 --- util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala | 3 --- .../com/twitter/zk/coordination/ShardCoordinatorTest.scala | 3 --- .../com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala | 3 --- 98 files changed, 1 insertion(+), 304 deletions(-) diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/GuavaCacheTest.scala b/util-cache-guava/src/test/scala/com/twitter/cache/guava/GuavaCacheTest.scala index 675fa956c0..13612f638a 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/GuavaCacheTest.scala +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/GuavaCacheTest.scala @@ -3,10 +3,7 @@ package com.twitter.cache.guava import com.google.common.cache.{CacheLoader, CacheBuilder} import com.twitter.cache.AbstractFutureCacheTest import com.twitter.util.{Future, Promise} -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class GuavaCacheTest extends AbstractFutureCacheTest { def name: String = "GuavaCache" diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/LoadingFutureCacheTest.scala b/util-cache-guava/src/test/scala/com/twitter/cache/guava/LoadingFutureCacheTest.scala index 3be8ea6cac..9cdefcf3ee 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/LoadingFutureCacheTest.scala +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/LoadingFutureCacheTest.scala @@ -3,10 +3,7 @@ package com.twitter.cache.guava import com.google.common.cache.{CacheLoader, CacheBuilder} import com.twitter.cache.AbstractLoadingFutureCacheTest import com.twitter.util.Future -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class LoadingFutureCacheTest extends AbstractLoadingFutureCacheTest { def name: String = "LoadingFutureCache (guava)" diff --git a/util-cache/src/test/scala/com/twitter/cache/ConcurrentMapCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/ConcurrentMapCacheTest.scala index 5d3c3ea6b1..1462b3dc17 100644 --- a/util-cache/src/test/scala/com/twitter/cache/ConcurrentMapCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/ConcurrentMapCacheTest.scala @@ -2,10 +2,7 @@ package com.twitter.cache import com.twitter.util.Future import java.util.concurrent.ConcurrentHashMap -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class ConcurrentMapCacheTest extends AbstractFutureCacheTest { def name: String = "ConcurrentMapCache" diff --git a/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala index 23b97e55fd..e7a0b6498c 100644 --- a/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala @@ -2,13 +2,10 @@ package com.twitter.cache import com.twitter.util.{Promise, Future} import java.util.concurrent.ConcurrentHashMap -import org.junit.runner.RunWith import org.mockito.Mockito.{verify, never} import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar -@RunWith(classOf[JUnitRunner]) class EvictingCacheTest extends FunSuite with MockitoSugar { test("EvictingCache should evict on failed futures for set") { val cache = mock[FutureCache[String, String]] diff --git a/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala index 5c696d40de..99bf6bed02 100644 --- a/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala @@ -2,10 +2,7 @@ package com.twitter.cache import java.util.concurrent.ConcurrentHashMap import com.twitter.util.Future -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class KeyEncodingCacheTest extends AbstractFutureCacheTest { def name: String = "KeyEncodingCache" diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/CaffeineCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/caffeine/CaffeineCacheTest.scala index 0a9dfeaff0..19ec88a095 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/CaffeineCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/CaffeineCacheTest.scala @@ -3,10 +3,7 @@ package com.twitter.cache.caffeine import com.github.benmanes.caffeine.cache.{Caffeine, CacheLoader, LoadingCache} import com.twitter.cache.AbstractFutureCacheTest import com.twitter.util.{Future, Promise} -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class CaffeineCacheTest extends AbstractFutureCacheTest { def name: String = "CaffeineCache" diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/LoadingFutureCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/caffeine/LoadingFutureCacheTest.scala index a13cd950f4..65b66ad9d9 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/LoadingFutureCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/LoadingFutureCacheTest.scala @@ -3,10 +3,7 @@ package com.twitter.cache.caffeine import com.github.benmanes.caffeine.cache.{Caffeine, CacheLoader} import com.twitter.cache.AbstractLoadingFutureCacheTest import com.twitter.util.Future -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class LoadingFutureCacheTest extends AbstractLoadingFutureCacheTest { def name: String = "LoadingFutureCache (caffeine)" diff --git a/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala b/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala index 5520a5f90c..c653281214 100644 --- a/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala +++ b/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala @@ -18,11 +18,8 @@ package com.twitter.util import java.lang.StringBuilder -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class StringEncoderTest extends WordSpec { val longString = "A string that is really really really really really really long and has more than 76 characters" @@ -38,7 +35,6 @@ class StringEncoderTest extends WordSpec { } } -@RunWith(classOf[JUnitRunner]) class Base64StringEncoderTest extends WordSpec { val urlUnsafeBytes = Array(-1.toByte, -32.toByte) val resultUnsafe = "/+A" @@ -59,7 +55,6 @@ class Base64StringEncoderTest extends WordSpec { } } -@RunWith(classOf[JUnitRunner]) class GZIPStringEncoderTest extends WordSpec { "a gzip string encoder" should { val gse = new GZIPStringEncoder {} diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala index 050ea9dbd2..aeba180642 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala @@ -3,11 +3,8 @@ package com.twitter.concurrent import com.twitter.util._ import com.twitter.conversions.DurationOps._ import java.util.concurrent.{RejectedExecutionException, CancellationException} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class AsyncMeterTest extends FunSuite { import AsyncMeter._ diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncMutexTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncMutexTest.scala index 81887c5bf3..db42ec9c38 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncMutexTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncMutexTest.scala @@ -1,12 +1,8 @@ package com.twitter.concurrent -import org.junit.runner.RunWith import org.scalatest.FlatSpec -import org.scalatest.junit.JUnitRunner - import com.twitter.util.Await -@RunWith(classOf[JUnitRunner]) class AsyncMutexTest extends FlatSpec { "AsyncMutex" should "admit only one operation at a time" in { val m = new AsyncMutex diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index 63085ff026..41310bc64f 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -3,13 +3,10 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.util.concurrent.{ConcurrentLinkedQueue, RejectedExecutionException, CountDownLatch} -import org.junit.runner.RunWith import org.scalatest.fixture.FunSpec -import org.scalatest.junit.JUnitRunner import scala.collection.mutable import org.scalatest.Outcome -@RunWith(classOf[JUnitRunner]) class AsyncSemaphoreTest extends FunSpec { class AsyncSemaphoreHelper( val sem: AsyncSemaphore, diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index ac2ba71537..01d56a1f7b 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -2,13 +2,10 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ import com.twitter.util._ -import org.junit.runner.RunWith import org.scalacheck.{Arbitrary, Gen} import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks -@RunWith(classOf[JUnitRunner]) class AsyncStreamTest extends FunSuite with GeneratorDrivenPropertyChecks { import AsyncStream.{mk, of} import AsyncStreamTest._ diff --git a/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala b/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala index 3a91785c52..a3cf23567e 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala @@ -1,11 +1,8 @@ package com.twitter.concurrent import com.twitter.util.{Await, Return, ObjectSizeCalculator} -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class BrokerTest extends WordSpec { "Broker" should { "send data (send, recv)" in { diff --git a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala index 7127cc2842..0638d86c4f 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala @@ -2,10 +2,8 @@ package com.twitter.concurrent import scala.util.Random -import org.junit.runner.RunWith import org.mockito.Mockito._ import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ @@ -22,7 +20,6 @@ class SimpleOffer[T](var futures: Stream[Future[Tx[T]]]) extends Offer[T] { } } -@RunWith(classOf[JUnitRunner]) class OfferTest extends WordSpec with MockitoSugar { import Tx.{Commit, Abort} diff --git a/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala index 0d52547087..868ba58903 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala @@ -2,11 +2,8 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ import com.twitter.util.{Promise, Await} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class OnceTest extends FunSuite { test("Once.apply should only be applied once") { var x = 0 diff --git a/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala b/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala index 06b13208c5..ee45366c0c 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala @@ -1,11 +1,8 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class PeriodTest extends FunSuite { test("Period#numPeriods should behave reasonably") { val period = new Period(1.second) diff --git a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala index 31c343fd7f..e2502824b4 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala @@ -4,10 +4,8 @@ import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.util.concurrent.{Executors, TimeUnit} import java.util.logging.{Handler, Level, LogRecord} -import org.junit.runner.RunWith import org.scalatest.{FunSuite, Matchers} import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import org.scalatest.junit.JUnitRunner abstract class LocalSchedulerTest(lifo: Boolean) extends FunSuite with Matchers { private val scheduler = new LocalScheduler(lifo) @@ -154,13 +152,10 @@ abstract class LocalSchedulerTest(lifo: Boolean) extends FunSuite with Matchers } -@RunWith(classOf[JUnitRunner]) class LocalSchedulerFifoTest extends LocalSchedulerTest(false) -@RunWith(classOf[JUnitRunner]) class LocalSchedulerLifoTest extends LocalSchedulerTest(true) -@RunWith(classOf[JUnitRunner]) class ThreadPoolSchedulerTest extends FunSuite with Eventually with IntegrationPatience { test("works") { val p = new Promise[Unit] diff --git a/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala index 7279bdffcc..7411cd1336 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala @@ -1,11 +1,8 @@ package com.twitter.concurrent import java.util.concurrent.CountDownLatch - -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) + class SerializedTest extends WordSpec with Serialized { "Serialized" should { "runs blocks, one at a time, in the order received" in { diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala index 942b674255..7b2c590a41 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala @@ -2,11 +2,8 @@ package com.twitter.concurrent import com.twitter.util.{Return, Await} import java.util.concurrent.atomic.AtomicInteger -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class SpoolSourceTest extends WordSpec { "SpoolSource" should { class SpoolSourceHelper { diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala index 35b51d7036..35d9dfeb71 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala @@ -4,14 +4,11 @@ import com.twitter.concurrent.Spool.{*::, seqToSpool} import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, Promise, Return, Throw} import java.io.EOFException -import org.junit.runner.RunWith import org.scalacheck.Arbitrary import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks import scala.collection.mutable.ArrayBuffer -@RunWith(classOf[JUnitRunner]) class SpoolTest extends WordSpec with GeneratorDrivenPropertyChecks { "Empty Spool" should { val s = Spool.empty[Int] diff --git a/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala b/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala index 051e1e7e5f..6abc0bcd30 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala @@ -1,12 +1,8 @@ package com.twitter.concurrent -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner - import com.twitter.util.Return -@RunWith(classOf[JUnitRunner]) class TxTest extends WordSpec { "Tx.twoParty" should { "commit when everything goes dandy" in { diff --git a/util-core/src/test/scala/com/twitter/io/BufInputStreamTest.scala b/util-core/src/test/scala/com/twitter/io/BufInputStreamTest.scala index c18c222d2c..47795ec9ad 100644 --- a/util-core/src/test/scala/com/twitter/io/BufInputStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufInputStreamTest.scala @@ -1,10 +1,7 @@ package com.twitter.io -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class BufInputStreamTest extends FunSuite { private[this] val fileString = "Test_All_Tests\nTest_java_io_BufferedInputStream\nTest_java_io_BufferedOutputStream\nTest_ByteArrayInputStream\nTest_java_io_ByteArrayOutputStream\nTest_java_io_DataInputStream\n" diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index 5e5febd66d..ba5ce2464f 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -2,12 +2,9 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.Checkers -@RunWith(classOf[JUnitRunner]) class BufReaderTest extends FunSuite with Checkers { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) diff --git a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala index 6e83eb6a10..7762bbf9f6 100644 --- a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala @@ -2,13 +2,10 @@ package com.twitter.io import java.lang.{Double => JDouble, Float => JFloat} import java.nio.charset.StandardCharsets -import org.junit.runner.RunWith import org.scalacheck.Gen import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks -@RunWith(classOf[JUnitRunner]) class ByteReaderTest extends FunSuite with GeneratorDrivenPropertyChecks { import ByteReader._ diff --git a/util-core/src/test/scala/com/twitter/io/FilesTest.scala b/util-core/src/test/scala/com/twitter/io/FilesTest.scala index 0ce3e1657f..0c09a18b55 100644 --- a/util-core/src/test/scala/com/twitter/io/FilesTest.scala +++ b/util-core/src/test/scala/com/twitter/io/FilesTest.scala @@ -1,11 +1,8 @@ package com.twitter.io import java.io.File -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class FilesTest extends WordSpec with TempFolder { "Files" should { diff --git a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala index ac62566073..be8458e7a7 100644 --- a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala @@ -3,12 +3,9 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, FuturePool} import java.io.ByteArrayInputStream -import org.junit.runner.RunWith import org.mockito.Mockito._ import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class InputStreamReaderTest extends FunSuite { def arr(i: Int, j: Int): Array[Byte] = Array.range(i, j).map(_.toByte) def buf(i: Int, j: Int): Buf = Buf.ByteArray.Owned(arr(i, j)) diff --git a/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala b/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala index 36753b42d2..cbffb0c3a2 100644 --- a/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala +++ b/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala @@ -2,11 +2,8 @@ package com.twitter.io import scala.util.Random -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class StreamIOTest extends WordSpec { "StreamIO.copy" should { "copy the entire stream" in { diff --git a/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala b/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala index ea0d6be031..49366cdb68 100644 --- a/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala +++ b/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala @@ -1,10 +1,7 @@ package com.twitter.io -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class TempDirectoryTest extends WordSpec { "TempDirectory" should { diff --git a/util-core/src/test/scala/com/twitter/io/TempFileTest.scala b/util-core/src/test/scala/com/twitter/io/TempFileTest.scala index b20b8496c2..bf792caf79 100644 --- a/util-core/src/test/scala/com/twitter/io/TempFileTest.scala +++ b/util-core/src/test/scala/com/twitter/io/TempFileTest.scala @@ -2,12 +2,8 @@ package com.twitter.io import java.io.{ByteArrayInputStream, DataInputStream} import java.util.Arrays - -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class TempFileTest extends WordSpec { "TempFile" should { diff --git a/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala index 9b042b093c..ae6cd586a6 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala +++ b/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala @@ -3,12 +3,9 @@ package com.twitter.io.exp import com.twitter.io.Buf import com.twitter.util.{Activity, FuturePool, MockTimer, Time} import java.io.{File, ByteArrayInputStream} -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner import org.scalatest.{BeforeAndAfter, FunSuite} import scala.util.Random -@RunWith(classOf[JUnitRunner]) class ActivitySourceTest extends FunSuite with BeforeAndAfter { val ok: ActivitySource[String] = new ActivitySource[String] { diff --git a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala index 2509d10519..7c53e9f9b8 100644 --- a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala @@ -1,11 +1,8 @@ package com.twitter.util import java.util.concurrent.atomic.AtomicReference -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class ActivityTest extends FunSuite { test("Activity#flatMap") { val v = Var(Activity.Pending: Activity.State[Int]) diff --git a/util-core/src/test/scala/com/twitter/util/BoundedStackTest.scala b/util-core/src/test/scala/com/twitter/util/BoundedStackTest.scala index 1954326f87..fa469eb13d 100644 --- a/util-core/src/test/scala/com/twitter/util/BoundedStackTest.scala +++ b/util-core/src/test/scala/com/twitter/util/BoundedStackTest.scala @@ -1,10 +1,7 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class BoundedStackTest extends WordSpec { "BoundedStack" should { "empty" in { diff --git a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala index a5f2307869..d64ec4512a 100644 --- a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala @@ -1,10 +1,7 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class CancellableTest extends WordSpec { "CancellableSink" should { "cancel once" in { diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index e006a98b6b..b742a797dd 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -1,13 +1,10 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import org.scalatest.junit.JUnitRunner import scala.language.reflectiveCalls -@RunWith(classOf[JUnitRunner]) class ClosableTest extends FunSuite with Eventually with IntegrationPatience { test("Closable.close(Duration)") { Time.withCurrentTimeFrozen { _ => diff --git a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala index e62c0c8acb..d46ceb168e 100644 --- a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala @@ -1,11 +1,8 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class CloseAwaitablyTest extends FunSuite { class Context extends Closable with CloseAwaitably { val p: Promise[Unit] = new Promise[Unit] diff --git a/util-core/src/test/scala/com/twitter/util/ConfigTest.scala b/util-core/src/test/scala/com/twitter/util/ConfigTest.scala index 70749bde78..6bb9f670d4 100644 --- a/util-core/src/test/scala/com/twitter/util/ConfigTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ConfigTest.scala @@ -1,10 +1,7 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.{Matchers, WordSpec} -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class ConfigTest extends WordSpec with Matchers { import Config._ diff --git a/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala b/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala index 5149232002..97ab1f7ade 100644 --- a/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala @@ -16,12 +16,9 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.Checkers -@RunWith(classOf[JUnitRunner]) class CredentialsTest extends FunSuite with Checkers { test("parse a simple auth file") { val content = "username: root\npassword: hellokitty\n" diff --git a/util-core/src/test/scala/com/twitter/util/DiffTest.scala b/util-core/src/test/scala/com/twitter/util/DiffTest.scala index ecc577275e..be324ad134 100644 --- a/util-core/src/test/scala/com/twitter/util/DiffTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DiffTest.scala @@ -1,12 +1,9 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalacheck.Arbitrary.arbitrary import org.scalatest.prop.GeneratorDrivenPropertyChecks -@RunWith(classOf[JUnitRunner]) class DiffTest extends FunSuite with GeneratorDrivenPropertyChecks { val f: Int => String = _.toString diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 3e0f299a04..6c2c701a28 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -18,10 +18,7 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import java.util.concurrent.TimeUnit -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class DurationTest extends { val ops: Duration.type = Duration } with TimeLikeSpec[Duration] { "Duration" should { diff --git a/util-core/src/test/scala/com/twitter/util/EventTest.scala b/util-core/src/test/scala/com/twitter/util/EventTest.scala index 612788b997..4c60225788 100644 --- a/util-core/src/test/scala/com/twitter/util/EventTest.scala +++ b/util-core/src/test/scala/com/twitter/util/EventTest.scala @@ -3,11 +3,8 @@ package com.twitter.util import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import java.util.concurrent.{CountDownLatch, Executors} import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -import org.junit.runner.RunWith import scala.collection.mutable -@RunWith(classOf[JUnitRunner]) class EventTest extends FunSuite { test("pub/sub while active") { diff --git a/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala b/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala index 2c4f7eb8de..ae24555c5d 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala @@ -1,13 +1,9 @@ package com.twitter.concurrent -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar - import com.twitter.util.{Promise, Return} -@RunWith(classOf[JUnitRunner]) class FutureOfferTest extends WordSpec with MockitoSugar { "Future.toOffer" should { "activate when future is satisfied (poll)" in { diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index 6899468db1..c8eac5fcad 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -2,15 +2,12 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import java.util.concurrent.{Future => JFuture, _} -import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.concurrent.Eventually -import org.scalatest.junit.JUnitRunner import org.scalatest.time.{Millis, Seconds, Span} import scala.runtime.NonLocalReturnControl import scala.util.control.NonFatal -@RunWith(classOf[JUnitRunner]) class FuturePoolTest extends FunSuite with Eventually { implicit override val patienceConfig: PatienceConfig = diff --git a/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala b/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala index a799d9be8f..b00894dcb9 100644 --- a/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala @@ -1,10 +1,7 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class LastWriteWinsQueueTest extends WordSpec { "LastWriteWinsQueue" should { val queue = new LastWriteWinsQueue[String] diff --git a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala index f2a613d27b..ff8d883389 100644 --- a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala @@ -3,12 +3,9 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.{TimeUnit, CountDownLatch => JavaCountDownLatch} -import org.junit.runner.RunWith import org.mockito.Mockito._ import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class MemoizeTest extends FunSuite { test("Memoize.apply: only runs the function once for the same input") { // mockito can't spy anonymous classes, diff --git a/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala b/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala index 8018fbc9a5..e9d9cb5062 100644 --- a/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala +++ b/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala @@ -2,11 +2,8 @@ package com.twitter.util import java.net.InetAddress -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class NetUtilTest extends WordSpec { "NetUtil" should { "isIpv4Address" in { diff --git a/util-core/src/test/scala/com/twitter/util/PoolTest.scala b/util-core/src/test/scala/com/twitter/util/PoolTest.scala index bbfa4c476c..e9e7d7adb0 100644 --- a/util-core/src/test/scala/com/twitter/util/PoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PoolTest.scala @@ -2,13 +2,10 @@ package com.twitter.util import scala.collection.mutable -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import com.twitter.conversions.DurationOps._ -@RunWith(classOf[JUnitRunner]) class PoolTest extends WordSpec { "SimplePool" should { "with a simple queue of items" should { diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index 8b167afbf4..79dc59ea41 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -1,11 +1,8 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class PromiseTest extends FunSuite { test("Promise.detach should not detach other attached promises") { diff --git a/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala b/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala index a93a416c8e..c472fee792 100644 --- a/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala @@ -1,10 +1,7 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class StateMachineTest extends WordSpec { "StateMachine" should { class StateMachineHelper { diff --git a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala index 1770334a62..cbfa0e6b0c 100644 --- a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala @@ -1,12 +1,9 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import com.twitter.conversions.StorageUnitOps._ -@RunWith(classOf[JUnitRunner]) class StorageUnitTest extends FunSuite { test("StorageUnit: should convert whole numbers into storage units (back and forth)") { assert(1.byte.inBytes == 1) diff --git a/util-core/src/test/scala/com/twitter/util/ThrowablesTest.scala b/util-core/src/test/scala/com/twitter/util/ThrowablesTest.scala index 626e404750..a9d1d55368 100644 --- a/util-core/src/test/scala/com/twitter/util/ThrowablesTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ThrowablesTest.scala @@ -1,10 +1,7 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class ThrowablesTest extends FunSuite { test("Throwables.mkString: flatten non-nested exception") { val t = new Throwable diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index a1ad02d134..3e2ab9f423 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -4,10 +4,8 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, import java.util.{Locale, TimeZone} import java.util.concurrent.TimeUnit -import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks import com.twitter.conversions.DurationOps._ @@ -373,7 +371,6 @@ trait TimeLikeSpec[T <: TimeLike[T]] extends WordSpec with GeneratorDrivenProper } } -@RunWith(classOf[JUnitRunner]) class TimeFormatTest extends WordSpec { "TimeFormat" should { "format correctly with non US locale" in { @@ -398,7 +395,6 @@ class TimeFormatTest extends WordSpec { } } -@RunWith(classOf[JUnitRunner]) class TimeTest extends { val ops: Time.type = Time } with TimeLikeSpec[Time] with Eventually with IntegrationPatience { diff --git a/util-core/src/test/scala/com/twitter/util/TimerTest.scala b/util-core/src/test/scala/com/twitter/util/TimerTest.scala index 42131993f6..9930ba8b49 100644 --- a/util-core/src/test/scala/com/twitter/util/TimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimerTest.scala @@ -3,16 +3,13 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import java.util.concurrent.{CancellationException, ExecutorService, TimeUnit, CountDownLatch} import java.util.concurrent.atomic.AtomicInteger -import org.junit.runner.RunWith import org.mockito.ArgumentCaptor import org.mockito.Matchers.any import org.mockito.Mockito.{never, verify, when} import org.scalatest.FunSuite import org.scalatest.concurrent.{IntegrationPatience, Eventually} -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar -@RunWith(classOf[JUnitRunner]) class TimerTest extends FunSuite with MockitoSugar with Eventually with IntegrationPatience { private def testTimerRunsWithLocals(timer: Timer): Unit = { diff --git a/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala b/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala index 830ac82ab1..a20f09e11a 100644 --- a/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala @@ -1,11 +1,8 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import com.twitter.conversions.DurationOps._ -@RunWith(classOf[JUnitRunner]) class TokenBucketTest extends FunSuite { test("a leaky bucket is leaky") { Time.withCurrentTimeFrozen { tc => diff --git a/util-core/src/test/scala/com/twitter/util/TryTest.scala b/util-core/src/test/scala/com/twitter/util/TryTest.scala index ffc1453cb2..f40a490109 100644 --- a/util-core/src/test/scala/com/twitter/util/TryTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TryTest.scala @@ -1,10 +1,7 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class TryTest extends FunSuite { class MyException extends Exception val e: Exception = new Exception("this is an exception") diff --git a/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala b/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala index 07293c9f81..d86073335a 100644 --- a/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala @@ -2,11 +2,8 @@ package com.twitter.util import java.util.Locale -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class TwitterDateFormatTest extends WordSpec { "TwitterDateFormat" should { "disallow Y without w" in { diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index ef2a55b7cd..88e49f5794 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -1,14 +1,11 @@ package com.twitter.util import java.util.concurrent.atomic.AtomicReference -import org.junit.runner.RunWith import org.scalacheck.Arbitrary.arbitrary import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks import scala.collection.mutable -@RunWith(classOf[JUnitRunner]) class VarTest extends FunSuite with GeneratorDrivenPropertyChecks { private case class U[T](init: T) extends UpdatableVar[T](init) { import Var.Observer diff --git a/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala b/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala index 7a8f552774..ffea53ecc2 100644 --- a/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala @@ -1,11 +1,8 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class WindowedAdderTest extends FunSuite { private def newAdder() = WindowedAdder(3 * 1000, 3, Stopwatch.timeMillis) diff --git a/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala index 502daea63d..f5777a7c0d 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala @@ -1,10 +1,7 @@ package com.twitter.hashing -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class DiagnosticsTest extends WordSpec { "Diagnostics" should { "print distribution" in { diff --git a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala index 43c8715111..b7fe1d3e27 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala @@ -1,11 +1,8 @@ package com.twitter.hashing -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner import org.scalatest.FunSuite import org.scalatest.prop.GeneratorDrivenPropertyChecks -@RunWith(classOf[JUnitRunner]) class HashableTest extends FunSuite with GeneratorDrivenPropertyChecks { private[this] val algorithms = Seq( diff --git a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala index 9c7f5e6620..313a806411 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala @@ -1,16 +1,13 @@ package com.twitter.hashing -import org.junit.runner.RunWith import org.scalacheck.Gen import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks import scala.collection.mutable import java.io.{BufferedReader, InputStreamReader} import java.nio.{ByteBuffer, ByteOrder} import java.security.MessageDigest -@RunWith(classOf[JUnitRunner]) class KetamaDistributorTest extends WordSpec with GeneratorDrivenPropertyChecks { "KetamaDistributor" should { val nodes = Seq( diff --git a/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala index 27cd1f67ce..b628b39abc 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala @@ -2,13 +2,10 @@ package com.twitter.hashing import com.twitter.io.TempFile import java.util.Base64 -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import scala.collection.mutable.ListBuffer import java.nio.charset.StandardCharsets.UTF_8 -@RunWith(classOf[JUnitRunner]) class KeyHasherTest extends WordSpec { def readResource(name: String) = { var lines = new ListBuffer[String]() diff --git a/util-lint/src/test/scala/com/twitter/util/lint/RuleTest.scala b/util-lint/src/test/scala/com/twitter/util/lint/RuleTest.scala index 20d4602c87..3cf47ccf69 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/RuleTest.scala +++ b/util-lint/src/test/scala/com/twitter/util/lint/RuleTest.scala @@ -1,10 +1,7 @@ package com.twitter.util.lint -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class RuleTest extends FunSuite { private def withName(name: String): Rule = diff --git a/util-lint/src/test/scala/com/twitter/util/lint/RulesTest.scala b/util-lint/src/test/scala/com/twitter/util/lint/RulesTest.scala index 780e240dd6..802b8522e0 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/RulesTest.scala +++ b/util-lint/src/test/scala/com/twitter/util/lint/RulesTest.scala @@ -1,10 +1,7 @@ package com.twitter.util.lint -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class RulesTest extends FunSuite { private var flag = false diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index 36aa95554c..d179d7ef20 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -18,15 +18,12 @@ package com.twitter.logging import java.io._ import java.util.{Calendar, Date, logging => javalog} -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import com.twitter.util.Time -@RunWith(classOf[JUnitRunner]) class FileHandlerTest extends WordSpec with TempFolder { def reader(filename: String) = { new BufferedReader(new InputStreamReader(new FileInputStream(new File(folderName, filename)))) diff --git a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala index be51e09564..0181b3d583 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala @@ -18,13 +18,10 @@ package com.twitter.logging import java.util.{logging => javalog} -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import com.twitter.conversions.StringOps._ -@RunWith(classOf[JUnitRunner]) class FormatterTest extends WordSpec { val basicFormatter = new Formatter diff --git a/util-logging/src/test/scala/com/twitter/logging/HasLogLevelTest.scala b/util-logging/src/test/scala/com/twitter/logging/HasLogLevelTest.scala index df570d8270..6191cd7464 100644 --- a/util-logging/src/test/scala/com/twitter/logging/HasLogLevelTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/HasLogLevelTest.scala @@ -1,10 +1,7 @@ package com.twitter.logging -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class HasLogLevelTest extends FunSuite { private class WithLogLevel(val logLevel: Level, cause: Throwable = null) diff --git a/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala b/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala index e83febd379..039c1937b7 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala @@ -1,11 +1,8 @@ package com.twitter.logging import java.util.logging.{LogRecord => JRecord} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class LogRecordTest extends FunSuite { test("LogRecord should getMethod properly") { Logger.withLoggers(Nil) { diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index a8fd342c0c..ef93b3b6c4 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -21,12 +21,9 @@ import com.twitter.io.TempFolder import java.net.InetSocketAddress import java.util.concurrent.{Callable, CountDownLatch, Executors, Future, TimeUnit} import java.util.{logging => javalog} -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner import org.scalatest.{BeforeAndAfter, WordSpec} import scala.collection.mutable -@RunWith(classOf[JUnitRunner]) class LoggerTest extends WordSpec with TempFolder with BeforeAndAfter { val logLevel = Logger.levelNames(Option[String](System.getenv("log")).getOrElse("FATAL").toUpperCase) diff --git a/util-logging/src/test/scala/com/twitter/logging/PolicyTest.scala b/util-logging/src/test/scala/com/twitter/logging/PolicyTest.scala index 389ef61030..9663ff61af 100644 --- a/util-logging/src/test/scala/com/twitter/logging/PolicyTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/PolicyTest.scala @@ -1,12 +1,9 @@ package com.twitter.logging -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import com.twitter.util.StorageUnit -@RunWith(classOf[JUnitRunner]) class PolicyTest extends FunSuite { import Policy._ diff --git a/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala index 7a1597ad26..52c36680de 100644 --- a/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala @@ -18,12 +18,9 @@ package com.twitter.logging import com.twitter.util.Local import java.util.{logging => javalog} -import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.concurrent.{IntegrationPatience, Eventually} -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class QueueingHandlerTest extends WordSpec with Eventually with IntegrationPatience { class MockHandler extends Handler(BareFormatter, None) { diff --git a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala index 992967e773..1e16ce18fd 100644 --- a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala @@ -19,11 +19,8 @@ package com.twitter.logging import java.net.{DatagramPacket, DatagramSocket} import java.util.{logging => javalog} -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class SyslogHandlerTest extends WordSpec { val record1 = new javalog.LogRecord(Level.FATAL, "fatal message!") record1.setLoggerName("net.lag.whiskey.Train") diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala index 77a9a898a4..ff10443d7a 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala @@ -1,9 +1,7 @@ package com.twitter.util.reflect import com.twitter.util.{Future, Promise, Stopwatch} -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner object ProxySpec { trait TestInterface { @@ -34,7 +32,6 @@ object ProxySpec { } } -@RunWith(classOf[JUnitRunner]) class ProxyTest extends WordSpec { import ProxySpec._ diff --git a/util-registry/src/test/scala/com/twitter/util/registry/FormatterTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/FormatterTest.scala index f601cb7068..9fdb617d34 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/FormatterTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/FormatterTest.scala @@ -1,11 +1,8 @@ package com.twitter.util.registry -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import scala.util.control.NoStackTrace -@RunWith(classOf[JUnitRunner]) class FormatterTest extends FunSuite { test("asMap generates reasonable Maps") { val registry = new SimpleRegistry diff --git a/util-registry/src/test/scala/com/twitter/util/registry/GlobalRegistryTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/GlobalRegistryTest.scala index e14c6a9665..c7997fd9c2 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/GlobalRegistryTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/GlobalRegistryTest.scala @@ -1,9 +1,6 @@ package com.twitter.util.registry -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class GlobalRegistryTest extends RegistryTest { def mkRegistry(): Registry = GlobalRegistry.withRegistry(new SimpleRegistry) { GlobalRegistry.get diff --git a/util-registry/src/test/scala/com/twitter/util/registry/LibraryTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/LibraryTest.scala index aad1b70b34..a5f58a6afd 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/LibraryTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/LibraryTest.scala @@ -1,10 +1,7 @@ package com.twitter.util.registry -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner import org.scalatest.FunSuite -@RunWith(classOf[JUnitRunner]) class LibraryTest extends FunSuite { test("Library.register registers libraries") { val simple = new SimpleRegistry diff --git a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala index e70c688fcb..0ff2f77c42 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala @@ -3,12 +3,9 @@ package com.twitter.util.registry import java.util.logging.Logger import org.mockito.Mockito.{never, verify} import org.mockito.Matchers.anyObject -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner import org.scalatest.FunSuite import org.scalatest.mockito.MockitoSugar -@RunWith(classOf[JUnitRunner]) class RosterTest extends FunSuite with MockitoSugar { def withRoster(fn: (Roster, Logger) => Unit): Unit = { val simple = new SimpleRegistry diff --git a/util-registry/src/test/scala/com/twitter/util/registry/SimpleRegistryTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/SimpleRegistryTest.scala index 33a08d229f..d43e8ae142 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/SimpleRegistryTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/SimpleRegistryTest.scala @@ -1,9 +1,6 @@ package com.twitter.util.registry -import org.junit.runner.RunWith -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class SimpleRegistryTest extends RegistryTest { def mkRegistry(): Registry = new SimpleRegistry def name: String = "SimpleRegistry" diff --git a/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala index f8a5cdae61..f6eb548bd0 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala @@ -3,11 +3,8 @@ package com.twitter.util.security import com.twitter.io.TempFile import com.twitter.util.Try import java.io.File -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class PemFileTest extends FunSuite { private[this] val assertLogMessage = diff --git a/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala index 29a1a4acb5..35bb97044e 100644 --- a/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala @@ -4,11 +4,8 @@ import com.twitter.io.TempFile import com.twitter.util.Try import java.io.File import java.security.spec.PKCS8EncodedKeySpec -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class Pkcs8EncodedKeySpecFileTest extends FunSuite { private[this] val assertLogMessage = diff --git a/util-security/src/test/scala/com/twitter/util/security/X500PrincipalInfoTest.scala b/util-security/src/test/scala/com/twitter/util/security/X500PrincipalInfoTest.scala index bebdb2b406..d28100ab05 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X500PrincipalInfoTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X500PrincipalInfoTest.scala @@ -1,11 +1,8 @@ package com.twitter.util.security import javax.security.auth.x500.X500Principal -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class X500PrincipalInfoTest extends FunSuite { test("Empty principal") { diff --git a/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala index 377fb7bd57..d5e0a2849d 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala @@ -4,11 +4,8 @@ import com.twitter.io.TempFile import com.twitter.util.Try import java.io.File import java.security.cert.{CertificateException, X509Certificate} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class X509CertificateFileTest extends FunSuite { private[this] def assertIsCslCert(cert: X509Certificate): Unit = { diff --git a/util-security/src/test/scala/com/twitter/util/security/X509TrustManagerFactoryTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509TrustManagerFactoryTest.scala index 3642c18361..5a3b8819cb 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509TrustManagerFactoryTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509TrustManagerFactoryTest.scala @@ -4,11 +4,8 @@ import com.twitter.io.TempFile import java.io.File import java.security.cert.X509Certificate import javax.net.ssl.{TrustManager, X509TrustManager} -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class X509TrustManagerFactoryTest extends FunSuite { private[this] def loadCertFromResource(resourcePath: String): X509Certificate = { diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala index 7d5ce36ae0..e440bc35ef 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala @@ -2,11 +2,8 @@ package com.twitter.finagle.stats import com.twitter.util.{Future, Await} -import org.junit.runner.RunWith import org.scalatest.{Matchers, FunSuite} -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class BroadcastStatsReceiverTest extends FunSuite with Matchers { test("counter") { diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala index ab5ae3c492..fbe15c44a4 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala @@ -1,10 +1,7 @@ package com.twitter.finagle.stats -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class CategorizingExceptionStatsHandlerTest extends FunSuite { val categorizer = (t: Throwable) => { "clienterrors" } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala index 5c60e415e2..6f3f41d206 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala @@ -1,12 +1,9 @@ package com.twitter.finagle.stats -import org.junit.runner.RunWith import org.scalacheck.{Arbitrary, Gen} import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalatest.prop.GeneratorDrivenPropertyChecks -@RunWith(classOf[JUnitRunner]) class DelegatingStatsReceiverTest extends FunSuite with GeneratorDrivenPropertyChecks { class Ctx { diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index ca767a954e..aa83dc4852 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -2,12 +2,9 @@ package com.twitter.finagle.stats import java.io.{ByteArrayOutputStream, PrintStream} import java.nio.charset.StandardCharsets -import org.junit.runner.RunWith import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class InMemoryStatsReceiverTest extends FunSuite with Eventually with IntegrationPatience { // scalafix:off StoreGaugesAsMemberVariables diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala index 8bc206cdcb..acb386c95e 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala @@ -1,10 +1,7 @@ package com.twitter.finagle.stats -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class MultiCategorizingExceptionStatsHandlerTest extends FunSuite { test("uses label, flags, source, exception chain and rolls up") { val receiver = new InMemoryStatsReceiver diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 79bd3a8421..f1b16e5a0a 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -3,13 +3,10 @@ package com.twitter.finagle.stats import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} import java.util.concurrent.TimeUnit -import org.junit.runner.RunWith import org.mockito.Mockito._ import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import scala.collection.mutable.ArrayBuffer -@RunWith(classOf[JUnitRunner]) class StatsReceiverTest extends FunSuite { test("RollupStatsReceiver counter/stats") { val mem = new InMemoryStatsReceiver diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/WithHistogramDetailsTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/WithHistogramDetailsTest.scala index c730ba8fb0..564065ce03 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/WithHistogramDetailsTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/WithHistogramDetailsTest.scala @@ -1,10 +1,7 @@ package com.twitter.finagle.stats -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class WithHistogramDetailsTest extends FunSuite { test("AggregateWithHistogramDetails rejects empties") { intercept[IllegalArgumentException] { diff --git a/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala b/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala index 1b43cb9c8a..25dfa14343 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala +++ b/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala @@ -1,13 +1,10 @@ package com.twitter.util.testing -import org.junit.runner.RunWith import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar -@RunWith(classOf[JUnitRunner]) class ArgumentCaptureTest extends FunSuite with MockitoSugar with ArgumentCapture { class MockSubject { def method1(arg: String): Long = 0L diff --git a/util-thrift/src/test/scala/com/twitter/util/ThriftCodecTest.scala b/util-thrift/src/test/scala/com/twitter/util/ThriftCodecTest.scala index 4cce8c44b1..c440c4ee16 100644 --- a/util-thrift/src/test/scala/com/twitter/util/ThriftCodecTest.scala +++ b/util-thrift/src/test/scala/com/twitter/util/ThriftCodecTest.scala @@ -1,10 +1,7 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.FunSuite -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class ThriftCodecTest extends FunSuite { private def roundTrip(codec: ThriftCodec[TestThriftStructure, _]): Unit = { diff --git a/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala b/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala index 136a845cf6..4890935641 100644 --- a/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala +++ b/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala @@ -16,11 +16,8 @@ package com.twitter.util -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class ThriftSerializerTest extends WordSpec { val aString = "me gustan los tacos y los burritos" val aNumber = 42 diff --git a/util-zk-test/src/test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala b/util-zk-test/src/test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala index 4aa27f6b81..fe56e51681 100644 --- a/util-zk-test/src/test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala +++ b/util-zk-test/src/test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala @@ -4,11 +4,8 @@ import com.twitter.io.TempDirectory import java.io.File import java.net.InetAddress import org.apache.zookeeper.server.ZooKeeperServer -import org.junit.runner.RunWith import org.scalatest.{BeforeAndAfter, FunSuite} -import org.scalatest.junit.JUnitRunner -@RunWith(classOf[JUnitRunner]) class ServerCnxnFactoryTest extends FunSuite with BeforeAndAfter { val addr = InetAddress.getLocalHost diff --git a/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala b/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala index 6bd7074b54..ca2b439fd9 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala @@ -1,14 +1,11 @@ package com.twitter.zk -import org.junit.runner.RunWith import org.mockito.Mockito._ import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar import com.twitter.util.Future -@RunWith(classOf[JUnitRunner]) class ConnectorTest extends WordSpec with MockitoSugar { "Connector.RoundRobin" should { "require underlying connections" in { diff --git a/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala index 913c70c4b4..1e5decaa57 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala @@ -3,12 +3,9 @@ package com.twitter.zk /** * @author ver@twitter.com */ -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar -@RunWith(classOf[JUnitRunner]) class ZNodeTest extends WordSpec with MockitoSugar { "ZNode" should { class ZNodeSpecHelper { diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index 9d378fbbf1..881707d8be 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -5,21 +5,18 @@ import scala.collection.Set import org.apache.zookeeper._ import org.apache.zookeeper.data.{ACL, Stat} -import org.junit.runner.RunWith import org.mockito.Matchers.{eq => meq, _} import org.mockito.Mockito._ import org.mockito._ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ import com.twitter.logging.{Level, Logger} import com.twitter.util._ -@RunWith(classOf[JUnitRunner]) class ZkClientTest extends WordSpec with MockitoSugar { Logger.get("").setLevel(Level.FATAL) diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala index b929d1366f..ef09ba9211 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala @@ -3,16 +3,13 @@ package com.twitter.zk.coordination import scala.collection.JavaConverters._ import org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE -import org.junit.runner.RunWith import org.scalatest.WordSpec -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, JavaTimer} import com.twitter.zk.{NativeConnector, RetryPolicy, ZkClient} -@RunWith(classOf[JUnitRunner]) class ShardCoordinatorTest extends WordSpec with MockitoSugar { "ShardCoordinator" should { diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala index 712d641c67..14e09af922 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala @@ -7,16 +7,13 @@ import com.twitter.zk.{NativeConnector, RetryPolicy, ZkClient, ZNode} import java.util.concurrent.ConcurrentLinkedQueue import org.apache.zookeeper.KeeperException.NoNodeException import org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE -import org.junit.runner.RunWith import org.scalatest.WordSpec import org.scalatest.concurrent.AsyncAssertions import org.scalatest.concurrent.Eventually._ -import org.scalatest.junit.JUnitRunner import org.scalatest.mockito.MockitoSugar import org.scalatest.time.{Millis, Seconds, Span} import scala.collection.JavaConverters._ -@RunWith(classOf[JUnitRunner]) class ZkAsyncSemaphoreTest extends WordSpec with MockitoSugar with AsyncAssertions { "ZkAsyncSemaphore" should { From ad0ddbf5618d8d26c95f16cadae4a9b995717ec2 Mon Sep 17 00:00:00 2001 From: Jordan Bull Date: Tue, 2 Jul 2019 23:14:07 +0000 Subject: [PATCH 170/796] Coord projects use the coord-review ldap group for reviews JIRA Issues: PUBSUB-35789 Differential Revision: https://phabricator.twitter.biz/D320888 --- util-zk-test/PROJECT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-zk-test/PROJECT b/util-zk-test/PROJECT index aaa4d791f9..a8f458ef04 100644 --- a/util-zk-test/PROJECT +++ b/util-zk-test/PROJECT @@ -1,5 +1,5 @@ owners: - - coord-team:ldap + - coord-review:ldap - banderson - koliver - mnakamura From f2bba6c4b8d7bf4d73929bba163ca3fbedb3992f Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Mon, 8 Jul 2019 23:53:44 +0000 Subject: [PATCH 171/796] util-logging: Remove setting the user.dir property Problem When testing with JDK 11, we found that it is [[ https://bugs.java.com/bugdatabase/view_bug.do?bug_id=JDK-8202127 | no longer possible ]] to set the `user.dir` system property. Solution Remove setting the `user.dir` property in the test and adjust the test so that it is testing the same behavior in a different way. Result The test passes when running with JDK 11. JIRA Issues: CSL-8439 Differential Revision: https://phabricator.twitter.biz/D336895 --- .../com/twitter/logging/FileHandlerTest.scala | 59 ++++++++++--------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index d179d7ef20..36532fe82d 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -17,12 +17,13 @@ package com.twitter.logging import java.io._ -import java.util.{Calendar, Date, logging => javalog} +import java.util.{Calendar, Date, UUID, logging => javalog} import org.scalatest.WordSpec import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import com.twitter.util.Time +import java.nio.file.{Files, Paths} class FileHandlerTest extends WordSpec with TempFolder { def reader(filename: String) = { @@ -33,6 +34,15 @@ class FileHandlerTest extends WordSpec with TempFolder { new OutputStreamWriter(new FileOutputStream(new File(folderName, filename)), "UTF-8") } + // Looks for files that contain `name` in the given `dirToSearch`. + // This method is used in place of setting the `user.dir` + // system property to ensure JDK11 compatibility. See + // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=JDK-8202127 + private def filesContainingName(dirToSearch: String, name: String): Array[String] = + Files + .find(Paths.get(dirToSearch), 1, (p, _) => p.toFile.getName.contains(name)).toArray.map( + _.toString) + "FileHandler" should { val record1 = new javalog.LogRecord(Level.INFO, "first post!") val record2 = new javalog.LogRecord(Level.INFO, "second post") @@ -226,38 +236,33 @@ class FileHandlerTest extends WordSpec with TempFolder { } "correctly handles relative paths" in { - withTempFolder { - // user.dir will be replaced with the temp folder, - // and will be restored when the test is complete - val wdir = System.getProperty("user.dir") - try { - System.setProperty("user.dir", folderName) + val userDir = System.getProperty("user.dir") + val filenamePrefix = UUID.randomUUID().toString - val handler = FileHandler( - filename = "test.log", // Note relative path! - rollPolicy = Policy.Hourly, - append = true, - rotateCount = 2, - formatter = BareFormatter - ).apply() + try { + val handler = FileHandler( + filename = filenamePrefix + ".log", // Note relative path! + rollPolicy = Policy.Hourly, + append = true, + rotateCount = 2, + formatter = BareFormatter + ).apply() - handler.publish(record1) - assert(new File(folderName).list().length == 1) - handler.roll() + handler.publish(record1) + assert(filesContainingName(userDir, filenamePrefix).length == 1) + handler.roll() - handler.publish(record1) - assert(new File(folderName).list().length == 2) - handler.roll() + handler.publish(record1) + assert(filesContainingName(userDir, filenamePrefix).length == 2) + handler.roll() - handler.publish(record1) - assert(new File(folderName).list().length == 2) - handler.close() - } finally { - // restore user.dir to its original configuration - System.setProperty("user.dir", wdir) - } + handler.publish(record1) + assert(filesContainingName(userDir, filenamePrefix).length == 2) + handler.close() + } finally { + filesContainingName(userDir, filenamePrefix).foreach(f => new File(f).delete()) } } From 65c91c3b2c84a8aed670bfb7182a3770976f8294 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 9 Jul 2019 16:58:23 +0000 Subject: [PATCH 172/796] util-logging: Fix FileHandlerTest to work in Scala 2.11 Problem/Solution Scala lambdas don't work out of the box as Java functions in Scala 2.11.x. Since we still support building with 2.11 the `FileHandlerTest` can't use a Scala lambda for calling the Java `java.nio.file.Files#find` function. Instead, we create a `BiPredicate` and use it for matching in the `java.nio.file.Files#find` call. JIRA Issues: CSL-8465 Differential Revision: https://phabricator.twitter.biz/D338397 --- .../com/twitter/logging/FileHandlerTest.scala | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index 36532fe82d..e3937ce697 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -23,25 +23,39 @@ import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import com.twitter.util.Time -import java.nio.file.{Files, Paths} +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.{Files, Path, Paths} +import java.util.function.BiPredicate class FileHandlerTest extends WordSpec with TempFolder { - def reader(filename: String) = { + def reader(filename: String): BufferedReader = { new BufferedReader(new InputStreamReader(new FileInputStream(new File(folderName, filename)))) } - def writer(filename: String) = { + def writer(filename: String):OutputStreamWriter = { new OutputStreamWriter(new FileOutputStream(new File(folderName, filename)), "UTF-8") } + private[this] val matcher = (name: String) => new BiPredicate[Path, BasicFileAttributes] { + override def test( + path: Path, + attributes: BasicFileAttributes + ): Boolean = { + path.toFile.getName.contains(name) + } + } + // Looks for files that contain `name` in the given `dirToSearch`. // This method is used in place of setting the `user.dir` // system property to ensure JDK11 compatibility. See // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=JDK-8202127 private def filesContainingName(dirToSearch: String, name: String): Array[String] = Files - .find(Paths.get(dirToSearch), 1, (p, _) => p.toFile.getName.contains(name)).toArray.map( - _.toString) + .find( + Paths.get(dirToSearch), + 1, + matcher(name) + ).toArray.map(_.toString) "FileHandler" should { val record1 = new javalog.LogRecord(Level.INFO, "first post!") @@ -51,7 +65,7 @@ class FileHandlerTest extends WordSpec with TempFolder { withTempFolder { val f = writer("test.log") f.write("hello!\n") - f.close + f.close() val handler = FileHandler( filename = folderName + "/test.log", @@ -69,7 +83,7 @@ class FileHandlerTest extends WordSpec with TempFolder { withTempFolder { val f = writer("test.log") f.write("hello!\n") - f.close + f.close() val handler = FileHandler( filename = folderName + "/test.log", @@ -131,9 +145,9 @@ class FileHandlerTest extends WordSpec with TempFolder { append = true, formatter = BareFormatter ).apply() - assert(handler.computeNextRollTime(1206769996722L) == Some(1206770400000L)) - assert(handler.computeNextRollTime(1206770400000L) == Some(1206774000000L)) - assert(handler.computeNextRollTime(1206774000001L) == Some(1206777600000L)) + assert(handler.computeNextRollTime(1206769996722L).contains(1206770400000L)) + assert(handler.computeNextRollTime(1206770400000L).contains(1206774000000L)) + assert(handler.computeNextRollTime(1206774000001L).contains(1206777600000L)) } } @@ -145,11 +159,11 @@ class FileHandlerTest extends WordSpec with TempFolder { append = true, formatter = new Formatter(timezone = Some("GMT-7:00")) ).apply() - assert(handler.computeNextRollTime(1250354734000L) == Some(1250406000000L)) - assert(handler.computeNextRollTime(1250404734000L) == Some(1250406000000L)) - assert(handler.computeNextRollTime(1250406001000L) == Some(1251010800000L)) - assert(handler.computeNextRollTime(1250486000000L) == Some(1251010800000L)) - assert(handler.computeNextRollTime(1250496000000L) == Some(1251010800000L)) + assert(handler.computeNextRollTime(1250354734000L).contains(1250406000000L)) + assert(handler.computeNextRollTime(1250404734000L).contains(1250406000000L)) + assert(handler.computeNextRollTime(1250406001000L).contains(1251010800000L)) + assert(handler.computeNextRollTime(1250486000000L).contains(1251010800000L)) + assert(handler.computeNextRollTime(1250496000000L).contains(1251010800000L)) } } } @@ -230,8 +244,8 @@ class FileHandlerTest extends WordSpec with TempFolder { flush() } val fileSet = new File(folderName).list().toSet - assert(fileSet.contains(name) == true) - assert(fileSet.contains(namePrefix) == true) + assert(fileSet.contains(name)) + assert(fileSet.contains(namePrefix)) } } From 64114324d8d0d36725cfddac28e99840077f036f Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Wed, 10 Jul 2019 22:15:57 +0000 Subject: [PATCH 173/796] util: Default to bypassing scheduler in Future.map Problem / Solution Bypassing the scheduler in ConstFuture's map is toggled on (via c.t.u.BypassScheduler) by default. This has been in production for over a quarter with minimal issue. Removing the toggle so that we always bypass the scheduler from now on. JIRA Issues: CSL-7806 Differential Revision: https://phabricator.twitter.biz/D338487 --- CHANGELOG.rst | 5 +++ .../scala/com/twitter/util/ConstFuture.scala | 32 +++---------------- .../main/scala/com/twitter/util/Future.scala | 10 ------ 3 files changed, 9 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0df8fe05aa..7ccaac2d62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -31,6 +31,11 @@ Runtime Behavior Changes currently disabled and can be enabled by a toggle (com.twitter.util.UseLocalInInterruptible) by setting it to 1.0 if you would like to try it out. ``PHAB_ID=D324315`` +* util-core: Remove experimental toggle `com.twitter.util.BypassScheduler` used + for speeding up `ConstFuture.map` (`transformTry`). Now, we always run map + operations immediately instead of via the Scheduler, where they may be queued + and potentially reordered. ``PHAB_ID=D338487`` + 19.5.1 ------ diff --git a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala index c49a6385d4..bc8ddbd568 100644 --- a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala @@ -39,35 +39,11 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { def raise(interrupt: Throwable): Unit = () - protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { - if (Future.BypassScheduler) { - try Future.const(f(result)) - catch { - case scala.util.control.NonFatal(e) => Future.const(Throw(e)) - } - } else { - val p = new Promise[B] - // see the note on `respond` for an explanation of why `Scheduler` is used. - val saved = Local.save() - Scheduler.submit(new Runnable { - def run(): Unit = { - val current = Local.save() - if (current ne saved) - Local.restore(saved) - val computed = try f(result) - catch { - case e: NonLocalReturnControl[_] => Throw(new FutureNonLocalReturnControl(e)) - case scala.util.control.NonFatal(e) => Throw(e) - case t: Throwable => - Monitor.handle(t) - throw t - } finally Local.restore(current) - p.update(computed) - } - }) - p + protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = + try Future.const(f(result)) + catch { + case scala.util.control.NonFatal(e) => Future.const(Throw(e)) } - } def transform[B](f: Try[A] => Future[B]): Future[B] = { val p = new Promise[B] diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 0ac48515f6..bf01f1b55f 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -14,16 +14,6 @@ import scala.util.control.NoStackTrace * on concurrent programming with Futures. */ object Future { - // An experimental property for bypassing the scheduler in Future.map - // operations. - @volatile private[util] var BypassScheduler: Boolean = - System.getProperty("com.twitter.util.BypassScheduler", "false").toBoolean - - // Set to true to bypass the scheduler in Future.map operations. - private[twitter] def bypassScheduler(setting: Boolean): Unit = { - BypassScheduler = setting - } - val DEFAULT_TIMEOUT: Duration = Duration.Top /** A successfully satisfied constant `Unit`-typed `Future` of `()` */ From a405b72cc018b63856cff9b3d65cdc47bb8af99b Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 11 Jul 2019 01:07:22 +0000 Subject: [PATCH 174/796] finagle-http: PayloadSizeFilter now measures streaming messages Problem We used to pass by streaming messages (isChunked) in PayloadSizeFilter. Solution Fork the filter in HTTP and measure chunked messages. Wrap the reader with an onRead callback which measures the payload size. Return a request proxy wrapping the reader we have callback there. JIRA Issues: CSL-8134 Differential Revision: https://phabricator.twitter.biz/D337877 --- util-core/src/main/scala/com/twitter/io/Reader.scala | 1 - util-core/src/test/scala/com/twitter/io/ReaderTest.scala | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index e489f198b2..7dbf07f4ab 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -483,5 +483,4 @@ object Reader { * of the framer. */ def framed(r: Reader[Buf], framer: Buf => Seq[Buf]): Reader[Buf] = new Framed(r, framer) - } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index eaf5d9f392..588c852aed 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -3,7 +3,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.conversions.DurationOps._ import com.twitter.conversions.StorageUnitOps._ -import com.twitter.util.{Await, Awaitable, Future, Promise} +import com.twitter.util.{Await, Awaitable, Future, Promise, Return, Try} import java.io.{ByteArrayInputStream, ByteArrayOutputStream} import java.nio.charset.{StandardCharsets => JChar} import java.util.concurrent.atomic.AtomicBoolean From 28d9de59eea2adeac82691a57c1954dffb9596fa Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 11 Jul 2019 15:42:40 +0000 Subject: [PATCH 175/796] util-logging: Remove Deprecated and Unused LoggerConfig Classes Problem / Solution The classes in LoggerConfig were deprecated in 2012. Let's remove the ones that are no longer in use. JIRA Issues: CSL-8477 Differential Revision: https://phabricator.twitter.biz/D339563 --- CHANGELOG.rst | 4 + .../twitter/logging/config/LoggerConfig.scala | 77 +------------------ 2 files changed, 5 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7ccaac2d62..fbc352a36d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,10 @@ Breaking API Changes `cpuTime`, and `wallTime`. These were deprecated in 2015 and have no replacement. ``PHAB_ID=D330386`` +* util-core: Removed deprecated `com.twitter.logging.config` classes `SyslogFormatterConfig`, + `ThrottledHandlerConfig`, `SyslogHandlerConfig`. These were deprecated in 2012 and have + no replacement. Users are encouraged to use 'util-slf4j-api' where possible. ``PHAB_ID=D339563`` + 19.6.0 ------ diff --git a/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala b/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala index 494b592335..ecc2143c21 100644 --- a/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala +++ b/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala @@ -18,7 +18,7 @@ package com.twitter.logging package config import com.twitter.conversions.DurationOps._ -import com.twitter.util.{Config, Duration, NetUtil} +import com.twitter.util.Config @deprecated("use LoggerFactory", "6.12.1") class LoggerConfig extends Config[Logger] { @@ -111,43 +111,6 @@ object BareFormatterConfig extends FormatterConfig { override def apply() = BareFormatter } -@deprecated("use SyslogFormatter directly", "6.12.1") -class SyslogFormatterConfig extends FormatterConfig { - - /** - * Hostname to prepend to log lines. - */ - var hostname: String = NetUtil.getLocalHostName() - - /** - * Optional server name to insert before log entries. - */ - var serverName: Option[String] = None - - /** - * Use new standard ISO-format timestamps instead of old BSD-format? - */ - var useIsoDateFormat: Boolean = true - - /** - * Priority level in syslog numbers. - */ - var priority: Int = SyslogHandler.PRIORITY_USER - - def serverName_=(name: String): Unit = { serverName = Some(name) } - - override def apply() = - new SyslogFormatter( - hostname, - serverName, - useIsoDateFormat, - priority, - timezone, - truncateAt, - truncateStackTracesAt - ) -} - @deprecated("use Formatter directly", "6.12.1") trait HandlerConfig extends Config[Handler] { var formatter: FormatterConfig = new FormatterConfig @@ -160,28 +123,6 @@ class ConsoleHandlerConfig extends HandlerConfig { def apply() = new ConsoleHandler(formatter(), level) } -@deprecated("use HandlerFactory", "6.12.1") -class ThrottledHandlerConfig extends HandlerConfig { - - /** - * Timespan to consider duplicates. After this amount of time, duplicate entries will be logged - * again. - */ - var duration: Duration = 0.seconds - - /** - * Maximum duplicate log entries to pass before suppressing them. - */ - var maxToDisplay: Int = Int.MaxValue - - /** - * Wrapped handler. - */ - var handler: HandlerConfig = null - - def apply() = new ThrottledHandler(handler(), duration, maxToDisplay) -} - @deprecated("use HandlerFactory", "6.12.1") class QueuingHandlerConfig extends HandlerConfig { def apply() = new QueueingHandler(handler(), maxQueueSize) @@ -223,22 +164,6 @@ class FileHandlerConfig extends HandlerConfig { def apply() = new FileHandler(filename, roll, append, rotateCount, formatter(), level) } -@deprecated("use HandlerFactory", "6.12.1") -class SyslogHandlerConfig extends HandlerConfig { - - /** - * Syslog server hostname. - */ - var server: String = "localhost" - - /** - * Syslog server port. - */ - var port: Int = SyslogHandler.DEFAULT_PORT - - def apply() = new SyslogHandler(server, port, formatter(), level) -} - @deprecated("use HandlerFactory", "6.12.1") class ScribeHandlerConfig extends HandlerConfig { // send a scribe message no more frequently than this: From 20d1a7ce267e7a0ef46c534bf5f32558b0c89e0a Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Wed, 17 Jul 2019 18:47:07 +0000 Subject: [PATCH 176/796] Twitter-oss: Prepare OSS libraries for release 19.7.0 Problem We want to release the next versions of our Twitter OSS libraries 19.7.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D342141 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fbc352a36d..b3e87cf0f1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.7.0 +------ + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 0b2d779b0a..5899c855b2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.6.0" +val collUtils = "com.twitter" %% "util-collection" % "19.7.0" ``` # Units diff --git a/build.sbt b/build.sbt index b33f7eab6f..d8b3976cb3 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.7.0-SNAPSHOT" +val releaseVersion = "19.7.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 9b687e47108c5c68de82617c7156d3be10eedaeb Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Fri, 19 Jul 2019 01:54:55 +0000 Subject: [PATCH 177/796] Twitter-oss: update OSS libraries post-release to 19.8.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries: 19.8.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D342926 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5899c855b2..ae44fffbb3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index d8b3976cb3..4da0d55217 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.7.0" +val releaseVersion = "19.8.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 0ff9a5129edf4efa44107ccd859ce74967954245 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Fri, 19 Jul 2019 16:15:38 +0000 Subject: [PATCH 178/796] util-core: correct version for BypassScheduler changelog entry Problem / Solution BypassScheduler toggle was removed in 19.7.0, not 19.6.0. Differential Revision: https://phabricator.twitter.biz/D343473 --- CHANGELOG.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b3e87cf0f1..8c5e0e7773 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,14 @@ Breaking API Changes `ThrottledHandlerConfig`, `SyslogHandlerConfig`. These were deprecated in 2012 and have no replacement. Users are encouraged to use 'util-slf4j-api' where possible. ``PHAB_ID=D339563`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-core: Remove experimental toggle `com.twitter.util.BypassScheduler` used + for speeding up `ConstFuture.map` (`transformTry`). Now, we always run map + operations immediately instead of via the Scheduler, where they may be queued + and potentially reordered. ``PHAB_ID=D338487`` + 19.6.0 ------ @@ -38,11 +46,6 @@ Runtime Behavior Changes currently disabled and can be enabled by a toggle (com.twitter.util.UseLocalInInterruptible) by setting it to 1.0 if you would like to try it out. ``PHAB_ID=D324315`` -* util-core: Remove experimental toggle `com.twitter.util.BypassScheduler` used - for speeding up `ConstFuture.map` (`transformTry`). Now, we always run map - operations immediately instead of via the Scheduler, where they may be queued - and potentially reordered. ``PHAB_ID=D338487`` - 19.5.1 ------ From 6ab4aeac5e6b77dba8471b1b06ae4160a3feed56 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 24 Jul 2019 19:48:39 +0000 Subject: [PATCH 179/796] util-logging: Remove Level and Policy Forwarders Problem Use of `com.twitter.logging.config` is highly discouraged. Most of the items contained within the namespace were deprecated in 2012, and should be removed. Solution Remove namespace forwarders from `com.twitter.logging.config` for `Level` and `Policy` and encourage folks to use the `com.twitter.logging` versions directly. JIRA Issues: CSL-8539 Differential Revision: https://phabricator.twitter.biz/D344439 --- CHANGELOG.rst | 8 +++++++ .../com/twitter/logging/config/package.scala | 24 ------------------- 2 files changed, 8 insertions(+), 24 deletions(-) delete mode 100644 util-logging/src/main/scala/com/twitter/logging/config/package.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8c5e0e7773..66b8c0675f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,14 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-logging: The namespace forwarders for `Level` and `Policy` in `com.twitter.logging.config` + have been removed. Code should be updated to use `com.twitter.logging.Level` and + `com.twitter.logging.Policy` where necessary. Users are encouraged to use 'util-slf4j-api' though + where possible. ``PHAB_ID=D344439`` + 19.7.0 ------ diff --git a/util-logging/src/main/scala/com/twitter/logging/config/package.scala b/util-logging/src/main/scala/com/twitter/logging/config/package.scala deleted file mode 100644 index a2117d664f..0000000000 --- a/util-logging/src/main/scala/com/twitter/logging/config/package.scala +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2010 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter.logging - -package object config { - type Level = com.twitter.logging.Level - val Level = com.twitter.logging.Level - type Policy = com.twitter.logging.Policy - val Policy = com.twitter.logging.Policy -} From 5d43773eb14c73240d4b9424e880ebcff8ecfe20 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 25 Jul 2019 16:02:30 +0000 Subject: [PATCH 180/796] util-logging: Remove LoggerConfig and Associated Classes Problem Use of `com.twitter.logging.config` is highly discouraged. Most of the items contained within the namespace were deprecated in 2012, and should be removed. Solution Remove `com.twitter.logging.config.LoggerConfig` and everything else in the `com.twitter.logging.config` namespace. JIRA Issues: CSL-8538 Differential Revision: https://phabricator.twitter.biz/D345381 --- CHANGELOG.rst | 5 + .../twitter/logging/config/LoggerConfig.scala | 194 ------------------ 2 files changed, 5 insertions(+), 194 deletions(-) delete mode 100644 util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 66b8c0675f..29d80b2760 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,11 @@ Breaking API Changes `com.twitter.logging.Policy` where necessary. Users are encouraged to use 'util-slf4j-api' though where possible. ``PHAB_ID=D344439`` +* util-logging: The deprecated `com.twitter.logging.config.LoggerConfig` and associated + classes have been removed. These have been deprecated since 2012. Code should be updated + to use `com.twitter.logging.LoggerFactory` where necessary. Users are encouraged to use + 'util-slf4j-api' though where possible. ``PHAB_ID=D345381`` + 19.7.0 ------ diff --git a/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala b/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala deleted file mode 100644 index ecc2143c21..0000000000 --- a/util-logging/src/main/scala/com/twitter/logging/config/LoggerConfig.scala +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright 2010 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter.logging -package config - -import com.twitter.conversions.DurationOps._ -import com.twitter.util.Config - -@deprecated("use LoggerFactory", "6.12.1") -class LoggerConfig extends Config[Logger] { - - /** - * Name of the logging node. The default ("") is the top-level logger. - */ - var node: String = "" - - /** - * Log level for this node. Leaving it None is java's secret signal to use the parent logger's - * level. - */ - var level: Option[Level] = None - - /** - * Where to send log messages. - */ - var handlers: List[HandlerConfig] = Nil - - /** - * Override to have log messages stop at this node. Otherwise they are passed up to parent - * nodes. - */ - var useParents = true - - def apply(): Logger = { - val logger = Logger.get(node) - level.foreach { x => - logger.setLevel(x) - } - handlers.foreach { h => - logger.addHandler(h()) - } - logger.setUseParentHandlers(useParents) - logger - } -} - -@deprecated("use Formatter directly", "6.12.1") -class FormatterConfig extends Config[Formatter] { - - /** - * Should dates in log messages be reported in a different time zone rather than local time? - * If set, the time zone name must be one known by the java `TimeZone` class. - */ - var timezone: Option[String] = None - - /** - * Truncate log messages after N characters. 0 = don't truncate (the default). - */ - var truncateAt: Int = 0 - - /** - * Truncate stack traces in exception logging (line count). - */ - var truncateStackTracesAt: Int = 30 - - /** - * Use full package names like "com.example.thingy" instead of just the toplevel name like - * "thingy"? - */ - var useFullPackageNames: Boolean = false - - /** - * Format for the log-line prefix, if any. - * - * There are two positional format strings (printf-style): the name of the level being logged - * (for example, "ERROR") and the name of the package that's logging (for example, "jobs"). - * - * A string in `<` angle brackets `>` will be used to format the log entry's timestamp, using - * java's `SimpleDateFormat`. - * - * For example, a format string of: - * - * "%.3s [] %s: " - * - * will generate a log line prefix of: - * - * "ERR [20080315-18:39:05.033] jobs: " - */ - var prefix: String = "%.3s [] %s: " - - def apply() = - new Formatter(timezone, truncateAt, truncateStackTracesAt, useFullPackageNames, prefix) -} - -@deprecated("use BareFormatter directly", "6.12.1") -object BareFormatterConfig extends FormatterConfig { - override def apply() = BareFormatter -} - -@deprecated("use Formatter directly", "6.12.1") -trait HandlerConfig extends Config[Handler] { - var formatter: FormatterConfig = new FormatterConfig - - var level: Option[Level] = None -} - -@deprecated("use HandlerFactory", "6.12.1") -class ConsoleHandlerConfig extends HandlerConfig { - def apply() = new ConsoleHandler(formatter(), level) -} - -@deprecated("use HandlerFactory", "6.12.1") -class QueuingHandlerConfig extends HandlerConfig { - def apply() = new QueueingHandler(handler(), maxQueueSize) - - /** - * Maximum queue size. Records are dropped when queue overflows. - */ - var maxQueueSize: Int = Int.MaxValue - - /** - * Wrapped handler. - */ - var handler: HandlerConfig = null -} - -@deprecated("use HandlerFactory", "6.12.1") -class FileHandlerConfig extends HandlerConfig { - - /** - * Filename to log to. - */ - var filename: String = null - - /** - * When to roll the logfile. - */ - var roll: Policy = Policy.Never - - /** - * Append to an existing logfile, or truncate it? - */ - var append: Boolean = true - - /** - * How many rotated logfiles to keep around, maximum. -1 means to keep them all. - */ - var rotateCount: Int = -1 - - def apply() = new FileHandler(filename, roll, append, rotateCount, formatter(), level) -} - -@deprecated("use HandlerFactory", "6.12.1") -class ScribeHandlerConfig extends HandlerConfig { - // send a scribe message no more frequently than this: - var bufferTime = 100.milliseconds - - // don't connect more frequently than this (when the scribe server is down): - var connectBackoff = 15.seconds - - var maxMessagesPerTransaction = 1000 - var maxMessagesToBuffer = 10000 - - var hostname = "localhost" - var port = 1463 - var category = "scala" - - def apply() = - new ScribeHandler( - hostname, - port, - category, - bufferTime, - connectBackoff, - maxMessagesPerTransaction, - maxMessagesToBuffer, - formatter(), - level - ) -} From 0f0bcd44d8a241c8b2e14af568cbe83dec0c7fae Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 25 Jul 2019 18:41:58 +0000 Subject: [PATCH 181/796] Update Jackson to 2.9.9 Problem In order to make util cross building with scala 2.13, we need to move to Jackson 2.9.9 for 2.13 support. Solution Upgrade it. Result As a bonus, Jackson fixes a bug https://github.com/FasterXML/jackson-core/issues/531 when feeding async json parser, remove the `slicedOffset` as now parser records the right position. Differential Revision: https://phabricator.twitter.biz/D345969 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 29d80b2760..5bab8d4e8f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,11 @@ Breaking API Changes to use `com.twitter.logging.LoggerFactory` where necessary. Users are encouraged to use 'util-slf4j-api' though where possible. ``PHAB_ID=D345381`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Upgrade to Jackson 2.9.9. ``PHAB_ID=D345969`` + 19.7.0 ------ diff --git a/build.sbt b/build.sbt index 4da0d55217..c12f41bda3 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll( ExclusionRule("javax.jms", "jms") ) val slf4jVersion = "1.7.21" -val jacksonVersion = "2.9.8" +val jacksonVersion = "2.9.9" val guavaLib = "com.google.guava" % "guava" % "19.0" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.3.4" From f22bfb79ab123a219878e351849371bfbe0828a7 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Mon, 5 Aug 2019 21:51:10 +0000 Subject: [PATCH 182/796] Twitter-oss: Prepare OSS libraries for release 19.8.0 Problem We want to release the next versions of our Twitter OSS libraries 19.8.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D351288 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5bab8d4e8f..6f80429d5b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.8.0 +------ + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index ae44fffbb3..8e16c8ecfd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.7.0" +val collUtils = "com.twitter" %% "util-collection" % "19.8.0" ``` # Units diff --git a/build.sbt b/build.sbt index c12f41bda3..8ee27a5d02 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.8.0-SNAPSHOT" +val releaseVersion = "19.8.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 862e1ed818fa666a04d57715057947501675d5c0 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 6 Aug 2019 01:36:30 +0000 Subject: [PATCH 183/796] util: remove extending URLClassLoader, which is not allowed in JDK 11 Problem JDK 11 no longer allows extending from URLClassLoader for security reasons. Solution Update usage where we extend from URLClassLoader. JIRA Issues: CSL-8445 Differential Revision: https://phabricator.twitter.biz/D351478 --- .../src/test/scala/com/twitter/app/GlobalFlagTest.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala index e76e7ad8b5..2600474766 100644 --- a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala @@ -1,6 +1,5 @@ package com.twitter.app -import java.net.URLClassLoader import org.scalatest.FunSuite object MyGlobalFlag extends GlobalFlag[String]("a test flag", "a global test flag") @@ -65,14 +64,13 @@ class GlobalFlagTest extends FunSuite { } test("GlobalFlag.getAll") { - val mockClassLoader = new MockClassLoader(getClass.getClassLoader.asInstanceOf[URLClassLoader]) + val mockClassLoader = new MockClassLoader(getClass.getClassLoader) val flags = GlobalFlag.getAll(mockClassLoader) assert(flags.length == 4) assert(flags.exists(_.help.equals("a package object test flag"))) } - private class MockClassLoader(realClassLoader: URLClassLoader) - extends URLClassLoader(realClassLoader.getURLs) { + private class MockClassLoader(realClassLoader: ClassLoader) extends ClassLoader(realClassLoader) { private val isValidClassName = (className: String) => List(MyGlobalFlag, MyGlobalBooleanFlag, MyGlobalFlagNoDefault, PackageObjectTest) .map(_.getClass.getName) From 57cda2a3700424999b88d4007a02fa6d1cba2990 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Tue, 6 Aug 2019 17:25:25 +0000 Subject: [PATCH 184/796] Twitter-oss: update OSS libraries post-release to 19.9.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.9.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D351636 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e16c8ecfd..a881747169 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 8ee27a5d02..904c65be3e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.8.0" +val releaseVersion = "19.9.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From d5d20cc337710b07343eb09ecdab005968c97879 Mon Sep 17 00:00:00 2001 From: Martijn Hoekstra Date: Tue, 6 Aug 2019 22:44:17 +0000 Subject: [PATCH 185/796] util: Cross-build with scala 2.13 Problem Aim for (but not always accomplish) source compatibility Solution Align with 2.13.0 collections Signed-off-by: Yufan Gong Differential Revision: https://phabricator.twitter.biz/D333021 --- .travis.yml | 1 + CHANGELOG.rst | 29 +++- CONTRIBUTING.md | 2 +- build.sbt | 33 ++++- project/build.properties | 2 +- project/plugins.sbt | 2 +- .../scala/com/twitter/app/ClassPath.scala | 92 ++++++------ .../scala/com/twitter/app/Flaggable.scala | 6 +- .../main/scala/com/twitter/app/Flags.scala | 4 +- .../scala/com/twitter/app/GlobalFlag.scala | 10 +- .../scala/com/twitter/app/LoadService.scala | 4 +- .../scala/com/twitter/app/ClassPathTest.scala | 2 +- .../com/twitter/app/LoadServiceTest.scala | 21 ++- .../twitter/concurrent/OfferBenchmark.scala | 2 +- .../com/twitter/cache/EvictingCacheTest.scala | 2 +- .../cache/LazilyEvictingCacheTest.scala | 2 +- .../scala/com/twitter/cache/RefreshTest.scala | 2 +- util-core/BUILD | 3 + .../scala/com/twitter/concurrent/Offer.scala | 5 +- .../java/com/twitter/concurrent/Offers.java | 2 +- .../java/com/twitter/concurrent/Spools.java | 3 +- .../main/java/com/twitter/util/Closables.java | 4 +- .../com/twitter/util/HealthyQueue.scala | 42 ++++++ .../com/twitter/util/HealthyQueue.scala | 42 ++++++ .../com/twitter/concurrent/AsyncStream.scala | 21 ++- .../scala/com/twitter/concurrent/Spool.scala | 5 +- .../main/scala/com/twitter/io/Reader.scala | 4 +- .../main/scala/com/twitter/io/SeqReader.scala | 21 +-- .../scala/com/twitter/util/Activity.scala | 25 ++-- .../scala/com/twitter/util/Awaitable.scala | 3 +- .../com/twitter/util/BatchExecutor.scala | 10 +- .../scala/com/twitter/util/BoundedStack.scala | 4 +- .../scala/com/twitter/util/Closable.scala | 1 + .../main/scala/com/twitter/util/Config.scala | 4 +- .../main/scala/com/twitter/util/Diff.scala | 2 +- .../main/scala/com/twitter/util/Event.scala | 15 +- .../main/scala/com/twitter/util/Future.scala | 69 +++++---- .../scala/com/twitter/util/FuturePool.scala | 2 +- .../main/scala/com/twitter/util/Futures.scala | 2 +- .../main/scala/com/twitter/util/Pool.scala | 39 +---- .../scala/com/twitter/util/StorageUnit.scala | 2 +- .../scala/com/twitter/util/Throwables.scala | 9 +- .../main/scala/com/twitter/util/Timer.scala | 2 +- .../src/main/scala/com/twitter/util/Var.scala | 27 ++-- .../concurrent/SpoolCompilationTest.java | 3 +- .../com/twitter/io/BufCompilationTest.java | 5 +- .../concurrent/AsyncSemaphoreTest.scala | 8 +- .../twitter/concurrent/AsyncStreamTest.scala | 4 +- .../com/twitter/concurrent/OfferTest.scala | 3 +- .../com/twitter/concurrent/SpoolTest.scala | 4 +- .../twitter/conversions/PercentOpsTest.scala | 4 +- .../com/twitter/conversions/U64OpsTest.scala | 4 +- .../com/twitter/io/BufByteWriterTest.scala | 4 +- .../scala/com/twitter/io/BufReaderTest.scala | 2 +- .../test/scala/com/twitter/io/BufTest.scala | 11 +- .../scala/com/twitter/io/ByteReaderTest.scala | 10 +- .../test/scala/com/twitter/io/PipeTest.scala | 25 ++-- .../scala/com/twitter/io/ReaderTest.scala | 4 +- .../scala/com/twitter/util/ActivityTest.scala | 1 + .../com/twitter/util/Base64LongTest.scala | 4 +- .../com/twitter/util/CredentialsTest.scala | 5 +- .../scala/com/twitter/util/DiffTest.scala | 4 +- .../scala/com/twitter/util/EventTest.scala | 10 +- .../com/twitter/util/FutureOfferTest.scala | 2 +- .../scala/com/twitter/util/FutureTest.scala | 136 +++++++++++------- .../scala/com/twitter/util/LocalTest.scala | 4 +- .../scala/com/twitter/util/MonitorTest.scala | 2 +- .../scala/com/twitter/util/PoolTest.scala | 3 +- .../scala/com/twitter/util/TimeTest.scala | 4 +- .../scala/com/twitter/util/TimerTest.scala | 2 +- .../test/scala/com/twitter/util/VarTest.scala | 7 +- .../com/twitter/util/WaitQueueTest.scala | 4 +- .../com/twitter/hashing/HashableTest.scala | 4 +- .../hashing/KetamaDistributorTest.scala | 7 +- .../main/scala/com/twitter/jvm/Hotspot.scala | 2 +- .../src/main/scala/com/twitter/jvm/Jvm.scala | 2 +- .../com/twitter/jvm/ContentionTest.scala | 2 +- .../scala/com/twitter/jvm/EstimatorApp.scala | 13 +- .../test/scala/com/twitter/jvm/JvmTest.scala | 6 +- .../twitter/logging/ScribeHandlerTest.scala | 2 +- .../util/registry/GlobalRegistryTest.scala | 1 - .../twitter/util/registry/RosterTest.scala | 2 +- .../util/registry/SimpleRegistryTest.scala | 1 - .../util/security/PemFileTestUtils.scala | 22 ++- .../scala/com/twitter/util/logging/Item.scala | 3 +- .../twitter/util/logging/LoggingTest.scala | 2 +- .../util/logging/MarkerLoggingTest.scala | 2 +- .../finagle/stats/CumulativeGauge.scala | 16 +-- .../finagle/stats/InMemoryStatsReceiver.scala | 10 +- .../twitter/finagle/stats/StatsReceiver.scala | 28 +++- .../stats/StatsReceiverCompilationTest.java | 21 ++- ...ategorizingExceptionStatsHandlerTest.scala | 2 +- .../stats/DelegatingStatsReceiverTest.scala | 4 +- .../stats/InMemoryStatsReceiverTest.scala | 5 +- .../finagle/stats/StatsReceiverTest.scala | 2 +- .../com/twitter/logging/TestLogging.scala | 2 +- .../util/testing/ArgumentCaptureTest.scala | 2 +- .../com/twitter/zk/AsyncCallbackPromise.scala | 4 +- .../com/twitter/zk/NativeConnector.scala | 6 +- .../src/main/scala/com/twitter/zk/ZNode.scala | 4 +- .../main/scala/com/twitter/zk/ZkClient.scala | 2 +- .../zk/coordination/ShardCoordinator.scala | 4 +- .../zk/coordination/ZkAsyncSemaphore.scala | 6 +- .../scala/com/twitter/zk/ConnectorTest.scala | 2 +- .../test/scala/com/twitter/zk/ZNodeTest.scala | 2 +- .../scala/com/twitter/zk/ZkClientTest.scala | 14 +- .../coordination/ShardCoordinatorTest.scala | 4 +- .../coordination/ZkAsyncSemaphoreTest.scala | 4 +- 108 files changed, 637 insertions(+), 441 deletions(-) create mode 100644 util-core/src/main/scala-2.12-/com/twitter/util/HealthyQueue.scala create mode 100644 util-core/src/main/scala-2.13+/com/twitter/util/HealthyQueue.scala diff --git a/.travis.yml b/.travis.yml index 0c8ac66297..f589ffee02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ before_cache: scala: - 2.11.12 - 2.12.8 + - 2.13.0 jdk: - oraclejdk8 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6f80429d5b..2cfa548320 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,12 +7,37 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util: Enables cross-build for 2.13.0. ``PHAB_ID=D333021`` + +Java Compatibility +~~~~~~~~~~~~~~~~~~ + +* util-stats: In `c.t.finagle.stats.AbstractStatsReceiver`, the `counter`, `stat` and + `addGauge` become final, override `counterImpl`, `statImpl` and `addGaugeImpl` instead. + ``PHAB_ID=D333021`` + +* util-core: + `c.t.concurrent.Offer.choose`, + `c.t.concurrent.AsyncStream.apply`, + `c.t.util.Await.all`, + `c.t.util.Closable.sequence` become available to java for passing varargs. ``PHAB_ID=D333021`` + +* util-stats: + `c.t.finagle.stats.StatsReceiver.provideGauge` and `addGauge` become available to java for + passing varags. ``PHAB_ID=D333021`` + 19.8.0 ------ Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ +* util-core: (not breaking) `c.t.util.Future.join` and `c.t.util.Future.collect` now take + `Iterable[Future[A]]` other than Seq. ``PHAB_ID=D333021`` + * util-logging: The namespace forwarders for `Level` and `Policy` in `com.twitter.logging.config` have been removed. Code should be updated to use `com.twitter.logging.Level` and `com.twitter.logging.Policy` where necessary. Users are encouraged to use 'util-slf4j-api' though @@ -29,11 +54,13 @@ Runtime Behavior Changes * util: Upgrade to Jackson 2.9.9. ``PHAB_ID=D345969`` 19.7.0 ------- Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ +* util-core: `com.twitter.util.Event#build` now builds a Seq of events. `Event#buildAny` builds + against any collection of events. ``PHAB_ID=D333021`` + * util-core: Removed deprecated `c.t.concurrent.Scheduler` methods `usrTime`, `cpuTime`, and `wallTime`. These were deprecated in 2015 and have no replacement. ``PHAB_ID=D330386`` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1d6880a9d..21dba985b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,7 +72,7 @@ locally). When appropriate, use [ScalaCheck][scalacheck] to write property-based tests for your code. This will often produce more thorough and effective inputs for your tests. We use ScalaTest's -[GeneratorDrivenPropertyChecks][gendrivenprop] as the entry point for +[ScalaCheckDrivenPropertyChecks][gendrivenprop] as the entry point for writing these tests. ## Compatibility diff --git a/build.sbt b/build.sbt index 904c65be3e..c74caa1dc0 100644 --- a/build.sbt +++ b/build.sbt @@ -17,12 +17,12 @@ val jacksonVersion = "2.9.9" val guavaLib = "com.google.guava" % "guava" % "19.0" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.3.4" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" -val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.13.4" % "test" +val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion val defaultProjectSettings = Seq( scalaVersion := "2.12.8", - crossScalaVersions := Seq("2.11.12", "2.12.8") + crossScalaVersions := Seq("2.11.12", "2.12.8", "2.13.0") ) val baseSettings = Seq( @@ -31,11 +31,22 @@ val baseSettings = Seq( // Workaround for a scaladoc bug which causes it to choke on empty classpaths. unmanagedClasspath in Compile += Attributed.blank(new java.io.File("doesnotexist")), libraryDependencies ++= Seq( + "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.1", // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.10.19" % "test", - "org.scalatest" %% "scalatest" % "3.0.0" % "test" + "org.scalatest" %% "scalatest" % "3.0.8" % "test" ), + fork in Test := true, + // Workaround for cross building HealthyQueue.scala, which is not compatible between + // 2.12- with 2.13+. + unmanagedSourceDirectories in Compile += { + val sourceDir = (sourceDirectory in Compile).value + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, n)) if n >= 13 => sourceDir / "scala-2.13+" + case _ => sourceDir / "scala-2.12-" + } + }, ScoverageKeys.coverageHighlighting := true, resolvers += @@ -48,7 +59,8 @@ val baseSettings = Seq( "-feature", "-encoding", "utf8", // Needs -missing-interpolator due to https://issues.scala-lang.org/browse/SI-8761 - "-Xlint:-missing-interpolator" + "-Xlint:-missing-interpolator", + "-Yrangepos" ), // Note: Use -Xlint rather than -Xlint:unchecked when TestThriftStructure @@ -216,7 +228,7 @@ lazy val utilCore = Project( caffeineLib % "test", scalacheckLib, "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4" + "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2" ), resourceGenerators in Compile += Def.task { val projectName = name.value @@ -288,7 +300,7 @@ lazy val utilIntellij = Project( baseSettings ).settings( name := "util-intellij", - scalaVersion := "2.11.12" + scalaVersion := "2.12.8" ).dependsOn(utilCore % "test") lazy val utilJvm = Project( @@ -368,7 +380,14 @@ lazy val utilStats = Project( sharedSettings ).settings( name := "util-stats", - libraryDependencies ++= Seq(caffeineLib, jsr305Lib, scalacheckLib) + libraryDependencies ++= Seq(caffeineLib, jsr305Lib, scalacheckLib) ++ { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, major)) if major >= 13 => + Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "0.2.0" % "test") + case _ => + Seq() + } +} ).dependsOn(utilCore, utilLint) lazy val utilTest = Project( diff --git a/project/build.properties b/project/build.properties index 23050492c6..c0bab04941 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.1.4 \ No newline at end of file +sbt.version=1.2.8 diff --git a/project/plugins.sbt b/project/plugins.sbt index 51c141d1b1..d83a672ad0 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,5 +3,5 @@ resolvers += Resolver.sonatypeRepo("snapshots") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.0") addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC13") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.0") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.4") diff --git a/util-app/src/main/scala/com/twitter/app/ClassPath.scala b/util-app/src/main/scala/com/twitter/app/ClassPath.scala index cd48546d9a..7f7a254b0d 100644 --- a/util-app/src/main/scala/com/twitter/app/ClassPath.scala +++ b/util-app/src/main/scala/com/twitter/app/ClassPath.scala @@ -6,6 +6,7 @@ import java.net.{URISyntaxException, URLClassLoader, URI} import java.nio.charset.MalformedInputException import java.util.jar.{JarEntry, JarFile} import scala.collection.mutable +import scala.collection.mutable.Builder import scala.collection.JavaConverters._ import scala.io.Source @@ -51,84 +52,84 @@ private[app] sealed abstract class ClassPath[CpInfo <: ClassPath.Info] { protected def ignoredPackages: Set[String] def browse(loader: ClassLoader): Seq[CpInfo] = { - val buf = mutable.Buffer[CpInfo]() + val buf = Vector.newBuilder[CpInfo] val seenUris = mutable.HashSet[URI]() for ((uri, loader) <- getEntries(loader)) { browseUri0(uri, loader, buf, seenUris) seenUris += uri } - buf + buf.result } // package protected for testing private[app] def getEntries(loader: ClassLoader): Seq[(URI, ClassLoader)] = { - val ents = mutable.Buffer[(URI, ClassLoader)]() - val parent = loader.getParent - if (parent != null) - ents ++= getEntries(parent) - - loader match { - case urlLoader: URLClassLoader => - Option(urlLoader.getURLs) match { - case Some(urls) => - urls.foreach { url => - if (url != null) - ents += (url.toURI -> loader) - } - case _ => - } - case _ => - } + val parent = Option(loader.getParent) + + val ownURIs: Vector[(URI, ClassLoader)] = for { + urlLoader <- Vector(loader).collect { case u: URLClassLoader => u } + urls <- Option(urlLoader.getURLs()).toVector + url <- urls if url != null + } yield (url.toURI -> loader) + + val p = parent.toSeq.flatMap(getEntries) + + ownURIs ++ p - ents } // package protected for testing - private[app] def browseUri(uri: URI, loader: ClassLoader, buf: mutable.Buffer[CpInfo]): Unit = + private[app] def browseUri( + uri: URI, + loader: ClassLoader, + buf: Builder[CpInfo, Seq[CpInfo]] + ): Unit = browseUri0(uri, loader, buf, mutable.Set[URI]()) private[this] def browseUri0( uri: URI, loader: ClassLoader, - buf: mutable.Buffer[CpInfo], + buf: Builder[CpInfo, Seq[CpInfo]], history: mutable.Set[URI] ): Unit = { - if (uri.getScheme != "file") - return + if (!history.contains(uri)) { + history.add(uri) + if (uri.getScheme != "file") + return - val f = new File(uri) - if (!(f.exists() && f.canRead)) - return + val f = new File(uri) + if (!(f.exists() && f.canRead)) + return - if (f.isDirectory) - browseDir(f, loader, "", buf) - else - browseJar(f, loader, buf, history) + if (f.isDirectory) + browseDir(f, loader, "", buf) + else + browseJar(f, loader, buf, history) + } } private[this] def browseDir( dir: File, loader: ClassLoader, prefix: String, - buf: mutable.Buffer[CpInfo] + buf: Builder[CpInfo, Seq[CpInfo]] ): Unit = { if (ignoredPackages.contains(prefix)) return for (f <- dir.listFiles) - if (f.isDirectory && f.canRead) + if (f.isDirectory && f.canRead) { browseDir(f, loader, prefix + f.getName + "/", buf) - else + } else processFile(prefix, f, buf) } - protected def processFile(prefix: String, file: File, buf: mutable.Buffer[CpInfo]): Unit + protected def processFile(prefix: String, file: File, buf: Builder[CpInfo, Seq[CpInfo]]): Unit private def browseJar( file: File, loader: ClassLoader, - buf: mutable.Buffer[CpInfo], + buf: Builder[CpInfo, Seq[CpInfo]], seenUris: mutable.Set[URI] ): Unit = { val jarFile = try new JarFile(file) @@ -138,10 +139,7 @@ private[app] sealed abstract class ClassPath[CpInfo <: ClassPath.Info] { try { for (uri <- jarClasspath(file, jarFile.getManifest)) { - if (!seenUris.contains(uri)) { - seenUris += uri - browseUri0(uri, loader, buf, seenUris) - } + browseUri0(uri, loader, buf, seenUris) } for { @@ -161,14 +159,14 @@ private[app] sealed abstract class ClassPath[CpInfo <: ClassPath.Info] { protected def processJarEntry( jarFile: JarFile, entry: JarEntry, - buf: mutable.Buffer[CpInfo] + buf: Builder[CpInfo, Seq[CpInfo]] ): Unit private def jarClasspath(jarFile: File, manifest: java.util.jar.Manifest): Seq[URI] = for { m <- Option(manifest).toSeq attr <- Option(m.getMainAttributes.getValue("Class-Path")).toSeq - el <- attr.split(" ") + el <- attr.split(" ").toSeq uri <- uriFromJarClasspath(jarFile, el) } yield uri @@ -196,7 +194,7 @@ private[app] class FlagClassPath extends ClassPath[ClassPath.FlagInfo] { protected def processFile( prefix: String, file: File, - buf: mutable.Buffer[ClassPath.FlagInfo] + buf: Builder[ClassPath.FlagInfo, Seq[ClassPath.FlagInfo]] ): Unit = { val name = file.getName if (isClass(name)) { @@ -207,7 +205,7 @@ private[app] class FlagClassPath extends ClassPath[ClassPath.FlagInfo] { protected def processJarEntry( jarFile: JarFile, entry: JarEntry, - buf: mutable.Buffer[ClassPath.FlagInfo] + buf: Builder[ClassPath.FlagInfo, Seq[ClassPath.FlagInfo]] ): Unit = { val name = entry.getName if (isClass(name)) { @@ -232,7 +230,7 @@ private[app] class LoadServiceClassPath extends ClassPath[ClassPath.LoadServiceI private[app] def readLines(source: Source): Seq[String] = { try { - source.getLines().toArray.flatMap { line => + source.getLines().toVector.flatMap { line => val commentIdx = line.indexOf('#') val end = if (commentIdx != -1) commentIdx else line.length val str = line.substring(0, end).trim @@ -248,7 +246,7 @@ private[app] class LoadServiceClassPath extends ClassPath[ClassPath.LoadServiceI protected def processFile( prefix: String, file: File, - buf: mutable.Buffer[ClassPath.LoadServiceInfo] + buf: Builder[ClassPath.LoadServiceInfo, Seq[ClassPath.LoadServiceInfo]] ): Unit = { for (iface <- ifaceOfName(prefix + file.getName)) { val source = Source.fromFile(file, "UTF-8") @@ -260,7 +258,7 @@ private[app] class LoadServiceClassPath extends ClassPath[ClassPath.LoadServiceI protected def processJarEntry( jarFile: JarFile, entry: JarEntry, - buf: mutable.Buffer[ClassPath.LoadServiceInfo] + buf: Builder[ClassPath.LoadServiceInfo, Seq[ClassPath.LoadServiceInfo]] ): Unit = { for (iface <- ifaceOfName(entry.getName)) { val source = Source.fromInputStream(jarFile.getInputStream(entry), "UTF-8") diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index 9275b10cc9..6a1c2cb4fd 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -172,14 +172,14 @@ object Flaggable { private[app] class SetFlaggable[T: Flaggable] extends Flaggable[Set[T]] { private val flag = implicitly[Flaggable[T]] assert(flag.default.isEmpty) - override def parse(v: String): Set[T] = v.split(",").map(flag.parse(_)).toSet + override def parse(v: String): Set[T] = v.split(",").iterator.map(flag.parse(_)).toSet override def show(set: Set[T]): String = set.map(flag.show).mkString(",") } private[app] class SeqFlaggable[T: Flaggable] extends Flaggable[Seq[T]] { private val flag = implicitly[Flaggable[T]] assert(flag.default.isEmpty) - def parse(v: String): Seq[T] = v.split(",").map(flag.parse) + def parse(v: String): Seq[T] = v.split(",").toSeq.map(flag.parse) override def show(seq: Seq[T]): String = seq.map(flag.show).mkString(",") } @@ -225,7 +225,7 @@ object Flaggable { implicit def ofJavaList[T: Flaggable]: Flaggable[JList[T]] = new Flaggable[JList[T]] { val seqFlaggable = new SeqFlaggable[T] override def parse(v: String): JList[T] = seqFlaggable.parse(v).asJava - override def show(list: JList[T]): String = seqFlaggable.show(list.asScala) + override def show(list: JList[T]): String = seqFlaggable.show(list.asScala.toSeq) } implicit def ofJavaMap[K: Flaggable, V: Flaggable]: Flaggable[JMap[K, V]] = { diff --git a/util-app/src/main/scala/com/twitter/app/Flags.scala b/util-app/src/main/scala/com/twitter/app/Flags.scala index be748f74ba..3f8a6683d4 100644 --- a/util-app/src/main/scala/com/twitter/app/Flags.scala +++ b/util-app/src/main/scala/com/twitter/app/Flags.scala @@ -196,7 +196,7 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) if (helpFlag()) Help(usage) else - Ok(remaining) + Ok(remaining.toSeq) } /** @@ -379,7 +379,7 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) ): (Iterable[String], Iterable[String]) = { val (set, unset) = getAll(includeGlobal, classLoader).partition { _.get.isDefined } - (set.map { _ + " \\" }, unset.map { _ + " \\" }) + (set.map { _.toString + " \\" }, unset.map { _.toString + " \\" }) } /** diff --git a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala index dfa1bad083..ddd9dae72e 100644 --- a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala +++ b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala @@ -1,7 +1,6 @@ package com.twitter.app import java.lang.reflect.Modifier -import scala.collection.mutable.ArrayBuffer import scala.util.control.NonFatal /** @@ -159,7 +158,7 @@ object GlobalFlag { //we don't hide the real issue that a developer just added an unparseable arg. case e: Throwable => log.log(java.util.logging.Level.SEVERE, "failure reading in flags", e) - new ArrayBuffer[Flag[_]] + Nil } } @@ -172,7 +171,7 @@ object GlobalFlag { className.endsWith("$") && !className.endsWith("package$") val markerClass = classOf[GlobalFlagVisible] - val flags = new ArrayBuffer[Flag[_]] + val flagBuilder = new scala.collection.immutable.VectorBuilder[Flag[_]] // Search for Scala objects annotated with GlobalFlagVisible: val cp = new FlagClassPath() @@ -181,7 +180,8 @@ object GlobalFlag { val cls: Class[_] = Class.forName(info.className, false, loader) if (cls.isAnnotationPresent(markerClass)) { get(info.className) match { - case Some(f) => flags += f + case Some(f) => + flagBuilder += f case None => println("failed for " + info.className) } } @@ -189,6 +189,6 @@ object GlobalFlag { case _: IllegalStateException | _: NoClassDefFoundError | _: ClassNotFoundException => } } - flags + flagBuilder.result() } } diff --git a/util-app/src/main/scala/com/twitter/app/LoadService.scala b/util-app/src/main/scala/com/twitter/app/LoadService.scala index 958395eb1b..4f0b556948 100644 --- a/util-app/src/main/scala/com/twitter/app/LoadService.scala +++ b/util-app/src/main/scala/com/twitter/app/LoadService.scala @@ -41,7 +41,7 @@ object LoadService { * @param implementations cannot be empty. */ def this(iface: Class[T], implementations: java.util.List[T]) = - this(iface, implementations.asScala) + this(iface, implementations.asScala.toSeq) if (implementations.isEmpty) { throw new IllegalArgumentException("implementations cannot be empty") @@ -157,7 +157,7 @@ object LoadService { val cp = new LoadServiceClassPath() val whenAbsent = new JFunction[ClassLoader, Seq[ClassPath.LoadServiceInfo]] { def apply(loader: ClassLoader): Seq[ClassPath.LoadServiceInfo] = { - cp.browse(loader) + cp.browse(loader).toSeq } } diff --git a/util-app/src/test/scala/com/twitter/app/ClassPathTest.scala b/util-app/src/test/scala/com/twitter/app/ClassPathTest.scala index 93984fddb2..a7b525ea15 100644 --- a/util-app/src/test/scala/com/twitter/app/ClassPathTest.scala +++ b/util-app/src/test/scala/com/twitter/app/ClassPathTest.scala @@ -3,7 +3,7 @@ package com.twitter.app import java.net.URLClassLoader import org.mockito.Mockito._ import org.scalatest.FunSuite -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar class ClassPathTest extends FunSuite with MockitoSugar { diff --git a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala index 1ea79207d1..d67cfdd7ea 100644 --- a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala +++ b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala @@ -9,10 +9,10 @@ import java.io.{File, InputStream} import java.net.URL import java.util import java.util.concurrent.{Callable, CountDownLatch, ExecutorService, Executors} -import java.util.{Random, concurrent} +import java.util.concurrent.{Future => JFuture} +import java.util.Random import org.scalatest.FunSuite -import org.scalatest.mockito.MockitoSugar -import scala.collection.mutable +import org.scalatestplus.mockito.MockitoSugar // These traits correspond to files in: // util-app/src/test/resources/META-INF/services @@ -99,7 +99,7 @@ class LoadServiceTest extends FunSuite with MockitoSugar { test("LoadService shouldn't fail on un-readable dir") { val loader = mock[ClassLoader] - val buf = mutable.Buffer.empty[ClassPath.LoadServiceInfo] + val buf = Vector.newBuilder[ClassPath.LoadServiceInfo] val rand = new Random() val f = File.createTempFile("tmp", "__utilapp_loadservice" + rand.nextInt(10000)) @@ -108,14 +108,14 @@ class LoadServiceTest extends FunSuite with MockitoSugar { f.setReadable(false) new LoadServiceClassPath().browseUri(f.toURI, loader, buf) - assert(buf.isEmpty) + assert(buf.result.isEmpty) f.delete() } } test("LoadService shouldn't fail on un-readable sub-dir") { val loader = mock[ClassLoader] - val buf = mutable.Buffer.empty[ClassPath.LoadServiceInfo] + val buf = Vector.newBuilder[ClassPath.LoadServiceInfo] val rand = new Random() val f = File.createTempFile("tmp", "__utilapp_loadservice" + rand.nextInt(10000)) @@ -127,7 +127,7 @@ class LoadServiceTest extends FunSuite with MockitoSugar { subDir.setReadable(false) new LoadServiceClassPath().browseUri(f.toURI, loader, buf) - assert(buf.isEmpty) + assert(buf.result.isEmpty) subDir.delete() f.delete() @@ -140,7 +140,7 @@ class LoadServiceTest extends FunSuite with MockitoSugar { // Run LoadService in a different thread from the custom classloader val clazz: Class[_] = loader.loadClass("com.twitter.app.LoadServiceCallable") val executor: ExecutorService = Executors.newSingleThreadExecutor() - val future: concurrent.Future[Seq[Any]] = + val future: JFuture[Seq[Any]] = executor.submit(clazz.newInstance().asInstanceOf[Callable[Seq[Any]]]) // Get the result @@ -165,7 +165,7 @@ class LoadServiceTest extends FunSuite with MockitoSugar { val jos = new JarOutputStream(new FileOutputStream(jarFile), manifest) jos.close() val loader = mock[ClassLoader] - val buf = mutable.Buffer.empty[ClassPath.LoadServiceInfo] + val buf = Vector.newBuilder[ClassPath.LoadServiceInfo] new LoadServiceClassPath().browseUri(jarFile.toURI, loader, buf) } finally { jarFile.delete @@ -187,7 +187,7 @@ class LoadServiceTest extends FunSuite with MockitoSugar { attributes.put(CLASS_PATH, jar1.getName) new JarOutputStream(new FileOutputStream(jar2), manifest).close() val loader = mock[ClassLoader] - val buf = mutable.Buffer.empty[ClassPath.LoadServiceInfo] + val buf = Vector.newBuilder[ClassPath.LoadServiceInfo] new LoadServiceClassPath().browseUri(jar1.toURI, loader, buf) } finally { jar1.delete @@ -262,7 +262,6 @@ class LoadServiceTest extends FunSuite with MockitoSugar { assert(lsds0.size == 1) assert(lsds0.head.getClass == classOf[LoadServiceDeadlockImpl]) } - } class LoadServiceCallable extends Callable[Seq[Any]] { diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala index 8fcd1871bf..a798fe7f55 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala @@ -56,7 +56,7 @@ object OfferBenchmark { ps(rng.nextInt(numToChooseFrom)).setValue(Tx.const(5)) - ofs + ofs.toSeq } } } diff --git a/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala index e7a0b6498c..a08c598def 100644 --- a/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala @@ -4,7 +4,7 @@ import com.twitter.util.{Promise, Future} import java.util.concurrent.ConcurrentHashMap import org.mockito.Mockito.{verify, never} import org.scalatest.FunSuite -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar class EvictingCacheTest extends FunSuite with MockitoSugar { test("EvictingCache should evict on failed futures for set") { diff --git a/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala index 1ee5271278..e4cf2cf812 100644 --- a/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala @@ -6,7 +6,7 @@ import com.twitter.util.{Await, Future, Promise} import org.mockito.Matchers._ import org.mockito.Mockito.{never, verify, when} import org.scalatest.FunSuite -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar class LazilyEvictingCacheTest extends FunSuite with MockitoSugar { private val explodingCacheLoader = diff --git a/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala b/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala index e092d119ef..aba8a3393d 100644 --- a/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala @@ -4,7 +4,7 @@ import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, Promise, Time} import org.mockito.Mockito._ import org.scalatest._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar class RefreshTest extends FunSuite with MockitoSugar { diff --git a/util-core/BUILD b/util-core/BUILD index 9512034baa..0c435d187b 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -20,6 +20,7 @@ scala_library( name = "util-core-util", sources = globs( "src/main/scala/com/twitter/util/*.scala", + "src/main/scala-2.12-/com/twitter/util/*.scala", "concurrent-extra/src/main/scala/com/twitter/concurrent/*.scala", ), compiler_option_sets = {"fatal_warnings"}, @@ -29,10 +30,12 @@ scala_library( repo = artifactory, ), dependencies = [ + "3rdparty/jvm/org/scala-lang/modules:scala-collection-compat", "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", "util/util-function/src/main/java", ], exports = [ + "3rdparty/jvm/org/scala-lang/modules:scala-collection-compat", "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", "util/util-function/src/main/java", ], diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala index 0ccc91fd73..dfc3db8fa1 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala @@ -2,6 +2,8 @@ package com.twitter.concurrent import com.twitter.util.{Await, Duration, Future, Promise, Time, Timer} import com.twitter.util.Return +import scala.annotation.varargs +import scala.collection.compat.immutable.ArraySeq import scala.util.Random /** @@ -206,6 +208,7 @@ object Offer { * The offer that chooses exactly one of the given offers. If there are any * Offers that are synchronizable immediately, one is chosen at random. */ + @varargs def choose[T](evs: Offer[T]*): Offer[T] = choose(rng, evs) /** @@ -279,7 +282,7 @@ object Offer { if (foundPos >= 0) { updateLosers(foundPos, prepd) } else { - Future.selectIndex(prepd) flatMap { winPos => + Future.selectIndex(ArraySeq.unsafeWrapArray(prepd)) flatMap { winPos => updateLosers(winPos, prepd) } } diff --git a/util-core/src/main/java/com/twitter/concurrent/Offers.java b/util-core/src/main/java/com/twitter/concurrent/Offers.java index 5ba7a04560..8270e6cbb9 100644 --- a/util-core/src/main/java/com/twitter/concurrent/Offers.java +++ b/util-core/src/main/java/com/twitter/concurrent/Offers.java @@ -60,7 +60,7 @@ public static Offer newTimeoutOffer(Duration duration, Timer timer) { * @see Offer$#choose(scala.collection.Seq) */ public static Offer choose(Collection> offers) { - scala.collection.Seq> scalaSeq = JavaConverters.collectionAsScalaIterableConverter(offers).asScala().toSeq(); + scala.collection.immutable.Seq> scalaSeq = JavaConverters.collectionAsScalaIterableConverter(offers).asScala().toVector(); return Offer$.MODULE$.choose(scalaSeq); } diff --git a/util-core/src/main/java/com/twitter/concurrent/Spools.java b/util-core/src/main/java/com/twitter/concurrent/Spools.java index dab7625eaa..b8a34ab030 100644 --- a/util-core/src/main/java/com/twitter/concurrent/Spools.java +++ b/util-core/src/main/java/com/twitter/concurrent/Spools.java @@ -5,6 +5,7 @@ import scala.collection.JavaConverters; import scala.collection.Seq; import scala.collection.immutable.List; +import scala.collection.immutable.Nil$; import java.util.ArrayList; import java.util.Collection; @@ -26,7 +27,7 @@ private Spools() { } */ @SuppressWarnings("unchecked") public static Spool newSpool(Collection elems) { - List buffer = (List)List.empty(); + List buffer = (List)Nil$.MODULE$; for (T item : elems) { buffer = buffer.$colon$colon(item); } diff --git a/util-core/src/main/java/com/twitter/util/Closables.java b/util-core/src/main/java/com/twitter/util/Closables.java index d1e38ce9c2..f1bd814c54 100644 --- a/util-core/src/main/java/com/twitter/util/Closables.java +++ b/util-core/src/main/java/com/twitter/util/Closables.java @@ -50,9 +50,7 @@ public static Closable all(Closable... closables) { * @see com.twitter.util.Closable$#sequence(scala.collection.Seq) */ public static Closable sequence(Closable... closables) { - return Closable$.MODULE$.sequence( - scala.collection.mutable.WrappedArray$.MODULE$.make(closables) - ); + return Closable$.MODULE$.sequence(closables); } /** diff --git a/util-core/src/main/scala-2.12-/com/twitter/util/HealthyQueue.scala b/util-core/src/main/scala-2.12-/com/twitter/util/HealthyQueue.scala new file mode 100644 index 0000000000..f6d8e1d0b2 --- /dev/null +++ b/util-core/src/main/scala-2.12-/com/twitter/util/HealthyQueue.scala @@ -0,0 +1,42 @@ +package com.twitter.util + +import scala.collection.mutable + +private[util] class HealthyQueue[A]( + makeItem: () => Future[A], + numItems: Int, + isHealthy: A => Boolean) + extends mutable.Queue[Future[A]] { + + 0.until(numItems) foreach { _ => + this += makeItem() + } + + override def +=(elem: Future[A]): HealthyQueue.this.type = synchronized { + super.+=(elem) + } + + override def +=:(elem: Future[A]): HealthyQueue.this.type = synchronized { + super.+=:(elem) + } + + override def enqueue(elems: Future[A]*): Unit = synchronized { + super.enqueue(elems: _*) + } + + override def dequeue(): Future[A] = synchronized { + if (isEmpty) throw new NoSuchElementException("queue empty") + + super.dequeue() flatMap { item => + if (isHealthy(item)) { + Future(item) + } else { + val item = makeItem() + synchronized { + super.enqueue(item) + dequeue() + } + } + } + } +} diff --git a/util-core/src/main/scala-2.13+/com/twitter/util/HealthyQueue.scala b/util-core/src/main/scala-2.13+/com/twitter/util/HealthyQueue.scala new file mode 100644 index 0000000000..caef8b13a9 --- /dev/null +++ b/util-core/src/main/scala-2.13+/com/twitter/util/HealthyQueue.scala @@ -0,0 +1,42 @@ +package com.twitter.util + +import scala.collection.mutable + +private[util] class HealthyQueue[A](makeItem: () => Future[A], numItems: Int, isHealthy: A => Boolean) + extends scala.collection.mutable.Queue[Future[A]] { + + 0.until(numItems) foreach { _ => + this += makeItem() + } + + override def addOne(elem: Future[A]): HealthyQueue.this.type = synchronized { + super.addOne(elem) + } + + override def prepend(elem: Future[A]): HealthyQueue.this.type = synchronized { + super.prepend(elem) + } + + override def enqueue(elem: Future[A]) = synchronized { + super.enqueue(elem) + } + override def enqueue(elem1: Future[A], elem2: Future[A], elems: Future[A]*) = synchronized { + super.enqueue(elem1, elem2, elems :_*) + } + + override def dequeue(): Future[A] = synchronized { + if (isEmpty) throw new NoSuchElementException("queue empty") + + super.dequeue() flatMap { item => + if (isHealthy(item)) { + Future(item) + } else { + val item = makeItem() + synchronized { + super.enqueue(item) + dequeue() + } + } + } + } +} \ No newline at end of file diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 3a112597a8..12695a2256 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -2,7 +2,6 @@ package com.twitter.concurrent import com.twitter.util.{Future, Return, Throw, Promise} import scala.annotation.varargs -import scala.collection.mutable /** * A representation of a lazy (and possibly infinite) sequence of asynchronous @@ -463,7 +462,7 @@ sealed abstract class AsyncStream[+A] { * Note: forces the stream. For infinite streams, the future never resolves. */ def observe(): Future[(Seq[A], Option[Throwable])] = { - val buf: mutable.ListBuffer[A] = mutable.ListBuffer.empty + val buf = Vector.newBuilder[A] def go(as: AsyncStream[A]): Future[Unit] = as match { @@ -483,8 +482,8 @@ sealed abstract class AsyncStream[+A] { } go(this).transform { - case Throw(exc) => Future.value(buf.toList -> Some(exc)) - case Return(_) => Future.value(buf.toList -> None) + case Throw(exc) => Future.value(buf.result -> Some(exc)) + case Return(_) => Future.value(buf.result -> None) } } @@ -497,21 +496,22 @@ sealed abstract class AsyncStream[+A] { */ private[concurrent] def buffer(n: Int): Future[(Seq[A], () => AsyncStream[A])] = { // pre-allocate the buffer, unless it's very large - val buffer = new mutable.ArrayBuffer[A](n.min(1024)) + val buffer = Vector.newBuilder[A] + buffer.sizeHint(n.max(0).min(1024)) def fillBuffer( sizeRemaining: Int )(s: => AsyncStream[A] ): Future[(Seq[A], () => AsyncStream[A])] = - if (sizeRemaining < 1) Future.value((buffer, () => s)) + if (sizeRemaining < 1) Future.value((buffer.result, () => s)) else s match { - case Empty => Future.value((buffer, () => s)) + case Empty => Future.value((buffer.result, () => s)) case FromFuture(fa) => fa.flatMap { a => buffer += a - Future.value((buffer, () => empty)) + Future.value((buffer.result, () => empty)) } case Cons(fa, more) => @@ -624,11 +624,8 @@ object AsyncStream { * {{{ * AsyncStream(1,2,3) * }}} - * - * Note: we can't annotate this with varargs because of - * https://issues.scala-lang.org/browse/SI-8743. This seems to be fixed in a - * more recent scala version so we will revisit this soon. */ + @varargs def apply[A](as: A*): AsyncStream[A] = fromSeq(as) /** diff --git a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala index 3f8ada588f..4b85196d4a 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala @@ -1,7 +1,6 @@ package com.twitter.concurrent import com.twitter.util.{Await, Duration, Future, Return, Throw, ConstFuture} -import scala.collection.mutable.ArrayBuffer import scala.collection.mutable import scala.language.implicitConversions import java.io.EOFException @@ -261,11 +260,11 @@ sealed trait Spool[+A] { * satisfied when the entire result is ready. */ def toSeq: Future[Seq[A]] = { - val as = new ArrayBuffer[A] + val as = Vector.newBuilder[A] foreach { a => as += a } map { _ => - as + as.result } } diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 7dbf07f4ab..fd9f650493 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -162,8 +162,8 @@ object Reader { require(chunkSize > 0, s"chunkSize should be > 0 but was $chunkSize") @tailrec - private def loop(acc: Seq[Buf], in: Buf): Seq[Buf] = { - if (in.length < chunkSize) acc :+ in + private def loop(acc: ListBuffer[Buf], in: Buf): Seq[Buf] = { + if (in.length < chunkSize) (acc :+ in).toSeq else { loop( acc :+ in.slice(0, chunkSize), diff --git a/util-core/src/main/scala/com/twitter/io/SeqReader.scala b/util-core/src/main/scala/com/twitter/io/SeqReader.scala index bbfe3b4f09..50407a0b9d 100644 --- a/util-core/src/main/scala/com/twitter/io/SeqReader.scala +++ b/util-core/src/main/scala/com/twitter/io/SeqReader.scala @@ -1,7 +1,7 @@ package com.twitter.io -import com.twitter.io.SeqReader._ import com.twitter.util.{Future, Promise} +import scala.collection.Iterable /** * We want to ensure that this reader always satisfies these invariants: @@ -11,22 +11,23 @@ import com.twitter.util.{Future, Promise} * We achieved this with a state machine where any access to the state is synchronized, * and by preventing changing the state when the reader is fully read or discarded. */ -private[io] final class SeqReader[A](seq: Seq[A]) extends Reader[A] { +private[io] final class SeqReader[A](it: Iterable[A]) extends Reader[A] { + import SeqReader._ + private[this] val closep = Promise[StreamTermination]() - private[this] var value: Seq[A] = seq + private[this] val value: Iterator[A] = synchronized { it.iterator } private[this] var state: State = State.Idle def read(): Future[Option[A]] = { val result = synchronized { state match { case State.Idle => - value match { - case head +: tail => - value = tail - Future.value(Some(head)) - case Nil => - state = State.FullyRead - Future.None + if (value.hasNext) { + val n = value.next() + Future.value(Some(n)) + } else { + state = State.FullyRead + Future.None } case State.FullyRead => Future.None diff --git a/util-core/src/main/scala/com/twitter/util/Activity.scala b/util-core/src/main/scala/com/twitter/util/Activity.scala index 7783af1083..574cc4bbc3 100644 --- a/util-core/src/main/scala/com/twitter/util/Activity.scala +++ b/util-core/src/main/scala/com/twitter/util/Activity.scala @@ -2,12 +2,13 @@ package com.twitter.util import java.util.{List => JList} -import scala.collection.generic.CanBuild import scala.collection.JavaConverters._ import scala.collection.mutable.Buffer import scala.language.higherKinds import scala.reflect.ClassTag import scala.util.control.NonFatal +import scala.Iterable +import scala.collection.compat._ /** * Activity, like [[com.twitter.util.Var Var]], is a value that varies over @@ -183,10 +184,10 @@ object Activity { * * @usecase def collect[T](activities: Coll[Activity[T]]): Activity[Coll[T]] */ - def collect[T: ClassTag, CC[X] <: Traversable[X]]( + def collect[T: ClassTag, CC[X] <: Iterable[X]]( acts: CC[Activity[T]] )( - implicit newBuilder: CanBuild[T, CC[T]] + implicit factory: Factory[T, CC[T]] ): Activity[CC[T]] = { collect(acts, false) } @@ -201,31 +202,31 @@ object Activity { * * @usecase def collectIndependent[T](activities: Coll[Activity[T]]): Activity[Coll[T]] */ - def collectIndependent[T: ClassTag, CC[X] <: Traversable[X]]( + def collectIndependent[T: ClassTag, CC[X] <: Iterable[X]]( acts: CC[Activity[T]] )( - implicit newBuilder: CanBuild[T, CC[T]] + implicit factory: Factory[T, CC[T]] ): Activity[CC[T]] = { collect(acts, true) } - private[this] def collect[T: ClassTag, CC[X] <: Traversable[X]]( + private[this] def collect[T: ClassTag, CC[X] <: Iterable[X]]( acts: CC[Activity[T]], collectIndependent: Boolean )( - implicit newBuilder: CanBuild[T, CC[T]] + implicit factory: Factory[T, CC[T]] ): Activity[CC[T]] = { if (acts.isEmpty) - return Activity.value(newBuilder().result) + return Activity.value(factory.fromSpecific(Nil)) - val states: Traversable[Var[State[T]]] = acts.map(_.run) - val stateVar: Var[Traversable[State[T]]] = if (collectIndependent) { + val states: Iterable[Var[State[T]]] = acts.map(_.run) + val stateVar: Var[Iterable[State[T]]] = if (collectIndependent) { Var.collectIndependent(states) } else { Var.collect(states) } - def flip(states: Traversable[State[T]]): State[CC[T]] = { + def flip(states: Iterable[State[T]]): State[CC[T]] = { val notOk = states find { case Pending | Failed(_) => true case Ok(_) => false @@ -238,7 +239,7 @@ object Activity { case Some(_) => assert(false) } - val ts = newBuilder() + val ts = factory.newBuilder states foreach { case Ok(t) => ts += t case _ => assert(false) diff --git a/util-core/src/main/scala/com/twitter/util/Awaitable.scala b/util-core/src/main/scala/com/twitter/util/Awaitable.scala index 48a5dffb85..70aa9027ad 100644 --- a/util-core/src/main/scala/com/twitter/util/Awaitable.scala +++ b/util-core/src/main/scala/com/twitter/util/Awaitable.scala @@ -2,6 +2,7 @@ package com.twitter.util import com.twitter.concurrent.Scheduler import java.util.concurrent.atomic.AtomicBoolean +import scala.annotation.varargs import scala.collection.JavaConverters._ import scala.util.control.{NonFatal => NF} @@ -160,7 +161,7 @@ object Await { /** $all */ @throws(classOf[TimeoutException]) @throws(classOf[InterruptedException]) - @scala.annotation.varargs + @varargs def all(awaitables: Awaitable[_]*): Unit = all(awaitables, Duration.Top) diff --git a/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala b/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala index 601781523a..26e5c2deaf 100644 --- a/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala +++ b/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala @@ -3,7 +3,7 @@ package com.twitter.util import java.util.concurrent.CancellationException import java.util.logging.Logger -import scala.collection.mutable +import scala.collection.{mutable, Iterable} import scala.collection.mutable.ArrayBuffer /** @@ -30,7 +30,7 @@ private[util] class BatchExecutor[In, Out]( sizeThreshold: Int, timeThreshold: Duration = Duration.Top, sizePercentile: => Float = 1.0f, - f: Seq[In] => Future[Seq[Out]] + f: Iterable[In] => Future[Seq[Out]] )( implicit timer: Timer) extends Function1[In, Future[Out]] { batcher => @@ -118,7 +118,7 @@ private[util] class BatchExecutor[In, Out]( def flushBatch(): () => Unit = { // this must be executed within a `synchronized` block. val prevBatch = new mutable.ArrayBuffer[(In, Promise[Out])](buf.length) - buf.copyToBuffer(prevBatch) + prevBatch ++= buf buf.clear() scheduled foreach { _.cancel() } @@ -134,8 +134,8 @@ private[util] class BatchExecutor[In, Out]( } } - def executeBatch(batch: Seq[(In, Promise[Out])]): Unit = { - val uncancelled = batch filter { + def executeBatch(batch: Iterable[(In, Promise[Out])]): Unit = { + val uncancelled = batch.filter { case (in, p) => p.isInterrupted match { case Some(_cause) => diff --git a/util-core/src/main/scala/com/twitter/util/BoundedStack.scala b/util-core/src/main/scala/com/twitter/util/BoundedStack.scala index ba309144a1..5d4a9b0fed 100644 --- a/util-core/src/main/scala/com/twitter/util/BoundedStack.scala +++ b/util-core/src/main/scala/com/twitter/util/BoundedStack.scala @@ -1,6 +1,7 @@ package com.twitter.util import scala.reflect.ClassTag +import scala.collection.Seq /** * A "stack" with a bounded size. If you push a new element on the top @@ -11,8 +12,7 @@ class BoundedStack[A: ClassTag](val maxSize: Int) extends Seq[A] { private var top = 0 private var count_ = 0 - def length: Int = count_ - override def size: Int = count_ + override def length: Int = count_ def clear(): Unit = { top = 0 diff --git a/util-core/src/main/scala/com/twitter/util/Closable.scala b/util-core/src/main/scala/com/twitter/util/Closable.scala index f30734ccdb..cd8ade8fe2 100644 --- a/util-core/src/main/scala/com/twitter/util/Closable.scala +++ b/util-core/src/main/scala/com/twitter/util/Closable.scala @@ -103,6 +103,7 @@ object Closable { * @note as with all `Closables`, the `deadline` passed to `close` * is advisory. */ + @varargs def sequence(closables: Closable*): Closable = new Closable { private final def closeSeq( deadline: Time, diff --git a/util-core/src/main/scala/com/twitter/util/Config.scala b/util-core/src/main/scala/com/twitter/util/Config.scala index e9d2a888bb..dfdd2b9b80 100644 --- a/util-core/src/main/scala/com/twitter/util/Config.scala +++ b/util-core/src/main/scala/com/twitter/util/Config.scala @@ -140,10 +140,10 @@ trait Config[T] extends (() => T) { val syntheticTermNames: Set[String] = { val configSymbol = mirror.reflect(config).symbol val configMembers = configSymbol.toType.members - configMembers.collect { + configMembers.iterator.collect { case symbol if symbol.isTerm && symbol.isSynthetic => symbol.name.decodedName.toString - }(scala.collection.breakOut) + }.toSet } for (method <- nullaryMethods) { val name = method.getName diff --git a/util-core/src/main/scala/com/twitter/util/Diff.scala b/util-core/src/main/scala/com/twitter/util/Diff.scala index 8090a40a81..285fba458a 100644 --- a/util-core/src/main/scala/com/twitter/util/Diff.scala +++ b/util-core/src/main/scala/com/twitter/util/Diff.scala @@ -86,7 +86,7 @@ object Diffable { } def map[U](f: T => U): SeqDiff[U] = - SeqDiff(limit, insert.mapValues(f)) + SeqDiff(limit, insert.map { case (k, v) => (k, f(v)) }) } implicit val ofSet: Diffable[Set] = new Diffable[Set] { diff --git a/util-core/src/main/scala/com/twitter/util/Event.scala b/util-core/src/main/scala/com/twitter/util/Event.scala index 9f6890c527..4dda24b9ec 100644 --- a/util-core/src/main/scala/com/twitter/util/Event.scala +++ b/util-core/src/main/scala/com/twitter/util/Event.scala @@ -3,7 +3,7 @@ package com.twitter.util import java.lang.ref.WeakReference import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import scala.annotation.tailrec -import scala.collection.generic.CanBuild +import scala.collection.compat._ import scala.collection.immutable.Queue import scala.language.higherKinds @@ -304,13 +304,13 @@ trait Event[+T] { self => } /** - * Progressively build a collection of events using the passed-in + * Progressively build any collection of events using the passed-in * builder. A value containing the current version of the collection * is notified for each incoming event. */ - def build[U >: T, That](implicit cbf: CanBuild[U, That]): Event[That] = new Event[That] { + def buildAny[That](implicit factory: Factory[T, That]): Event[That] = new Event[That] { def register(s: Witness[That]): Closable = { - val b = cbf() + val b = factory.newBuilder self.respond { t => b += t s.notify(b.result()) @@ -318,6 +318,13 @@ trait Event[+T] { self => } } + /** + * Progressively build a Seq of events using the passed-in + * builder. A value containing the current version of the collection + * is notified for each incoming event. + */ + def build[U >: T, That <: Seq[U]](implicit factory: Factory[U, That]): Event[That] = buildAny(factory) + /** * A Future which is satisfied by the first value observed. */ diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index bf01f1b55f..4b2a159872 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -4,7 +4,7 @@ import com.twitter.concurrent.{Offer, Tx} import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicReference} import java.util.concurrent.{CancellationException, TimeUnit, Future => JavaFuture} import java.util.{List => JList} -import scala.collection.mutable +import scala.collection.compat.immutable.ArraySeq import scala.runtime.NonLocalReturnControl import scala.util.control.NoStackTrace @@ -227,7 +227,7 @@ object Future { // but because these are both Function1 of different types, this cannot be done. Because // the respond handler is invoked for each Future in the sequence, and the interrupt handler is // only set once, we prefer to mix in Function1. - private[this] class JoinPromise[A](fs: Seq[Future[A]], size: Int) + private[this] class JoinPromise[A](fs: Iterable[Future[A]], size: Int) extends Promise[Unit] with (Try[A] => Unit) { @@ -263,7 +263,7 @@ object Future { * @see [[collectToTry]] if you want to be able to see the results of each * `Future` regardless of if they succeed or fail. */ - def join[A](fs: Seq[Future[A]]): Future[Unit] = { + def join[A](fs: Iterable[Future[A]]): Future[Unit] = { if (fs.isEmpty) Future.Unit else { val size = fs.size @@ -297,7 +297,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( meths foreach println ' - */ + """*/ /** * Join 2 futures. The returned future is complete when all @@ -1065,12 +1065,16 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( } yield results :+ nextResult } - private[this] final class CollectPromise[A](fs: Seq[Future[A]]) + private[this] final class CollectPromise[A](fs: Iterable[Future[A]]) extends Promise[Seq[A]] with Promise.InterruptHandler { - private[this] val results = new mutable.ArraySeq[A](fs.size) - private[this] val count = new AtomicInteger(results.size) + private[this] val results = Array.ofDim[Any](fs.size) + private[this] def setResultsAsValue() = { + val seq = ArraySeq.unsafeWrapArray(results).asInstanceOf[ArraySeq[A]] + setValue(seq) + } + private[this] val count = new AtomicInteger(fs.size) // Respond handler. It's safe to write into different array cells concurrently. // We guarantee the thread writing a last value will observe all previous writes @@ -1079,7 +1083,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( def collectTo(index: Int): Try[A] => Unit = { case Return(a) => results(index) = a - if (count.decrementAndGet() == 0) setValue(results) + if (count.decrementAndGet() == 0) setResultsAsValue() case t @ Throw(_) => updateIfEmpty(t.cast[Seq[A]]) } @@ -1105,7 +1109,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * @see [[join]] if you are not interested in the results of the individual * `Futures`, only when they are complete. */ - def collect[A](fs: Seq[Future[A]]): Future[Seq[A]] = + def collect[A](fs: Iterable[Future[A]]): Future[Seq[A]] = if (fs.isEmpty) emptySeq else { val result = new CollectPromise[A](fs) @@ -1133,7 +1137,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( else { val (keys, values) = fs.toSeq.unzip Future.collect(values).map { seq => - keys.zip(seq)(scala.collection.breakOut): Map[A, B] + keys.iterator.zip(seq.iterator).toMap: Map[A, B] } } @@ -1158,13 +1162,24 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * @param fs a sequence of Futures * @return a `Future[Seq[Try[A]]]` containing the collected values from fs. */ - def collectToTry[A](fs: Seq[Future[A]]): Future[Seq[Try[A]]] = - Future.collect { - val seq = new mutable.ArrayBuffer[Future[Try[A]]](fs.size) + def collectToTry[A](fs: Iterable[Future[A]]): Future[Seq[Try[A]]] = { + //unroll cases 0 and 1 + if (fs.isEmpty) Nil + else { val iterator = fs.iterator - while (iterator.hasNext) seq += iterator.next().liftToTry - seq + val h = iterator.next() + if (iterator.hasNext) { + val buf = Vector.newBuilder[Future[Try[A]]] + buf.sizeHint(fs.size) + buf += h.liftToTry + buf += iterator.next().liftToTry + while (iterator.hasNext) buf += iterator.next().liftToTry + Future.collect(buf.result) + } else { + Future.collect(List(h.liftToTry)) + } } + } /** * Collect the results from the given futures into a new future of java.util.List[Try[A]]. @@ -1211,15 +1226,16 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( a.respond { t => if (!p.isDefined) { val filtered = { - val seq = new mutable.ArrayBuffer[Future[A]](size - 1) + val buf = Vector.newBuilder[Future[A]] + buf.sizeHint(size - 1) var j = 0 while (j < size) { val (_, fi) = as(j) if (fi ne f) - seq += fi + buf += fi j += 1 } - seq + buf.result() } p.updateIfEmpty(Return(t -> filtered)) @@ -1348,17 +1364,18 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( def parallel[A](n: Int)(f: => Future[A]): Seq[Future[A]] = { var i = 0 - val result = new mutable.ArrayBuffer[Future[A]](n) + val buf = Vector.newBuilder[Future[A]] + buf.sizeHint(n) while (i < n) { - result += f + buf += f i += 1 } - result + buf.result } /** * Creates a "batched" Future that, given a function - * `Seq[In] => Future[Seq[Out]]`, returns a `In => Future[Out]` interface + * `scala.collection.Seq[In] => Future[Seq[Out]]`, returns a `In => Future[Out]` interface * that batches the underlying asynchronous operations. Thus, one can * incrementally submit tasks to be performed when the criteria for batch * flushing is met. @@ -1394,12 +1411,16 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( sizeThreshold: Int, timeThreshold: Duration = Duration.Top, sizePercentile: => Float = 1.0f - )(f: Seq[In] => Future[Seq[Out]] + )(f: scala.collection.Seq[In] => Future[Seq[Out]] )( implicit timer: Timer ): Batcher[In, Out] = { new Batcher[In, Out]( - new BatchExecutor[In, Out](sizeThreshold, timeThreshold, sizePercentile, f) + new BatchExecutor[In, Out]( + sizeThreshold, + timeThreshold, + sizePercentile, + f.compose(it => it.toSeq)) ) } } diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index d06a8717d3..b14a419c9a 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -2,7 +2,7 @@ package com.twitter.util import com.twitter.concurrent.NamedPoolThreadFactory import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent._ +import java.util.concurrent.{Future => _, _} import scala.runtime.NonLocalReturnControl /** diff --git a/util-core/src/main/scala/com/twitter/util/Futures.scala b/util-core/src/main/scala/com/twitter/util/Futures.scala index 741f92f423..b7719db384 100644 --- a/util-core/src/main/scala/com/twitter/util/Futures.scala +++ b/util-core/src/main/scala/com/twitter/util/Futures.scala @@ -776,7 +776,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * to be satisfied and the rest of the futures. */ def select[A](fs: JList[Future[A]]): Future[(Try[A], JList[Future[A]])] = - Future.select(fs.asScala).map { + Future.select(fs.asScala.toSeq).map { case (first, rest) => (first, rest.asJava) } diff --git a/util-core/src/main/scala/com/twitter/util/Pool.scala b/util-core/src/main/scala/com/twitter/util/Pool.scala index 86e2a36a80..7ee7ec87bb 100644 --- a/util-core/src/main/scala/com/twitter/util/Pool.scala +++ b/util-core/src/main/scala/com/twitter/util/Pool.scala @@ -1,6 +1,7 @@ package com.twitter.util import scala.collection.mutable +import scala.collection.Iterable trait Pool[A] { def reserve(): Future[A] @@ -8,7 +9,7 @@ trait Pool[A] { } class SimplePool[A](items: mutable.Queue[Future[A]]) extends Pool[A] { - def this(initialItems: Seq[A]) = this { + def this(initialItems: Iterable[A]) = this { val queue = new mutable.Queue[Future[A]] queue ++= initialItems.map(Future(_)) queue @@ -53,39 +54,3 @@ abstract class FactoryPool[A](numItems: Int) extends Pool[A] { protected def makeItem(): Future[A] protected def isHealthy(a: A): Boolean } - -private class HealthyQueue[A](makeItem: () => Future[A], numItems: Int, isHealthy: A => Boolean) - extends mutable.Queue[Future[A]] { - - 0.until(numItems) foreach { _ => - this += makeItem() - } - - override def +=(elem: Future[A]): HealthyQueue.this.type = synchronized { - super.+=(elem) - } - - override def +=:(elem: Future[A]): HealthyQueue.this.type = synchronized { - super.+=:(elem) - } - - override def enqueue(elems: Future[A]*): Unit = synchronized { - super.enqueue(elems: _*) - } - - override def dequeue(): Future[A] = synchronized { - if (isEmpty) throw new NoSuchElementException("queue empty") - - super.dequeue() flatMap { item => - if (isHealthy(item)) { - Future(item) - } else { - val item = makeItem() - synchronized { - enqueue(item) - dequeue() - } - } - } - } -} diff --git a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala index 58f51b18e9..90d0ac2d50 100644 --- a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala +++ b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala @@ -126,7 +126,7 @@ class StorageUnit(val bytes: Long) extends Ordered[StorageUnit] { def max(other: StorageUnit): StorageUnit = if (this > other) this else other - override def toString(): String = inBytes + ".bytes" + override def toString(): String = s"$inBytes.bytes" def toHuman(): String = { val prefix = "KMGTPE" diff --git a/util-core/src/main/scala/com/twitter/util/Throwables.scala b/util-core/src/main/scala/com/twitter/util/Throwables.scala index 67bdf8f010..7e33824745 100644 --- a/util-core/src/main/scala/com/twitter/util/Throwables.scala +++ b/util-core/src/main/scala/com/twitter/util/Throwables.scala @@ -1,7 +1,6 @@ package com.twitter.util import scala.annotation.tailrec -import scala.collection.mutable.ArrayBuffer object Throwables { @@ -27,14 +26,14 @@ object Throwables { * classname strings. */ def mkString(ex: Throwable): Seq[String] = { - @tailrec def rec(ex: Throwable, buf: ArrayBuffer[String]): Seq[String] = { + @tailrec def rec(ex: Throwable, buf: List[String]): Seq[String] = { if (ex eq null) - buf.result + buf.reverse else - rec(ex.getCause, buf += ex.getClass.getName) + rec(ex.getCause, ex.getClass.getName :: buf) } - rec(ex, ArrayBuffer.empty) + rec(ex, Nil) } object RootCause { diff --git a/util-core/src/main/scala/com/twitter/util/Timer.scala b/util-core/src/main/scala/com/twitter/util/Timer.scala index 124f1af3fc..6731d7eb18 100644 --- a/util-core/src/main/scala/com/twitter/util/Timer.scala +++ b/util-core/src/main/scala/com/twitter/util/Timer.scala @@ -1,7 +1,7 @@ package com.twitter.util import com.twitter.concurrent.NamedPoolThreadFactory -import java.util.concurrent._ +import java.util.concurrent.{Future => _, _} import scala.collection.mutable.ArrayBuffer import scala.util.control diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index c1db345a08..062edd013e 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -3,12 +3,13 @@ package com.twitter.util import java.util.concurrent.atomic.{AtomicLong, AtomicReference, AtomicReferenceArray} import java.util.{List => JList} import scala.annotation.tailrec +import scala.collection.compat._ import scala.collection.JavaConverters._ -import scala.collection.generic.CanBuildFrom import scala.collection.immutable import scala.collection.mutable.Buffer import scala.language.higherKinds import scala.reflect.ClassTag +import scala.Iterable /** * Vars are values that vary over time. To create one, you must give it an @@ -296,10 +297,10 @@ object Var { * // refCollectIndependent == Vector((1,2), (2,2), (2,4)) * }}} */ - def collect[T: ClassTag, CC[X] <: Traversable[X]]( + def collect[T: ClassTag, CC[X] <: Iterable[X]]( vars: CC[Var[T]] )( - implicit newBuilder: CanBuildFrom[CC[T], T, CC[T]] + implicit factory: Factory[T, CC[T]] ): Var[CC[T]] = { val vs = vars.toArray @@ -316,7 +317,7 @@ object Var { } tree(0, vs.length).map { ts => - val b = newBuilder() + val b = factory.newBuilder b ++= ts b.result() } @@ -344,17 +345,17 @@ object Var { * // refCollectIndependent == Vector((1,2), (2,2), (2,4)) * }}} */ - def collectIndependent[T: ClassTag, CC[X] <: Traversable[X]]( + def collectIndependent[T: ClassTag, CC[X] <: Iterable[X]]( vars: CC[Var[T]] )( - implicit newBuilder: CanBuildFrom[CC[T], T, CC[T]] + implicit factory: Factory[T, CC[T]] ): Var[CC[T]] = - async(newBuilder().result()) { v => + async(factory.newBuilder.result()) { v => val N = vars.size val cur = new AtomicReferenceArray[T](N) @volatile var filling = true def build() = { - val b = newBuilder() + val b = factory.newBuilder var i = 0 while (i < N) { b += cur.get(i) @@ -368,13 +369,9 @@ object Var { if (!filling) v() = build() } - val closes = new Array[Closable](N) - var i = 0 - for (v <- vars) { - val j = i - closes(j) = v.observe(0, Observer(newValue => publish(j, newValue))) - i += 1 - } + val closes = vars.iterator.zipWithIndex.map { + case (v, i) => v.observe(0, Observer(newValue => publish(i, newValue))) + }.toSeq filling = false v() = build() diff --git a/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java b/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java index c75c515b25..2b89e1b9d6 100644 --- a/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java +++ b/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java @@ -25,8 +25,7 @@ public Future> tail() { public String head() { return "spool"; } - } - + } @Test public void testOwnSpool() { Spool a = new OwnSpool(); diff --git a/util-core/src/test/java/com/twitter/io/BufCompilationTest.java b/util-core/src/test/java/com/twitter/io/BufCompilationTest.java index d1e98587c8..386ae74b9c 100644 --- a/util-core/src/test/java/com/twitter/io/BufCompilationTest.java +++ b/util-core/src/test/java/com/twitter/io/BufCompilationTest.java @@ -3,7 +3,7 @@ import org.junit.Assert; import org.junit.Test; import scala.Option; -import scala.collection.JavaConversions; +import scala.collection.JavaConverters; import java.nio.ByteBuffer; import java.util.Arrays; @@ -55,7 +55,8 @@ public void testBufApply() { Buf abc = Bufs.utf8Buf("abc"); Buf def = Bufs.utf8Buf("def"); Buf merged = Buf.apply( - JavaConversions.asScalaBuffer(Arrays.asList(empty, abc, def))); + JavaConverters.asScalaBufferConverter(Arrays.asList(empty, abc, def)).asScala() + ); } @Test diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index 41310bc64f..cadbc2cf1f 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -268,11 +268,11 @@ class AsyncSemaphoreTest extends FunSpec { assert(counter == 2) assert(semHelper.sem.numPermitsAvailable == 0) - queue.dequeue().setValue(Unit) + queue.dequeue().setValue(()) assert(counter == 3) assert(semHelper.sem.numPermitsAvailable == 0) - queue.dequeue().setValue(Unit) + queue.dequeue().setValue(()) assert(semHelper.sem.numPermitsAvailable == 1) queue.dequeue().setException(new RuntimeException("test")) @@ -320,7 +320,7 @@ class AsyncSemaphoreTest extends FunSpec { // sync func is blocked at this point. // But it should be executed as soon as one of the queued up future functions finish - queue.dequeue().setValue(Unit) + queue.dequeue().setValue(()) assert(counter == 3) val result = Await.result(future) assert(result == 3) @@ -359,7 +359,7 @@ class AsyncSemaphoreTest extends FunSpec { // sync func is blocked at this point. // But it should be executed as soon as one of the queued up future functions finish - queue.dequeue().setValue(Unit) + queue.dequeue().setValue(()) assert(counter == 2) assert(Try(Await.result(future)).isThrow) assert(semHelper.sem.numPermitsAvailable == 1) diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 01d56a1f7b..8d3395069d 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -4,9 +4,9 @@ import com.twitter.conversions.DurationOps._ import com.twitter.util._ import org.scalacheck.{Arbitrary, Gen} import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class AsyncStreamTest extends FunSuite with GeneratorDrivenPropertyChecks { +class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { import AsyncStream.{mk, of} import AsyncStreamTest._ diff --git a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala index 0638d86c4f..00c4096c8e 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala @@ -4,8 +4,7 @@ import scala.util.Random import org.mockito.Mockito._ import org.scalatest.WordSpec -import org.scalatest.mockito.MockitoSugar - +import org.scalatestplus.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, MockTimer, Promise, Return, Time} diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala index 35d9dfeb71..eff045d682 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala @@ -6,10 +6,10 @@ import com.twitter.util.{Await, Future, Promise, Return, Throw} import java.io.EOFException import org.scalacheck.Arbitrary import org.scalatest.WordSpec -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable.ArrayBuffer -class SpoolTest extends WordSpec with GeneratorDrivenPropertyChecks { +class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { "Empty Spool" should { val s = Spool.empty[Int] diff --git a/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala index 68f71b9163..118d4d3829 100644 --- a/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala @@ -4,9 +4,9 @@ import com.twitter.conversions.PercentOps._ import org.scalacheck.Arbitrary.arbitrary import org.scalacheck.Gen import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class PercentOpsTest extends FunSuite with GeneratorDrivenPropertyChecks { +class PercentOpsTest extends FunSuite with ScalaCheckDrivenPropertyChecks { private[this] val Precision = 0.0000000001 diff --git a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala index 32736a0b38..db43061cbc 100644 --- a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala @@ -1,9 +1,9 @@ package com.twitter.conversions import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class U64OpsTest extends FunSuite with GeneratorDrivenPropertyChecks { +class U64OpsTest extends FunSuite with ScalaCheckDrivenPropertyChecks { import com.twitter.conversions.U64Ops._ test("toU64HextString") { diff --git a/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala b/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala index 422e44b5d9..544253c112 100644 --- a/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala @@ -3,9 +3,9 @@ package com.twitter.io import java.lang.{Double => JDouble, Float => JFloat} import java.nio.charset.StandardCharsets import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -final class BufByteWriterTest extends FunSuite with GeneratorDrivenPropertyChecks { +final class BufByteWriterTest extends FunSuite with ScalaCheckDrivenPropertyChecks { import ByteWriter.OverflowException private[this] def assertIndex(bw: ByteWriter, index: Int): Unit = bw match { diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index ba5ce2464f..f66780c60a 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -3,7 +3,7 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} import org.scalatest.FunSuite -import org.scalatest.prop.Checkers +import org.scalatestplus.scalacheck.Checkers class BufReaderTest extends FunSuite with Checkers { diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index 80a5b7122e..151daddd59 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -6,16 +6,17 @@ import java.nio.charset.{StandardCharsets => JChar} import java.util.Arrays import org.scalacheck.{Arbitrary, Gen, Prop} import org.scalatest.FunSuite -import org.scalatest.junit.AssertionsForJUnit -import org.scalatest.mockito.MockitoSugar -import org.scalatest.prop.{Checkers, GeneratorDrivenPropertyChecks} +import org.scalatestplus.junit.AssertionsForJUnit +import org.scalatestplus.mockito.MockitoSugar +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatestplus.scalacheck.Checkers import scala.collection.mutable import java.nio.charset.Charset class BufTest extends FunSuite with MockitoSugar - with GeneratorDrivenPropertyChecks + with ScalaCheckDrivenPropertyChecks with Checkers with AssertionsForJUnit { @@ -106,7 +107,7 @@ class BufTest test("Buf.equals") { val bytes = Array[Byte](0, 1, 2) val byteBuffer = Buf.ByteBuffer.Owned(java.nio.ByteBuffer.wrap(bytes)) - val byteArray = Buf.ByteArray(bytes: _*) + val byteArray = Buf.ByteArray(bytes.toSeq: _*) val composite = Buf.ByteArray(0).concat(Buf.ByteArray(1)).concat(Buf.ByteArray(2)) val bufs = Seq(byteBuffer, byteArray, composite) diff --git a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala index 7762bbf9f6..45ee50f88c 100644 --- a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala @@ -4,9 +4,9 @@ import java.lang.{Double => JDouble, Float => JFloat} import java.nio.charset.StandardCharsets import org.scalacheck.Gen import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class ByteReaderTest extends FunSuite with GeneratorDrivenPropertyChecks { +class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { import ByteReader._ def readerWith(bytes: Byte*): ByteReader = ByteReader(Buf.ByteArray.Owned(bytes.toArray)) @@ -29,7 +29,7 @@ class ByteReaderTest extends FunSuite with GeneratorDrivenPropertyChecks { test("readString")(forAll { (str1: String, str2: String) => val bytes1 = str1.getBytes(StandardCharsets.UTF_8) val bytes2 = str2.getBytes(StandardCharsets.UTF_8) - val br = readerWith(bytes1 ++ bytes2: _*) + val br = readerWith((bytes1 ++ bytes2).toSeq: _*) assert(br.readString(bytes1.length, StandardCharsets.UTF_8) == str1) assert(br.readString(bytes2.length, StandardCharsets.UTF_8) == str2) intercept[UnderflowException] { br.readByte() } @@ -167,12 +167,12 @@ class ByteReaderTest extends FunSuite with GeneratorDrivenPropertyChecks { test("readUnsignedIntBE")(forAll { i: Int => val br = new ByteReaderImpl(Buf.Empty) { override def readIntBE() = i } - assert(br.readUnsignedIntBE() == (i & 0xffffffffl)) + assert(br.readUnsignedIntBE() == (i & 0xffffffffL)) }) test("readUnsignedIntLE")(forAll { i: Int => val br = new ByteReaderImpl(Buf.Empty) { override def readIntLE() = i } - assert(br.readUnsignedIntLE() == (i & 0xffffffffl)) + assert(br.readUnsignedIntLE() == (i & 0xffffffffL)) }) val uInt64s: Gen[BigInt] = Gen diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index 5cd0f329f4..d2b79646b0 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -141,19 +141,20 @@ class PipeTest extends FunSuite with Matchers { assert(closed) } - test("write then reads then close") { + test("write then reads then close") { testWriteReadClose } + def testWriteReadClose = { val rw = new Pipe[Buf] val wf = rw.write(buf(0, 6)) assert(!wf.isDone) assertRead(rw, 0, 6) assert(wf.isDone) - assert(!rw.close().isDone) assertReadEofAndClosed(rw) } - test("read then write then close") { + test("read then write then close") { readWriteClose } + def readWriteClose = { val rw = new Pipe[Buf] val rf = rw.read() @@ -213,13 +214,16 @@ class PipeTest extends FunSuite with Matchers { assertFailedEx(rw.read()) } - test("read after close with no pending reads") { + def readAfterCloseNoPendingReads = { val rw = new Pipe[Buf] assert(!rw.close().isDone) assertReadEofAndClosed(rw) } + test("read after close with no pending reads") { + readAfterCloseNoPendingReads + } - test("read after close with pending data") { + def readAfterClosePendingData = { val rw = new Pipe[Buf] val wf = rw.write(buf(0, 1)) @@ -235,6 +239,7 @@ class PipeTest extends FunSuite with Matchers { await(rw.onClose) } } + test("read after close with pending data") { readAfterClosePendingData } test("read while reading") { val rw = new Pipe[Buf] @@ -280,7 +285,7 @@ class PipeTest extends FunSuite with Matchers { assertReadEofAndClosed(rw) } - test("close not satisfied until reads are fulfilled") { + def closeNotSatisfiedUntillAllReadsDone = { val rw = new Pipe[Buf] val rf = rw.read() val cf = rf.flatMap { _ => @@ -295,8 +300,9 @@ class PipeTest extends FunSuite with Matchers { assert(!cf.isDone) assertReadEofAndClosed(rw) } + test("close not satisfied until reads are fulfilled")(closeNotSatisfiedUntillAllReadsDone) - test("close while read pending") { + def closeWhileReadPending = { val rw = new Pipe[Buf] val rf = rw.read() assert(!rf.isDefined) @@ -304,14 +310,16 @@ class PipeTest extends FunSuite with Matchers { assert(rw.close().isDone) assert(rf.isDefined) } + test("close while read pending")(closeWhileReadPending) - test("close then close") { + def closeTwice = { val rw = new Pipe[Buf] assert(!rw.close().isDone) assertReadEofAndClosed(rw) assert(rw.close().isDone) assertReadEofAndClosed(rw) } + test("close then close")(closeTwice) test("close after fail") { val rw = new Pipe[Buf] @@ -432,4 +440,5 @@ class PipeTest extends FunSuite with Matchers { assertReadEofAndClosed(rw) } } + } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 588c852aed..b2115941cc 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -10,14 +10,14 @@ import java.util.concurrent.atomic.AtomicBoolean import org.mockito.Mockito._ import org.scalacheck.{Arbitrary, Gen} import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.scalatest.{FunSuite, Matchers} import scala.annotation.tailrec import scala.collection.mutable.ArrayBuffer class ReaderTest extends FunSuite - with GeneratorDrivenPropertyChecks + with ScalaCheckDrivenPropertyChecks with Matchers with Eventually with IntegrationPatience { diff --git a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala index 7c53e9f9b8..80f0987307 100644 --- a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala @@ -2,6 +2,7 @@ package com.twitter.util import java.util.concurrent.atomic.AtomicReference import org.scalatest.FunSuite +import scala.collection.compat._ class ActivityTest extends FunSuite { test("Activity#flatMap") { diff --git a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala index 2f78779773..874d39f2b2 100644 --- a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala +++ b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala @@ -5,7 +5,7 @@ import org.scalacheck.Arbitrary import org.scalacheck.Arbitrary.arbitrary import org.scalacheck.Gen import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks object Base64LongTest { private val BoundaryValues = @@ -52,7 +52,7 @@ object Base64LongTest { } } -class Base64LongTest extends FunSuite with GeneratorDrivenPropertyChecks { +class Base64LongTest extends FunSuite with ScalaCheckDrivenPropertyChecks { import Base64LongTest._ private[this] def forAllLongs(f: (Alphabet, Long) => Unit): Unit = { diff --git a/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala b/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala index 97ab1f7ade..afa72565ee 100644 --- a/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala @@ -17,12 +17,13 @@ package com.twitter.util import org.scalatest.FunSuite -import org.scalatest.prop.Checkers +import org.scalatestplus.scalacheck.Checkers class CredentialsTest extends FunSuite with Checkers { test("parse a simple auth file") { val content = "username: root\npassword: hellokitty\n" - assert(Credentials(content) == Map("username" -> "root", "password" -> "hellokitty")) + val result = Credentials(content) + assert(result == Map("username" -> "root", "password" -> "hellokitty")) } test("parse a more complex auth file") { diff --git a/util-core/src/test/scala/com/twitter/util/DiffTest.scala b/util-core/src/test/scala/com/twitter/util/DiffTest.scala index be324ad134..2b12ac0697 100644 --- a/util-core/src/test/scala/com/twitter/util/DiffTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DiffTest.scala @@ -2,9 +2,9 @@ package com.twitter.util import org.scalatest.FunSuite import org.scalacheck.Arbitrary.arbitrary -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class DiffTest extends FunSuite with GeneratorDrivenPropertyChecks { +class DiffTest extends FunSuite with ScalaCheckDrivenPropertyChecks { val f: Int => String = _.toString test("Diffable.ofSet") { diff --git a/util-core/src/test/scala/com/twitter/util/EventTest.scala b/util-core/src/test/scala/com/twitter/util/EventTest.scala index 4c60225788..bea5ab5611 100644 --- a/util-core/src/test/scala/com/twitter/util/EventTest.scala +++ b/util-core/src/test/scala/com/twitter/util/EventTest.scala @@ -4,6 +4,7 @@ import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import java.util.concurrent.{CountDownLatch, Executors} import org.scalatest.FunSuite import scala.collection.mutable +import scala.collection.compat._ class EventTest extends FunSuite { @@ -327,8 +328,7 @@ class EventTest extends FunSuite { test("Event.dedupWith") { val e = Event[Int]() - val ref = new AtomicReference[IndexedSeq[Int]] - + val ref = new AtomicReference[Seq[Int]] e.dedupWith { (a, b) => a >= b } @@ -347,9 +347,8 @@ class EventTest extends FunSuite { test("Event.dedup") { val e = Event[Int]() - val ref = new AtomicReference[IndexedSeq[Int]] - - e.dedup.build.register(Witness(ref)) + val ref = new AtomicReference[Seq[Int]] + e.dedup.buildAny[IndexedSeq[Int]].register(Witness(ref)) e.notify(0) e.notify(0) e.notify(1) @@ -359,4 +358,5 @@ class EventTest extends FunSuite { assert(ref.get() == List(0, 1, 0)) } + } diff --git a/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala b/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala index ae24555c5d..d02eda29bd 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala @@ -1,7 +1,7 @@ package com.twitter.concurrent import org.scalatest.WordSpec -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import com.twitter.util.{Promise, Return} class FutureOfferTest extends WordSpec with MockitoSugar { diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index bf39f45196..8e2defd953 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -9,8 +9,8 @@ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalacheck.{Arbitrary, Gen} import org.scalatest.WordSpec -import org.scalatest.mockito.MockitoSugar -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.mockito.MockitoSugar +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.JavaConverters._ import scala.runtime.NonLocalReturnControl import scala.util.Random @@ -38,7 +38,7 @@ private object FutureTest { } } -class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenPropertyChecks { +class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropertyChecks { import FutureTest._ implicit class FutureMatcher[A](future: Future[A]) { @@ -263,92 +263,112 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty val result = Seq(4, 5, 6) "execute after threshold is reached" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(3)(f) - when(f.apply(Seq(1, 2, 3))).thenReturn(Future.value(result)) + when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(result)) batcher(1) - verify(f, never()).apply(any[Seq[Int]]) + verify(m, never()).apply(any[Seq[Int]]) batcher(2) - verify(f, never()).apply(any[Seq[Int]]) + verify(m, never()).apply(any[Seq[Int]]) batcher(3) - verify(f).apply(Seq(1, 2, 3)) + verify(m).apply(Iterable(1, 2, 3)) } "execute after bufSizeFraction threshold is reached" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] - val batcher = Future.batched(3, sizePercentile = 0.67f)(f) + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - when(f.apply(Seq(1, 2, 3))).thenReturn(Future.value(result)) + val batcher = Future.batched(3, sizePercentile = 0.67f)(f) + when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(result)) batcher(1) - verify(f, never()).apply(any[Seq[Int]]) + verify(m, never()).apply(any[Seq[Int]]) batcher(2) - verify(f).apply(Seq(1, 2)) + verify(m).apply(Iterable(1, 2)) } "treat bufSizeFraction return value < 0.0f as 1" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(3, sizePercentile = 0.4f)(f) - when(f.apply(Seq(1, 2, 3))).thenReturn(Future.value(result)) + when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(result)) batcher(1) - verify(f).apply(Seq(1)) + verify(m).apply(Iterable(1)) } "treat bufSizeFraction return value > 1.0f should return maxSizeThreshold" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(3, sizePercentile = 1.3f)(f) - when(f.apply(Seq(1, 2, 3))).thenReturn(Future.value(result)) + when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(result)) batcher(1) - verify(f, never()).apply(any[Seq[Int]]) + verify(m, never()).apply(any[Seq[Int]]) batcher(2) - verify(f, never()).apply(any[Seq[Int]]) + verify(m, never()).apply(any[Seq[Int]]) batcher(3) - verify(f).apply(Seq(1, 2, 3)) + verify(m).apply(Iterable(1, 2, 3)) } "execute after time threshold" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(3, 3.seconds)(f) Time.withCurrentTimeFrozen { control => - when(f(Seq(1))).thenReturn(Future.value(Seq(4))) + when(m(Iterable(1))).thenReturn(Future.value(Seq(4))) batcher(1) - verify(f, never()).apply(any[Seq[Int]]) + verify(m, never()).apply(any[Seq[Int]]) control.advance(1.second) timer.tick() - verify(f, never()).apply(any[Seq[Int]]) + verify(m, never()).apply(any[Seq[Int]]) control.advance(1.second) timer.tick() - verify(f, never()).apply(any[Seq[Int]]) + verify(m, never()).apply(any[Seq[Int]]) control.advance(1.second) timer.tick() - verify(f).apply(Seq(1)) + verify(m).apply(Iterable(1)) } } "only execute once if both are reached" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(3)(f) Time.withCurrentTimeFrozen { control => - when(f(Seq(1, 2, 3))).thenReturn(Future.value(result)) + when(m(Iterable(1, 2, 3))).thenReturn(Future.value(result)) batcher(1) batcher(2) batcher(3) control.advance(10.seconds) timer.tick() - verify(f).apply(Seq(1, 2, 3)) + verify(m).apply(Iterable(1, 2, 3)) } } "execute when flushBatch is called" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(4)(f) batcher(1) @@ -356,11 +376,14 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty batcher(3) batcher.flushBatch() - verify(f).apply(Seq(1, 2, 3)) + verify(m).apply(Iterable(1, 2, 3)) } "only execute for remaining items when flushBatch is called after size threshold is reached" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(4)(f) batcher(1) @@ -368,14 +391,17 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty batcher(3) batcher(4) batcher(5) - verify(f, times(1)).apply(Seq(1, 2, 3, 4)) + verify(m, times(1)).apply(Iterable(1, 2, 3, 4)) batcher.flushBatch() - verify(f, times(1)).apply(Seq(5)) + verify(m, times(1)).apply(Iterable(5)) } "only execute once when time threshold is reached after flushBatch is called" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(4, 3.seconds)(f) Time.withCurrentTimeFrozen { control => @@ -386,12 +412,15 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty control.advance(10.seconds) timer.tick() - verify(f, times(1)).apply(Seq(1, 2, 3)) + verify(m, times(1)).apply(Iterable(1, 2, 3)) } } "only execute once when time threshold is reached before flushBatch is called" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(4, 3.seconds)(f) Time.withCurrentTimeFrozen { control => @@ -402,16 +431,19 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty timer.tick() batcher.flushBatch() - verify(f, times(1)).apply(Seq(1, 2, 3)) + verify(m, times(1)).apply(Iterable(1, 2, 3)) } } "propagates results" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(3)(f) Time.withCurrentTimeFrozen { _ => - when(f(Seq(1, 2, 3))).thenReturn(Future.value(result)) + when(m(Iterable(1, 2, 3))).thenReturn(Future.value(result)) val res1 = batcher(1) assert(!res1.isDefined) val res2 = batcher(2) @@ -425,28 +457,31 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty assert(await(res2) == 5) assert(await(res3) == 6) - verify(f).apply(Seq(1, 2, 3)) + verify(m).apply(Iterable(1, 2, 3)) } } "not block other batches" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(3)(f) Time.withCurrentTimeFrozen { _ => val blocker = new Promise[Unit] val thread = new Thread { override def run(): Unit = { - when(f(result)).thenReturn(Future.value(Seq(7, 8, 9))) + when(m(result)).thenReturn(Future.value(Seq(7, 8, 9))) batcher(4) batcher(5) batcher(6) - verify(f).apply(result) + verify(m).apply(result) blocker.setValue(()) } } - when(f(Seq(1, 2, 3))).thenAnswer { + when(m(Seq(1, 2, 3))).thenAnswer { new Answer[Future[Seq[Int]]] { def answer(invocation: InvocationOnMock): Future[Seq[Int]] = { thread.start() @@ -459,15 +494,18 @@ class FutureTest extends WordSpec with MockitoSugar with GeneratorDrivenProperty batcher(1) batcher(2) batcher(3) - verify(f).apply(Seq(1, 2, 3)) + verify(m).apply(Iterable(1, 2, 3)) } } "swallow exceptions" in { - val f = mock[Seq[Int] => Future[Seq[Int]]] + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + val batcher = Future.batched(3)(f) - when(f(Seq(1, 2, 3))).thenAnswer { + when(m(Iterable(1, 2, 3))).thenAnswer { new Answer[Unit] { def answer(invocation: InvocationOnMock): Unit = { throw new Exception diff --git a/util-core/src/test/scala/com/twitter/util/LocalTest.scala b/util-core/src/test/scala/com/twitter/util/LocalTest.scala index 9651b6e2f9..0c2bf8ca3e 100644 --- a/util-core/src/test/scala/com/twitter/util/LocalTest.scala +++ b/util-core/src/test/scala/com/twitter/util/LocalTest.scala @@ -3,9 +3,9 @@ package com.twitter.util import org.scalacheck.Arbitrary import org.scalacheck.Arbitrary.arbitrary import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class LocalTest extends FunSuite with GeneratorDrivenPropertyChecks { +class LocalTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test("Local: should be undefined by default") { val local = new Local[Int] assert(local() == None) diff --git a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala index b48d4025ac..1a5a5acba6 100644 --- a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala @@ -3,7 +3,7 @@ package com.twitter.util import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.WordSpec -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar object MonitorTest { class MockMonitor extends Monitor { diff --git a/util-core/src/test/scala/com/twitter/util/PoolTest.scala b/util-core/src/test/scala/com/twitter/util/PoolTest.scala index e9e7d7adb0..076d42f47b 100644 --- a/util-core/src/test/scala/com/twitter/util/PoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PoolTest.scala @@ -1,12 +1,13 @@ package com.twitter.util import scala.collection.mutable +import scala.collection.Seq import org.scalatest.WordSpec - import com.twitter.conversions.DurationOps._ class PoolTest extends WordSpec { + "SimplePool" should { "with a simple queue of items" should { "it reseves items in FIFO order" in { diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 3e2ab9f423..3f9226d5ec 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -6,11 +6,11 @@ import java.util.concurrent.TimeUnit import org.scalatest.WordSpec import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import com.twitter.conversions.DurationOps._ -trait TimeLikeSpec[T <: TimeLike[T]] extends WordSpec with GeneratorDrivenPropertyChecks { +trait TimeLikeSpec[T <: TimeLike[T]] extends WordSpec with ScalaCheckDrivenPropertyChecks { val ops: TimeLikeOps[T] import ops._ diff --git a/util-core/src/test/scala/com/twitter/util/TimerTest.scala b/util-core/src/test/scala/com/twitter/util/TimerTest.scala index 9930ba8b49..02f7382164 100644 --- a/util-core/src/test/scala/com/twitter/util/TimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimerTest.scala @@ -8,7 +8,7 @@ import org.mockito.Matchers.any import org.mockito.Mockito.{never, verify, when} import org.scalatest.FunSuite import org.scalatest.concurrent.{IntegrationPatience, Eventually} -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar class TimerTest extends FunSuite with MockitoSugar with Eventually with IntegrationPatience { diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 88e49f5794..8e16c0d1b4 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -3,10 +3,11 @@ package com.twitter.util import java.util.concurrent.atomic.AtomicReference import org.scalacheck.Arbitrary.arbitrary import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable +import scala.collection.compat._ -class VarTest extends FunSuite with GeneratorDrivenPropertyChecks { +class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { private case class U[T](init: T) extends UpdatableVar[T](init) { import Var.Observer @@ -217,7 +218,7 @@ class VarTest extends FunSuite with GeneratorDrivenPropertyChecks { } test("Var.collect: empty") { - assert(Var.collect(Seq.empty[Var[Int]]).sample == Seq.empty) + assert(Var.collect(List.empty[Var[Int]]).sample == Nil) } test("Var.collect[Seq]") { diff --git a/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala b/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala index 4e96878bdc..03fa81e233 100644 --- a/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala @@ -2,9 +2,9 @@ package com.twitter.util import org.scalacheck.Gen import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class WaitQueueTest extends FunSuite with GeneratorDrivenPropertyChecks { +class WaitQueueTest extends FunSuite with ScalaCheckDrivenPropertyChecks { private class EmptyK[A] extends Promise.K[A] { protected[util] def depth: Short = 0 diff --git a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala index b7fe1d3e27..b2f7a31e78 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala @@ -1,9 +1,9 @@ package com.twitter.hashing import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class HashableTest extends FunSuite with GeneratorDrivenPropertyChecks { +class HashableTest extends FunSuite with ScalaCheckDrivenPropertyChecks { private[this] val algorithms = Seq( Hashable.CRC32_ITU, diff --git a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala index 313a806411..313985ff67 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala @@ -2,13 +2,13 @@ package com.twitter.hashing import org.scalacheck.Gen import org.scalatest.WordSpec -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable import java.io.{BufferedReader, InputStreamReader} import java.nio.{ByteBuffer, ByteOrder} import java.security.MessageDigest -class KetamaDistributorTest extends WordSpec with GeneratorDrivenPropertyChecks { +class KetamaDistributorTest extends WordSpec with ScalaCheckDrivenPropertyChecks { "KetamaDistributor" should { val nodes = Seq( KetamaNode("10.0.1.1", 600, 1), @@ -81,8 +81,7 @@ class KetamaDistributorTest extends WordSpec with GeneratorDrivenPropertyChecks val md = MessageDigest.getInstance("MD5") ketama.hashInt(i, md) val array = md.digest() - - assert(array.deep == md.digest(i.toString.getBytes("UTF-8")).deep) + assert(array.toSeq == md.digest(i.toString.getBytes("UTF-8")).toSeq) } } diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index 539a6b5296..9d0189e9bc 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -206,7 +206,7 @@ class Hotspot extends Jvm { def tenuringThreshold: Long = counter("sun.gc.policy.tenuringThreshold").map(long).getOrElse(0L) def snapCounters: Map[String, String] = - counters("").mapValues(_.getValue().toString) + counters("").map { case (k, v) => (k -> v.getValue().toString) } def forceGc(): Unit = System.gc() } diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala index d1c4e6be5e..10fd812145 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala @@ -35,7 +35,7 @@ trait Jvm { */ def snap: Snapshot - def edenPool: Pool + def edenPool: com.twitter.jvm.Pool /** * Gets the current usage of the metaspace, if available. diff --git a/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala index 5de408fdcd..a4e283d02e 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala @@ -11,7 +11,7 @@ class Philosopher { private val lock = new ReentrantLock() def dine(neighbor: Philosopher): Unit = { lock.lockInterruptibly() - ready.setValue(Unit) + ready.setValue(()) Await.ready(neighbor.ready) neighbor.dine(this) lock.unlock() diff --git a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala index f318994ce8..d600b054b8 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala @@ -20,12 +20,13 @@ object EstimatorApp extends App { case Array("windowed", n, windows) => new WindowedMeans( n.toInt, - windows.split(",") map { w => - w.split(":") match { - case Array(w, i) => (w.toInt, i.toInt) - case _ => throw new IllegalArgumentException("bad weight, count pair " + w) - } - } + windows + .split(",").iterator.map { w => + w.split(":") match { + case Array(w, i) => (w.toInt, i.toInt) + case _ => throw new IllegalArgumentException("bad weight, count pair " + w) + } + }.toSeq ) case Array("load", interval) => new LoadAverage(interval.toDouble) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala index 089a0449e7..d525c96f58 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala @@ -60,7 +60,7 @@ class JvmTest extends WordSpec with TestLogging { assert(jvm.executor.schedules == List()) jvm foreachGc { b += _ } assert(jvm.executor.schedules.size == 1) - val Seq((r, _, _, _)) = jvm.executor.schedules + val r = jvm.executor.schedules.head._1 r.run() assert(b == List()) val gc = Gc(0, "pcopy", Time.now, 1.millisecond) @@ -97,7 +97,7 @@ class JvmTest extends WordSpec with TestLogging { jvm foreachGc { _ => /*ignore*/ } assert(jvm.executor.schedules.size == 1) - val Seq((r, _, _, _)) = jvm.executor.schedules + val r = jvm.executor.schedules.head._1 val gc = Gc(0, "pcopy", Time.now, 1.millisecond) r.run() jvm.pushGc(gc) @@ -130,7 +130,7 @@ class JvmTest extends WordSpec with TestLogging { val query = jvm.monitorGcs(10.seconds) assert(jvm.executor.schedules.size == 1) - val Seq((r, _, _, _)) = jvm.executor.schedules + val r = jvm.executor.schedules.head._1 val gc0 = Gc(0, "pcopy", Time.now, 1.millisecond) val gc1 = Gc(1, "CMS", Time.now, 1.millisecond) jvm.pushGc(gc1) diff --git a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala index 4756b161f0..8b7672f781 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala @@ -36,8 +36,8 @@ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.{BeforeAndAfter, WordSpec} import org.scalatest.concurrent.Eventually -import org.scalatest.mockito.MockitoSugar import org.scalatest.time.{Millis, Minutes, Span} +import org.scalatestplus.mockito.MockitoSugar class ScribeHandlerTest extends WordSpec with BeforeAndAfter with Eventually with MockitoSugar { val record1 = new javalog.LogRecord(Level.INFO, "This is a message.") diff --git a/util-registry/src/test/scala/com/twitter/util/registry/GlobalRegistryTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/GlobalRegistryTest.scala index c7997fd9c2..c8ca5345eb 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/GlobalRegistryTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/GlobalRegistryTest.scala @@ -1,6 +1,5 @@ package com.twitter.util.registry - class GlobalRegistryTest extends RegistryTest { def mkRegistry(): Registry = GlobalRegistry.withRegistry(new SimpleRegistry) { GlobalRegistry.get diff --git a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala index 0ff2f77c42..2745158de2 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala @@ -4,7 +4,7 @@ import java.util.logging.Logger import org.mockito.Mockito.{never, verify} import org.mockito.Matchers.anyObject import org.scalatest.FunSuite -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar class RosterTest extends FunSuite with MockitoSugar { def withRoster(fn: (Roster, Logger) => Unit): Unit = { diff --git a/util-registry/src/test/scala/com/twitter/util/registry/SimpleRegistryTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/SimpleRegistryTest.scala index d43e8ae142..05608e7d1c 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/SimpleRegistryTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/SimpleRegistryTest.scala @@ -1,6 +1,5 @@ package com.twitter.util.registry - class SimpleRegistryTest extends RegistryTest { def mkRegistry(): Registry = new SimpleRegistry def name: String = "SimpleRegistry" diff --git a/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala b/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala index eada596bbd..21648e3653 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala @@ -52,6 +52,16 @@ object PemFileTestUtils { assert(logMessage.contains(expectedExMessage)) } + def assertAnyLogMessage( + name: String + )(logMessage: String, + filename: String, + expectedExMessages: String* + ): Unit = { + assert(logMessage.startsWith(s"$name ($filename) failed to load: ")) + assert(expectedExMessages.exists(expectedExMessage => logMessage.contains(expectedExMessage))) + } + def testFileDoesntExist[A](name: String, read: File => Try[A]): Unit = { val handler = newHandler() val tempFile = File.createTempFile("test", "ext") @@ -62,7 +72,11 @@ object PemFileTestUtils { assert(!tempFile.exists()) val tryReadPem: Try[A] = read(tempFile) - assertLogMessage(name)(handler.get, tempFile.getName(), "No such file or directory") + assertAnyLogMessage(name)( + handler.get, + tempFile.getName(), + "No such file or directory", + "The system cannot find the file specified") assertIOException(tryReadPem) } @@ -74,7 +88,11 @@ object PemFileTestUtils { val parentFile = tempFile.getParentFile() val tryReadPem: Try[A] = read(parentFile) - assertLogMessage(name)(handler.get, parentFile.getName(), "Is a directory") + assertAnyLogMessage(name)( + handler.get, + parentFile.getName(), + "Is a directory", + "Access is denied") assertIOException(tryReadPem) } diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/Item.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/Item.scala index f1a5f91197..a4a8a41a28 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/Item.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/Item.scala @@ -1,7 +1,5 @@ package com.twitter.util.logging -import com.twitter.util.logging.Item._ - object Item extends Logging { def baz: String = { @@ -11,6 +9,7 @@ object Item extends Logging { } class Item(val name: String, val description: String, size: Int) extends Logging { + import Item._ info(s"New item: name = $name, description = $description, size = $size") def dimension: Int = { diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala index 72481fde34..3c914e0291 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala @@ -2,7 +2,7 @@ package com.twitter.util.logging import org.mockito.Matchers._ import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.scalatest.{FunSuite, Matchers} import org.slf4j import scala.language.reflectiveCalls diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala index def4806048..aeba68ea36 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala @@ -2,7 +2,7 @@ package com.twitter.util.logging import org.mockito.Matchers._ import org.mockito.Mockito._ -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import org.scalatest.{FunSuite, Matchers} import org.slf4j import org.slf4j.Marker diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index f4dec5bada..5accc7db61 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -110,19 +110,9 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => "of Gauges. Indicative of a leak or code registering the same gauge more " + s"often than expected. (For $toString)" ) { - val largeCgs = gauges.asScala.flatMap { - case (ks, cg) => - if (cg.totalSize >= 10000) Some(ks -> cg.totalSize) - else None - } - if (largeCgs.isEmpty) { - Nil - } else { - largeCgs.map { - case (ks, size) => - Issue(ks.mkString("/") + "=" + size) - }.toSeq - } + gauges.asScala.collect { + case (ks, cg) if (cg.totalSize >= 10000) => Issue(ks.mkString("/") + "=" + cg.totalSize) + }.toSeq } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index c9af5acd83..03494414b1 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -208,14 +208,10 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { new HistogramDetail { def counts: Seq[BucketAndCount] = { addedValues - .map { x => - nearestPosInt(x) - } - .groupBy(identity) - .mapValues(_.size) + .groupBy(nearestPosInt) + .map { case (k, vs) => BucketAndCount(k, k + 1, vs.size) } .toSeq - .sortWith(_._1 < _._1) - .map { case (k, v) => BucketAndCount(k, k + 1, v) } + .sortBy(_.lowerLimit) } } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 906a221ac3..d032fefc16 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -150,6 +150,7 @@ trait StatsReceiver { * @see [[StatsReceiver.addGauge]] if you can properly control the lifecycle * of the returned [[Gauge gauge]]. */ + @varargs def provideGauge(name: String*)(f: => Float): Unit = { val gauge = addGauge(name: _*)(f) StatsReceiver.synchronized { @@ -174,6 +175,7 @@ trait StatsReceiver { * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ + @varargs def addGauge(name: String*)(f: => Float): Gauge = addGauge(Verbosity.Default, name: _*)(f) /** @@ -193,6 +195,7 @@ trait StatsReceiver { * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ + @varargs def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge /** @@ -253,4 +256,27 @@ trait StatsReceiver { } } -abstract class AbstractStatsReceiver extends StatsReceiver +abstract class AbstractStatsReceiver extends StatsReceiver { + @varargs + final override def counter(name: String*): Counter = counter(Verbosity.Default, name: _*) + @varargs + final def counter(verbosity: Verbosity, name: String*): Counter = + counterImpl(verbosity, name) + protected def counterImpl(verbosity: Verbosity, name: scala.collection.Seq[String]): Counter + + @varargs + final override def stat(name: String*): Stat = stat(Verbosity.Default, name: _*) + @varargs + final def stat(verbosity: Verbosity, name: String*): Stat = statImpl(verbosity, name) + protected def statImpl(verbosity: Verbosity, name: scala.collection.Seq[String]): Stat + + final override def addGauge(name: String*)(f: => Float): Gauge = + addGauge(Verbosity.Default, name: _*)(f) + final def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = + addGaugeImpl(verbosity, name, f) + protected def addGaugeImpl( + verbosity: Verbosity, + name: scala.collection.Seq[String], + f: => Float + ): Gauge +} diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java index 9407f9709e..91e83c0e1e 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java @@ -9,7 +9,7 @@ import java.util.concurrent.TimeUnit; import scala.Function0; -import scala.collection.Seq; +import scala.collection.immutable.Seq; /** * Java compatibility layer for {@link com.twitter.finagle.stats.StatsReceiver}. @@ -80,10 +80,10 @@ public void testAbstractStatsReceiver() { StatsReceiver sr = new AbstractStatsReceiver() { @Override - public Stat stat(Verbosity verbosity, String... name) { - return nullSr.stat(verbosity, name); + public Stat statImpl(Verbosity verbosity, scala.collection.Seq name) { + return nullSr.stat(verbosity, name.toSeq()); } - + @Override public Object repr() { return this; @@ -95,18 +95,13 @@ public boolean isNull() { } @Override - public Counter counter(Verbosity verbosity, Seq name) { - return nullSr.counter(name); - } - - @Override - public Stat stat(Verbosity verbosity, Seq name) { - return nullSr.stat(name); + public Counter counterImpl(Verbosity verbosity, scala.collection.Seq name) { + return nullSr.counter(name.toSeq()); } @Override - public Gauge addGauge(Verbosity verbosity, Seq name, Function0 f) { - return nullSr.addGauge(verbosity, name, f); + public Gauge addGaugeImpl(Verbosity verbosity, scala.collection.Seq name, Function0 f) { + return nullSr.addGauge(verbosity, name.toSeq(), f); } }; } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala index fbe15c44a4..4c03a81529 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala @@ -10,7 +10,7 @@ class CategorizingExceptionStatsHandlerTest extends FunSuite { val esh = new CategorizingExceptionStatsHandler( _ => Some("clienterrors"), - PartialFunction.apply(_ => Some("service")), + { case _ => Some("service") }, true ) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala index 6f3f41d206..c18b08f375 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala @@ -2,9 +2,9 @@ package com.twitter.finagle.stats import org.scalacheck.{Arbitrary, Gen} import org.scalatest.FunSuite -import org.scalatest.prop.GeneratorDrivenPropertyChecks +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class DelegatingStatsReceiverTest extends FunSuite with GeneratorDrivenPropertyChecks { +class DelegatingStatsReceiverTest extends FunSuite with ScalaCheckDrivenPropertyChecks { class Ctx { var leafCounter = 0 diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index aa83dc4852..25b87a9fc5 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -4,6 +4,7 @@ import java.io.{ByteArrayOutputStream, PrintStream} import java.nio.charset.StandardCharsets import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} +import scala.collection.parallel.immutable.ParRange class InMemoryStatsReceiverTest extends FunSuite with Eventually with IntegrationPatience { @@ -30,7 +31,7 @@ class InMemoryStatsReceiverTest extends FunSuite with Eventually with Integratio test("threadsafe counter") { val inMemoryStatsReceiver = new InMemoryStatsReceiver - (1 to 50).par.foreach(_ => inMemoryStatsReceiver.counter("same").incr()) + new ParRange(1 to 50).foreach(_ => inMemoryStatsReceiver.counter("same").incr()) eventually { assert(inMemoryStatsReceiver.counter("same")() == 50) } @@ -38,7 +39,7 @@ class InMemoryStatsReceiverTest extends FunSuite with Eventually with Integratio test("threadsafe stats") { val inMemoryStatsReceiver = new InMemoryStatsReceiver - (1 to 50).par.foreach(_ => inMemoryStatsReceiver.stat("same").add(1.0f)) + new ParRange(1 to 50).foreach(_ => inMemoryStatsReceiver.stat("same").add(1.0f)) eventually { assert(inMemoryStatsReceiver.stat("same")().size == 50) } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index f1b16e5a0a..eafd7af4b2 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -40,7 +40,7 @@ class StatsReceiverTest extends FunSuite { assert(c2.c == 1) class MemStat extends Stat { - var values: Seq[Float] = ArrayBuffer.empty[Float] + var values: scala.collection.Seq[Float] = ArrayBuffer.empty[Float] def add(f: Float): Unit = { values = values :+ f } } val s1 = new MemStat diff --git a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala index e01d82d33a..e510314759 100644 --- a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala +++ b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala @@ -66,7 +66,7 @@ trait TestLogging extends BeforeAndAfter { self: WordSpec => logger.addHandler(traceHandler) } - def logLines(): Seq[String] = traceHandler.get.split("\n") + def logLines(): Seq[String] = traceHandler.get.split("\n").toSeq /** * Verify that the logger set up with `traceLogger` has received a log line with the given diff --git a/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala b/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala index 25dfa14343..86484cfc4a 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala +++ b/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala @@ -3,7 +3,7 @@ package com.twitter.util.testing import org.mockito.Matchers._ import org.mockito.Mockito._ import org.scalatest.FunSuite -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar class ArgumentCaptureTest extends FunSuite with MockitoSugar with ArgumentCapture { class MockSubject { diff --git a/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala b/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala index 8f0fa1aa07..97fc18e561 100644 --- a/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala +++ b/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala @@ -23,13 +23,13 @@ trait AsyncCallbackPromise[T] extends Promise[T] { class StringCallbackPromise extends AsyncCallbackPromise[String] with AsyncCallback.StringCallback { def processResult(rc: Int, path: String, ctx: AnyRef, name: String): Unit = { - process(rc, path) { name } + process(rc, path)(name) } } class UnitCallbackPromise extends AsyncCallbackPromise[Unit] with AsyncCallback.VoidCallback { def processResult(rc: Int, path: String, ctx: AnyRef): Unit = { - process(rc, path) { Unit } + process(rc, path)(()) } } diff --git a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala index c80c2ee00b..c0e6970891 100644 --- a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala +++ b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala @@ -48,14 +48,14 @@ case class NativeConnector( */ def apply(): Future[ZooKeeper] = serialized { - connection getOrElse { + connection.getOrElse { val c = mkConnection c.sessionEvents foreach { event => sessionBroker.send(event()).sync() } connection = Some(c) c - } apply () + }.apply }.flatten .rescue { case e: NativeConnector.ConnectTimeoutException => @@ -194,7 +194,7 @@ object NativeConnector { log.debug("release") zk.close() zookeeper = None - releasePromise.setValue(Unit) + releasePromise.setValue(()) } } } diff --git a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala index d7f550f375..de7af2606c 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala @@ -239,7 +239,7 @@ trait ZNode { removed = knownChildren -- children ) log.debug("updating %s with %d children", path, treeUpdate.added.size) - broker send (treeUpdate) sync () onSuccess { _ => + broker.send(treeUpdate).sync.onSuccess { _ => log.debug("updated %s with %d children", path, treeUpdate.added.size) treeUpdate.added foreach { z => pipeSubTreeUpdates(z.monitorTree()) @@ -256,7 +256,7 @@ trait ZNode { // Tell the broker about the children we lost; otherwise, if there were no children, // this deletion should be reflected in a watch on the parent node, if one exists. if (knownChildren.size > 0) { - broker send (ZNode.TreeUpdate(this, removed = knownChildren)) sync () + broker.send(ZNode.TreeUpdate(this, removed = knownChildren)).sync } else { Future.Done } onSuccess { _ => diff --git a/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala b/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala index 12425fe3f3..2a97c0c1c7 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala @@ -42,7 +42,7 @@ trait ZkClient { /* Settings */ /** Default: Creator access */ - val acl: Seq[ACL] = CREATOR_ALL_ACL.asScala + val acl: Seq[ACL] = CREATOR_ALL_ACL.asScala.toSeq /** Build a ZkClient with another default creation ACL. */ def withAcl(a: Seq[ACL]): ZkClient = transform(_acl = a) diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala index 4ddc1b6d3d..e259046556 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala @@ -100,9 +100,9 @@ class ShardCoordinator(zk: ZkClient, path: String, numShards: Int) { private[this] def shardNodes(): Future[Seq[ZNode]] = { zk(path).getChildren() map { zop => - zop.children filter { child => + (zop.children filter { child => child.path.startsWith(shardPathPrefix) - } sortBy (child => shardIdOf(child.path)) + } sortBy (child => shardIdOf(child.path))).toSeq } } diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala index 99a52ff7f3..0b88ca492e 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala @@ -161,9 +161,9 @@ class ZkAsyncSemaphore( * The sequence includes both nodes that have entered the semaphore as well as waiters. */ private[this] def permitNodes(): Future[Seq[ZNode]] = { - futureSemaphoreNode flatMap { semaphoreNode => + futureSemaphoreNode.flatMap { semaphoreNode => semaphoreNode.getChildren() map { zop => - zop.children filter { child => + zop.children.toSeq filter { child => child.path.startsWith(permitNodePathPrefix) } sortBy (child => sequenceNumberOf(child.path)) } @@ -181,7 +181,7 @@ class ZkAsyncSemaphore( val monitor = node.getChildren.monitor() monitor foreach { tryChildren => tryChildren map { zop => - checkWaiters(zop.children) + checkWaiters(zop.children.toSeq) } } } diff --git a/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala b/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala index ca2b439fd9..05717650f3 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala @@ -2,7 +2,7 @@ package com.twitter.zk import org.mockito.Mockito._ import org.scalatest.WordSpec -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import com.twitter.util.Future diff --git a/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala index 1e5decaa57..9f944522cc 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala @@ -4,7 +4,7 @@ package com.twitter.zk * @author ver@twitter.com */ import org.scalatest.WordSpec -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar class ZNodeTest extends WordSpec with MockitoSugar { "ZNode" should { diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index 881707d8be..e1f19d9724 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -11,7 +11,7 @@ import org.mockito._ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalatest.WordSpec -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ import com.twitter.logging.{Level, Logger} @@ -325,7 +325,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { } "transform" should { - val acl = ZooDefs.Ids.OPEN_ACL_UNSAFE.asScala + val acl = ZooDefs.Ids.OPEN_ACL_UNSAFE.asScala.toSeq val mode = CreateMode.EPHEMERAL_SEQUENTIAL val retries = 37 val retryPolicy = new RetryPolicy.Basic(retries) @@ -594,7 +594,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { ) offer.sync() intercept[KeeperException.NoNodeException] { - offer syncWait () get () + offer.syncWait().get } assert(offer.sync().isDefined == false) } @@ -679,7 +679,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { val update = znode.getChildren.monitor() results foreach { result => - val r = update syncWait () get () + val r = update.syncWait().get assert(r.path == result.path) } } @@ -819,7 +819,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { val offer = treeRoot.monitorTree() treeChildren foreach { _ => - val ztu = Await.result(offer sync (), 1.second) + val ztu = Await.result(offer.sync(), 1.second) val e = expectedByPath(ztu.parent.path) assert(ztu.parent.path == e.parent.path) assert(ztu.added.map { _.path } == e.added.map { _.path }.toSet) @@ -832,7 +832,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { updateTree foreach { z => updatePromises get (z.path) foreach { _.setValue(event(z.path)) } - val ztu = Await.result(offer sync (), 1.second) + val ztu = Await.result(offer.sync(), 1.second) val e = updatesByPath(z.path) assert(ztu.parent.path == e.parent.path) assert(ztu.added.map { _.path } == e.added.map { _.path }.toSet) @@ -859,7 +859,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { val offer = treeRoot.monitorTree() intercept[TimeoutException] { treeChildren foreach { _ => - val ztu = Await.result(offer sync (), 1.second) + val ztu = Await.result(offer.sync(), 1.second) val e = expectedByPath(ztu.parent.path) assert(ztu.parent == e.parent) assert(ztu.added.map { _.path } == e.added.map { _.path }.toSet) diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala index ef09ba9211..0633f320e9 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala @@ -4,7 +4,7 @@ import scala.collection.JavaConverters._ import org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE import org.scalatest.WordSpec -import org.scalatest.mockito.MockitoSugar +import org.scalatestplus.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, JavaTimer} @@ -22,7 +22,7 @@ class ShardCoordinatorTest extends WordSpec with MockitoSugar { val connector = NativeConnector(connectString, 5.seconds, 10.minutes) val zk = ZkClient(connector) .withRetryPolicy(RetryPolicy.Basic(3)) - .withAcl(OPEN_ACL_UNSAFE.asScala) + .withAcl(OPEN_ACL_UNSAFE.asScala.toSeq) Await.result(Future { f(zk) } ensure { zk.release }) } diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala index 14e09af922..ffa523eafe 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala @@ -10,8 +10,8 @@ import org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE import org.scalatest.WordSpec import org.scalatest.concurrent.AsyncAssertions import org.scalatest.concurrent.Eventually._ -import org.scalatest.mockito.MockitoSugar import org.scalatest.time.{Millis, Seconds, Span} +import org.scalatestplus.mockito.MockitoSugar import scala.collection.JavaConverters._ class ZkAsyncSemaphoreTest extends WordSpec with MockitoSugar with AsyncAssertions { @@ -27,7 +27,7 @@ class ZkAsyncSemaphoreTest extends WordSpec with MockitoSugar with AsyncAssertio val connector = NativeConnector(connectString, 5.seconds, 10.minutes) val zk = ZkClient(connector) .withRetryPolicy(RetryPolicy.Basic(3)) - .withAcl(OPEN_ACL_UNSAFE.asScala) + .withAcl(OPEN_ACL_UNSAFE.asScala.toSeq) Await.result(Future { f(zk) } ensure { zk.release }) } From 9fab97d0fd4c498efe33b5d965f04c965cd3c0dc Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 7 Aug 2019 18:48:52 +0000 Subject: [PATCH 186/796] util: Clear up CHANGELOG Problem/Solution: Some of the util 2.13 migration CHANGELOG entries sneaked in other releases (but the actual item is not released). Move those to the Unreleased. JIRA Issues: CSL-8637 Differential Revision: https://phabricator.twitter.biz/D352472 --- CHANGELOG.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2cfa548320..fcfbd93e88 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -29,15 +29,21 @@ Java Compatibility `c.t.finagle.stats.StatsReceiver.provideGauge` and `addGauge` become available to java for passing varags. ``PHAB_ID=D333021`` -19.8.0 ------- - Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ * util-core: (not breaking) `c.t.util.Future.join` and `c.t.util.Future.collect` now take `Iterable[Future[A]]` other than Seq. ``PHAB_ID=D333021`` +* util-core: `com.twitter.util.Event#build` now builds a Seq of events. `Event#buildAny` builds + against any collection of events. ``PHAB_ID=D333021`` + +19.8.0 +------ + +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + * util-logging: The namespace forwarders for `Level` and `Policy` in `com.twitter.logging.config` have been removed. Code should be updated to use `com.twitter.logging.Level` and `com.twitter.logging.Policy` where necessary. Users are encouraged to use 'util-slf4j-api' though @@ -54,13 +60,11 @@ Runtime Behavior Changes * util: Upgrade to Jackson 2.9.9. ``PHAB_ID=D345969`` 19.7.0 +------ Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ -* util-core: `com.twitter.util.Event#build` now builds a Seq of events. `Event#buildAny` builds - against any collection of events. ``PHAB_ID=D333021`` - * util-core: Removed deprecated `c.t.concurrent.Scheduler` methods `usrTime`, `cpuTime`, and `wallTime`. These were deprecated in 2015 and have no replacement. ``PHAB_ID=D330386`` From 22b99087ade627f4e23404b12968e3609fe36f1f Mon Sep 17 00:00:00 2001 From: Anurag Sharma Date: Thu, 8 Aug 2019 21:44:09 +0000 Subject: [PATCH 187/796] util-app: Reduce overhead for GlobalFlag#get method Problem/Solution: This is a follow up PR to address https://github.com/twitter/util/issues/245. Signed-off-by: Yufan Gong JIRA Issues: CSL-8642 Differential Revision: https://phabricator.twitter.biz/D352522 --- .../scala/com/twitter/app/GlobalFlag.scala | 29 ++++++++++--------- .../com/twitter/app/GlobalFlagTest.scala | 5 ++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala index ddd9dae72e..8583199f1c 100644 --- a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala +++ b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala @@ -116,35 +116,39 @@ abstract class GlobalFlag[T] private[app] ( object GlobalFlag { private[app] def get(flagName: String): Option[Flag[_]] = { - def tryMethod(className: String, methodName: String): Option[Flag[_]] = + val className = if (!flagName.endsWith("$")) flagName + "$" else flagName + try { + get(Class.forName(className)) + } catch { case _: ClassNotFoundException => None } + } + + private[app] def get(cls: Class[_]): Option[Flag[_]] = { + def tryMethod(cls: Class[_], methodName: String): Option[Flag[_]] = try { - val cls = Class.forName(className) val m = cls.getMethod(methodName) val isValid = Modifier.isStatic(m.getModifiers) && m.getReturnType == classOf[Flag[_]] && m.getParameterCount == 0 if (isValid) Some(m.invoke(null).asInstanceOf[Flag[_]]) else None } catch { - case _: ClassNotFoundException | _: NoSuchMethodException | _: IllegalArgumentException => + case _: NoSuchMethodException | _: IllegalArgumentException => None } - def tryModuleField(className: String): Option[Flag[_]] = + def tryModuleField(cls: Class[_]): Option[Flag[_]] = try { - val cls = Class.forName(className) val f = cls.getField("MODULE$") val isValid = Modifier.isStatic(f.getModifiers) && classOf[Flag[_]] .isAssignableFrom(f.getType) if (isValid) Some(f.get(null).asInstanceOf[Flag[_]]) else None } catch { - case _: ClassNotFoundException | _: NoSuchFieldException | _: IllegalArgumentException => + case _: NoSuchFieldException | _: IllegalArgumentException => None } - val className = if (!flagName.endsWith("$")) flagName + "$" else flagName - tryModuleField(className).orElse { + tryModuleField(cls).orElse { // fallback for GlobalFlags declared in Java - tryMethod(className, "globalFlagInstance") + tryMethod(cls, "globalFlagInstance") } } @@ -179,10 +183,9 @@ object GlobalFlag { try { val cls: Class[_] = Class.forName(info.className, false, loader) if (cls.isAnnotationPresent(markerClass)) { - get(info.className) match { - case Some(f) => - flagBuilder += f - case None => println("failed for " + info.className) + get(cls) match { + case Some(f) => flagBuilder += f + case None => log.log(java.util.logging.Level.INFO, "failed for " + info.className) } } } catch { diff --git a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala index 2600474766..3c0ad02136 100644 --- a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala @@ -7,6 +7,7 @@ object MyGlobalFlagNoDefault extends GlobalFlag[Int]("a global test flag with no object MyGlobalBooleanFlag extends GlobalFlag[Boolean](false, "a boolean flag") class GlobalFlagTest extends FunSuite { + val flagSet = Set(MyGlobalFlag, MyGlobalBooleanFlag, MyGlobalFlagNoDefault, PackageObjectTest) test("GlobalFlag.get") { assert(MyGlobalBooleanFlag.get.isEmpty) @@ -66,13 +67,13 @@ class GlobalFlagTest extends FunSuite { test("GlobalFlag.getAll") { val mockClassLoader = new MockClassLoader(getClass.getClassLoader) val flags = GlobalFlag.getAll(mockClassLoader) - assert(flags.length == 4) + assert(flags.toSet == flagSet) assert(flags.exists(_.help.equals("a package object test flag"))) } private class MockClassLoader(realClassLoader: ClassLoader) extends ClassLoader(realClassLoader) { private val isValidClassName = (className: String) => - List(MyGlobalFlag, MyGlobalBooleanFlag, MyGlobalFlagNoDefault, PackageObjectTest) + flagSet .map(_.getClass.getName) .contains(className) From 17ebca043226e826a7d59e9bfa06f1706e6d9c6e Mon Sep 17 00:00:00 2001 From: Jonathan Simms Date: Mon, 12 Aug 2019 07:43:56 +0000 Subject: [PATCH 188/796] util-tunables: Add an optional comment field to JsonTunable Problem Tunables are incredibly useful, however, it would be helpful to be able to add human-readable context for each of the values in the JSON config. Solution Added a "comment" field that is ignored by the parser, but is readable by humans. Result: Humans will be able to add comments to config. Differential Revision: https://phabricator.twitter.biz/D353957 --- .../scala/com/twitter/util/tunable/JsonTunableMapper.scala | 6 ++++-- .../com/twitter/util/tunable/JsonTunableMapperTest.scala | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala index c5bf1a361e..f298a67f1e 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala @@ -15,7 +15,8 @@ object JsonTunableMapper { private case class JsonTunable( @JsonProperty(required = true) id: String, @JsonProperty(value = "type", required = true) valueType: Class[Any], - @JsonProperty(required = true) value: Any) + @JsonProperty(required = true) value: Any, + @JsonProperty(required = false) comment: String) private case class JsonTunables(@JsonProperty(required = true) tunables: Seq[JsonTunable]) @@ -92,7 +93,8 @@ object JsonTunableMapper { * { * "id" : "$id2", * "value" : $value, - * "type" : "$class" + * "type" : "$class", + * "comment": "optional comment" * } * ] * } diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala index c818e2c1b6..3fea91c811 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala @@ -82,7 +82,8 @@ class JsonTunableMapperTest extends FunSuite { |{ "tunables": [ | { "id" : "timeoutId1", | "value" : "5.seconds", - | "type" : "com.twitter.util.Duration" + | "type" : "com.twitter.util.Duration", + | "comment": "a very important timeout" | }, | { "id" : "timeoutId2", | "value" : "Duration.Top", @@ -94,7 +95,8 @@ class JsonTunableMapperTest extends FunSuite { | }, | { "id" : "timeoutId4", | "value" : "Duration.Undefined", - | "type" : "com.twitter.util.Duration" + | "type" : "com.twitter.util.Duration", + | "comment": "You'll never believe it: *another* timeout." | } | ] |}""".stripMargin From e2630df6ecca4ad03e9e5c0e73101e9b1fc8c089 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 12 Aug 2019 13:15:52 +0000 Subject: [PATCH 189/796] util: Revert utilIntelliJ scala version back to 2.11.12 Problem/Solution The `utilIntelliJ` project is still expected to be built with Scala 2.11.x. Revert the changes to the `scalaVersion` here back to 2.11.12. Differential Revision: https://phabricator.twitter.biz/D354185 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index c74caa1dc0..4d381a95c2 100644 --- a/build.sbt +++ b/build.sbt @@ -300,7 +300,7 @@ lazy val utilIntellij = Project( baseSettings ).settings( name := "util-intellij", - scalaVersion := "2.12.8" + scalaVersion := "2.11.12" ).dependsOn(utilCore % "test") lazy val utilJvm = Project( From 87a9447d0a2698c6fca296f19d1e1ab3beb9cda6 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Tue, 13 Aug 2019 13:45:54 +0000 Subject: [PATCH 190/796] finagle: move flags out of package objects Problem Declaring flags inside package objects is problematic because the name of the flag ends up being not what we expect. scala> com.twitter.finagle.thrift.maxReusableBufferSize.name res0: String = com.twitter.finagle.thrift.package$maxReusableBufferSize Solution Remove all flags from package objects and ensure that they are in the namespaces (and have the names) that we expect. Result scala> com.twitter.finagle.thrift.maxReusableBufferSize.name res0: String = com.twitter.finagle.thrift.maxReusableBufferSize JIRA Issues: CSL-8267 Differential Revision: https://phabricator.twitter.biz/D353045 --- CHANGELOG.rst | 2 ++ util-app/src/main/scala/com/twitter/app/GlobalFlag.scala | 8 ++++++++ .../src/test/scala/com/twitter/app/GlobalFlagTest.scala | 3 +-- util-app/src/test/scala/com/twitter/app/package.scala | 5 ----- 4 files changed, 11 insertions(+), 7 deletions(-) delete mode 100644 util-app/src/test/scala/com/twitter/app/package.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fcfbd93e88..9a380b8441 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -59,6 +59,8 @@ Runtime Behavior Changes * util: Upgrade to Jackson 2.9.9. ``PHAB_ID=D345969`` +* util-app: It is now illegal to define GlobalFlags enclosed in package objects. ``PHAB_ID=D353045`` + 19.7.0 ------ diff --git a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala index 8583199f1c..88683c0d0d 100644 --- a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala +++ b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala @@ -50,6 +50,10 @@ abstract class GlobalFlag[T] private[app] ( implicit _f: Flaggable[T]) extends Flag[T](null, help, defaultOrUsage, false) { + // Flags defined inside package objects have unexpected names. + require(!GlobalFlag.isEnclosedInPackageObject(getClass), + s"package object encloses flag definition: $getClass") + override protected[this] def parsingDone: Boolean = true private[this] lazy val propertyValue = @@ -115,6 +119,10 @@ abstract class GlobalFlag[T] private[app] ( object GlobalFlag { + private def isEnclosedInPackageObject(klass: Class[_]): Boolean = + klass.getEnclosingClass != null && + klass.getEnclosingClass.getSimpleName == "package" + private[app] def get(flagName: String): Option[Flag[_]] = { val className = if (!flagName.endsWith("$")) flagName + "$" else flagName try { diff --git a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala index 3c0ad02136..0f96629fa0 100644 --- a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala @@ -7,7 +7,7 @@ object MyGlobalFlagNoDefault extends GlobalFlag[Int]("a global test flag with no object MyGlobalBooleanFlag extends GlobalFlag[Boolean](false, "a boolean flag") class GlobalFlagTest extends FunSuite { - val flagSet = Set(MyGlobalFlag, MyGlobalBooleanFlag, MyGlobalFlagNoDefault, PackageObjectTest) + val flagSet = Set(MyGlobalFlag, MyGlobalBooleanFlag, MyGlobalFlagNoDefault) test("GlobalFlag.get") { assert(MyGlobalBooleanFlag.get.isEmpty) @@ -68,7 +68,6 @@ class GlobalFlagTest extends FunSuite { val mockClassLoader = new MockClassLoader(getClass.getClassLoader) val flags = GlobalFlag.getAll(mockClassLoader) assert(flags.toSet == flagSet) - assert(flags.exists(_.help.equals("a package object test flag"))) } private class MockClassLoader(realClassLoader: ClassLoader) extends ClassLoader(realClassLoader) { diff --git a/util-app/src/test/scala/com/twitter/app/package.scala b/util-app/src/test/scala/com/twitter/app/package.scala deleted file mode 100644 index d25cfd23dc..0000000000 --- a/util-app/src/test/scala/com/twitter/app/package.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter - -package object app { - object PackageObjectTest extends GlobalFlag[Boolean](true, "a package object test flag") -} From 59a8c91c8a89f2d21c5efff59f8617a428badaef Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 13 Aug 2019 18:00:14 +0000 Subject: [PATCH 191/796] util/stats: Fix the flaky test in CumulativeGaugeTest Problem We have this test failing in Travis CI even though it is marked as SKIP_FLAKY. After we forked JVM for both sbt console and Test for util, the JAVA_OPTIONS got dropped when running tests, so sbt test doesn't skip flaky tests anymore. The failing test is flaky because `System.gc()` starts a new thread and cleanRef happens sometimes before gc finish. Solution We have only one test in util marked SKIP_FLAKY, fix it by adding `eventually{}`. JIRA Issues: CSL-8660 Differential Revision: https://phabricator.twitter.biz/D354777 --- .../finagle/stats/CumulativeGaugeTest.scala | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala index a9d6cf5dd8..5b931cd784 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala @@ -3,8 +3,9 @@ package com.twitter.finagle.stats import java.util.concurrent.Executor import java.util.concurrent.atomic.AtomicInteger import org.scalatest.FunSuite +import org.scalatest.concurrent.{Eventually, IntegrationPatience} -class CumulativeGaugeTest extends FunSuite { +class CumulativeGaugeTest extends FunSuite with Eventually with IntegrationPatience { private[this] val sameThreadExecutor = new Executor { def execute(command: Runnable): Unit = command.run() @@ -53,21 +54,22 @@ class CumulativeGaugeTest extends FunSuite { assert(0 == gauge.numDeregisters.get) } - if (!sys.props.contains("SKIP_FLAKY")) - test( - "a CumulativeGauge with size = 1 should deregister after a System.gc when no references are held onto, after enough gets" - ) { - val gauge = new TestGauge() - var added = gauge.addGauge { 1.0f } - assert(0 == gauge.numDeregisters.get) + test( + "a CumulativeGauge with size = 1 should deregister after a System.gc when no references are held onto, after enough gets" + ) { + val gauge = new TestGauge() + var added = gauge.addGauge { 1.0f } + assert(0 == gauge.numDeregisters.get) - added = null - System.gc() - gauge.cleanRefs() + added = null + System.gc() + eventually { + gauge.cleanRefs() assert(gauge.getValue == 0.0f) assert(gauge.numDeregisters.get > 0) } + } test("a CumulativeGauge should sum values across all registered gauges") { val gauge = new TestGauge() From 116988077ed1452c749286087195ce984262083a Mon Sep 17 00:00:00 2001 From: Travis Brown Date: Tue, 13 Aug 2019 18:36:25 +0000 Subject: [PATCH 192/796] Make flatMap on AsyncStream stack-safe Problem The recent `com.twitter.util.BypassScheduler` optimization breaks stack-safety for monadic recursion for `AsyncStream`. For example, on 19.7.0 or 19.8.0 (or 19.6.0 with `-Dcom.twitter.util.BypassScheduler=true`) the following will overflow, while it used to work just fine: ```scala import com.twitter.concurrent.AsyncStream def deep(i: Int): AsyncStream[Int] = if (i == 0) AsyncStream.of(i) else AsyncStream.of(i).flatMap(i => deep(i - 1)) com.twitter.util.Await.result(deep(10000).toSeq) ``` Solution After some discussion we decided the best solution would be to replace `Future#map` with `Future#flatMap` in the definition of `AsyncStream`'s `flatMap` in order to make it stack-safe again. Result `flatMap` can be used recursively again without worrying about stack overflows. Signed-off-by: Neuman Vong JIRA Issues: CSL-8643, CSL-8659 Differential Revision: https://phabricator.twitter.biz/D355163 --- .../com/twitter/concurrent/AsyncStream.scala | 11 ++++++++--- .../com/twitter/concurrent/AsyncStreamTest.scala | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 12695a2256..4f70b2edd0 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -216,13 +216,18 @@ sealed abstract class AsyncStream[+A] { /** * Map a function `f` over the elements in this stream and concatenate the * results. + * + * @note We use `flatMap` on `Future` instead of `map` to maintain + * stack-safety for monadic recursion. */ def flatMap[B](f: A => AsyncStream[B]): AsyncStream[B] = this match { case Empty => empty - case FromFuture(fa) => Embed(fa.map(f)) - case Cons(fa, more) => Embed(fa.map(f)) ++ more().flatMap(f) - case Embed(fas) => Embed(fas.map(_.flatMap(f))) + case FromFuture(fa) => Embed(fa.flatMap(a => Future.value(f(a)))) + case Cons(fa, more) => + Embed(fa.flatMap(a => Future.value(f(a)))) ++ more().flatMap(f) + case Embed(fas) => + Embed(fas.flatMap(as => Future.value(as.flatMap(f)))) } /** diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 8d3395069d..12dfc8dda9 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -735,6 +735,21 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } } + test(s"$impl: flatMap is stack-safe") { + val n = 10000 + + def tailRecM[A, B](a: A)(f: A => AsyncStream[Either[A, B]]): AsyncStream[B] = + f(a).flatMap { + case Right(b) => AsyncStream.of(b) + case Left(a) => tailRecM(a)(f) + } + + val stream = tailRecM(0) { i => + AsyncStream.of(if (i < n) Left(i + 1) else Right(i)) + } + + assert(Await.result(stream.toSeq) == Seq(n)) + } } } From 4649dd1ebe615ecdc5c78a72e4b4007708d1002f Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Tue, 13 Aug 2019 19:54:33 +0000 Subject: [PATCH 193/796] util-core: Add More Doc to Explain Reader#map Behavior Problem The "child" Readers' behaviors in `Reader#map` and `Reader#flatMap` are not well documented. Solution/Result Clarify this aspect of the documentation. JIRA Issues: CSL-8544 Differential Revision: https://phabricator.twitter.biz/D353329 --- util-core/src/main/scala/com/twitter/io/Reader.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index fd9f650493..15a9da1f6e 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -107,12 +107,20 @@ trait Reader[+A] { self => /** * Construct a new Reader by applying `f` to every item read from this Reader * @param f the function constructs a new Reader[B] from the value of this Reader.read + * + * @note All operations of the new Reader will be in sync with self Reader. Discarding one Reader + * will discard the other Reader. When one Reader's onClose resolves, the other Reader's + * onClose will be resolved immediately with the same value. */ final def flatMap[B](f: A => Reader[B]): Reader[B] = Reader.flatten(map(f)) /** * Construct a new Reader by applying `f` to every item read from this Reader * @param f the function transforms data of type A to B + * + * @note All operations of the new Reader will be in sync with self Reader. Discarding one Reader + * will discard the other Reader. When one Reader's onClose resolves, the other Reader's + * onClose will be resolved immediately with the same value. */ final def map[B](f: A => B): Reader[B] = new Reader[B] { def read(): Future[Option[B]] = self.read().map(oa => oa.map(f)) From 0ec66f3383ebab9ebc7c008a4ba8892aacc239b0 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Wed, 14 Aug 2019 01:28:57 +0000 Subject: [PATCH 194/796] util-core: Add Clarification for Reader#flatten Problem The child reader's behavior in `Reader#flatten` is not well documented Solution/Result Add more clarifications for `Reader#flatten` behavior in scala doc JIRA Issues: CSL-8673 Differential Revision: https://phabricator.twitter.biz/D355506 --- .../src/main/scala/com/twitter/io/Reader.scala | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 15a9da1f6e..e66ee60005 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -130,6 +130,13 @@ trait Reader[+A] { self => /** * Converts a `Reader[Reader[B]]` into a `Reader[B]` + * + * @define flattenBehavior + * @note All operations of the new Reader will be in sync with the outermost Reader. Discarding + * one Reader will discard the other Reader. When one Reader's onClose resolves, the other + * Reader's onClose will be resolved immediately with the same value. + * The subsequent readers are unmanaged, the caller is responsible for discarding those + * when abandoned. */ def flatten[B](implicit ev: A <:< Reader[B]): Reader[B] = Reader.flatten(this.asInstanceOf[Reader[Reader[B]]]) @@ -379,10 +386,10 @@ object Reader { } /** - * Convenient abstraction to read from a stream (Reader) of Readers as if - * it were a single Reader. The subsequent readers are unmanaged, the caller is - * responsible for discarding those when abandoned. + * Convenient abstraction to read from a stream (Reader) of Readers as if it were a single Reader. * @param readers A Reader holds a stream of Reader[A] + * + * $flattenBehavior */ def flatten[A](readers: Reader[Reader[A]]): Reader[A] = new Reader[A] { self => // access currentReader and curReaderClosep are synchronized on `self` From e451e3b28dd96e1ebc72aebe679f7c2c0f00fd26 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 14 Aug 2019 02:37:02 +0000 Subject: [PATCH 195/796] util-core: Change three methods to take scala.collection.Seq Problem Future#collect, collectToTry and join were taking an Iterable which cause binary Incompatibility Solution This change is due to scala 2.13's default Seq is pointing to scala.collection.immutable.Seq, change those signatures to scala.collection.Seq. JIRA Issues: CSL-8677 Differential Revision: https://phabricator.twitter.biz/D355403 --- CHANGELOG.rst | 3 +++ util-core/src/main/scala/com/twitter/util/Future.scala | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9a380b8441..8553384411 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,6 +35,9 @@ Breaking API Changes * util-core: (not breaking) `c.t.util.Future.join` and `c.t.util.Future.collect` now take `Iterable[Future[A]]` other than Seq. ``PHAB_ID=D333021`` +* util-core: Revert the change above, in `c.t.util.Future`, `collect`, `collectToTry` and `join` + take `scala.collection.Seq[Future[A]]`. ``PHAB_ID=D355403`` + * util-core: `com.twitter.util.Event#build` now builds a Seq of events. `Event#buildAny` builds against any collection of events. ``PHAB_ID=D333021`` diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 4b2a159872..929a8be933 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -5,6 +5,7 @@ import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicReferenc import java.util.concurrent.{CancellationException, TimeUnit, Future => JavaFuture} import java.util.{List => JList} import scala.collection.compat.immutable.ArraySeq +import scala.collection.{Seq => AnySeq} import scala.runtime.NonLocalReturnControl import scala.util.control.NoStackTrace @@ -263,7 +264,7 @@ object Future { * @see [[collectToTry]] if you want to be able to see the results of each * `Future` regardless of if they succeed or fail. */ - def join[A](fs: Iterable[Future[A]]): Future[Unit] = { + def join[A](fs: AnySeq[Future[A]]): Future[Unit] = { if (fs.isEmpty) Future.Unit else { val size = fs.size @@ -1109,7 +1110,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * @see [[join]] if you are not interested in the results of the individual * `Futures`, only when they are complete. */ - def collect[A](fs: Iterable[Future[A]]): Future[Seq[A]] = + def collect[A](fs: AnySeq[Future[A]]): Future[Seq[A]] = if (fs.isEmpty) emptySeq else { val result = new CollectPromise[A](fs) @@ -1162,7 +1163,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * @param fs a sequence of Futures * @return a `Future[Seq[Try[A]]]` containing the collected values from fs. */ - def collectToTry[A](fs: Iterable[Future[A]]): Future[Seq[Try[A]]] = { + def collectToTry[A](fs: AnySeq[Future[A]]): Future[Seq[Try[A]]] = { //unroll cases 0 and 1 if (fs.isEmpty) Nil else { @@ -1411,7 +1412,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( sizeThreshold: Int, timeThreshold: Duration = Duration.Top, sizePercentile: => Float = 1.0f - )(f: scala.collection.Seq[In] => Future[Seq[Out]] + )(f: AnySeq[In] => Future[Seq[Out]] )( implicit timer: Timer ): Batcher[In, Out] = { From 950cd24c9359d216145a4327c30d55d0d7bcfa58 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 15 Aug 2019 16:45:03 +0000 Subject: [PATCH 196/796] Move JVM flags from the sbt script to build.sbt Problem / Solution We should be defining the JVM flags as part of the build, not part of the sbt script, so lets move them. JIRA Issues: CSL-8245 Differential Revision: https://phabricator.twitter.biz/D354562 --- .travis.yml | 3 --- build.sbt | 44 +++++++++++++++++++++++++++++++++++++++++++- sbt | 13 ------------- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index f589ffee02..7b8d907f92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,6 @@ git: depth: 3 quiet: true -env: - - JAVA_OPTS="-DSKIP_FLAKY=true -Dsbt.log.noformat=true" - # These directories are cached to S3 at the end of the build cache: directories: diff --git a/build.sbt b/build.sbt index 4d381a95c2..3aba50711d 100644 --- a/build.sbt +++ b/build.sbt @@ -20,6 +20,38 @@ val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion +def travisTestJavaOptions: Seq[String] = { + // We have some custom configuration for the Travis environment + // https://docs.travis-ci.com/user/environment-variables/#default-environment-variables + val travisBuild = sys.env.getOrElse("TRAVIS", "false").toBoolean + if (travisBuild) { + Seq( + "-DSKIP_FLAKY=true", + "-DSKIP_FLAKY_TRAVIS=true", + "-Dsbt.log.noformat=true" + ) + } else { + Seq( + "-DSKIP_FLAKY=true" + ) + } +} + +def gcJavaOptions: Seq[String] = { + Seq( + "-XX:+UseParNewGC", + "-XX:+UseConcMarkSweepGC", + "-XX:+CMSParallelRemarkEnabled", + "-XX:+CMSClassUnloadingEnabled", + "-XX:ReservedCodeCacheSize=128m", + "-XX:SurvivorRatio=128", + "-XX:MaxTenuringThreshold=0", + "-Xss8M", + "-Xms512M", + "-Xmx2G" + ) +} + val defaultProjectSettings = Seq( scalaVersion := "2.12.8", crossScalaVersions := Seq("2.11.12", "2.12.8", "2.13.0") @@ -37,7 +69,7 @@ val baseSettings = Seq( "org.mockito" % "mockito-all" % "1.10.19" % "test", "org.scalatest" %% "scalatest" % "3.0.8" % "test" ), - fork in Test := true, + fork in Test := true, // We have to fork to get the JavaOptions // Workaround for cross building HealthyQueue.scala, which is not compatible between // 2.12- with 2.13+. unmanagedSourceDirectories in Compile += { @@ -68,6 +100,16 @@ val baseSettings = Seq( javacOptions ++= Seq("-Xlint:unchecked", "-source", "1.8", "-target", "1.8"), javacOptions in doc := Seq("-source", "1.8"), + javaOptions ++= Seq( + "-Djava.net.preferIPv4Stack=true", + "-XX:+AggressiveOpts", + "-server" + ), + + javaOptions ++= gcJavaOptions, + + javaOptions in Test ++= travisTestJavaOptions, + // -a: print stack traces for failing asserts testOptions += Tests.Argument(TestFrameworks.JUnit, "-a"), diff --git a/sbt b/sbt index 9e21a00f08..d18b842cd2 100755 --- a/sbt +++ b/sbt @@ -24,17 +24,4 @@ fi java -ea \ $SBT_OPTS \ $JAVA_OPTS \ - -Djava.net.preferIPv4Stack=true \ - -XX:+AggressiveOpts \ - -XX:+UseParNewGC \ - -XX:+UseConcMarkSweepGC \ - -XX:+CMSParallelRemarkEnabled \ - -XX:+CMSClassUnloadingEnabled \ - -XX:ReservedCodeCacheSize=128m \ - -XX:SurvivorRatio=128 \ - -XX:MaxTenuringThreshold=0 \ - -Xss8M \ - -Xms512M \ - -Xmx2G \ - -server \ -jar $sbtjar "$@" From 0b65a3152bae9a7ac9ca411357428c855bcf5469 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 15 Aug 2019 18:21:05 +0000 Subject: [PATCH 197/796] Twitter-oss: Prepare OSS util for release 19.8.1 Problem We want to release the next versions of our Twitter OSS util 19.8.1 Solution Prepare util for their next releases. Result We will need to temporarily upgrade all versions for passing build. But we will only publish util 19.8.1. post-release will reset all versions back to 19.9.0-SNAPSHOT. JIRA Issues: CSL-8688 Differential Revision: https://phabricator.twitter.biz/D356166 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8553384411..c160eeedb5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.8.1 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index a881747169..b1768a2090 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.8.0" +val collUtils = "com.twitter" %% "util-collection" % "19.8.1" ``` # Units diff --git a/build.sbt b/build.sbt index 3aba50711d..a2adad4bfe 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.9.0-SNAPSHOT" +val releaseVersion = "19.8.1" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 3c52a6d5c82614a5922f4fa528a66ad9f6caf413 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 15 Aug 2019 21:02:02 +0000 Subject: [PATCH 198/796] Twitter-oss: update OSS libraries post-release to 19.9.0 Problem Update to the next SNAPSHOT version of our Twitter OSS libraries 19.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Result We have only published a patch release for util 19.8.1 JIRA Issues: CSL-8688 Differential Revision: https://phabricator.twitter.biz/D356784 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b1768a2090..3d84a810f7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index a2adad4bfe..3aba50711d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.8.1" +val releaseVersion = "19.9.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From eeb0e947f9456075af34209cd09dfffe04cf0a58 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Fri, 16 Aug 2019 19:01:48 +0000 Subject: [PATCH 199/796] util-core: Remove c.t.u.BoundedStack Problem / Solution `BoundedStack` is unused internally and really old code. It's worth deleting to reduce our API surface area. JIRA Issues: CSL-8454 Differential Revision: https://phabricator.twitter.biz/D357338 --- CHANGELOG.rst | 5 + .../scala/com/twitter/util/BoundedStack.scala | 96 ----------------- .../com/twitter/util/BoundedStackTest.scala | 102 ------------------ 3 files changed, 5 insertions(+), 198 deletions(-) delete mode 100644 util-core/src/main/scala/com/twitter/util/BoundedStack.scala delete mode 100644 util-core/src/test/scala/com/twitter/util/BoundedStackTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c160eeedb5..9ed2df688a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: BoundedStack is unused and really old code. Delete it. ``PHAB_ID=D357338`` + 19.8.1 ------ diff --git a/util-core/src/main/scala/com/twitter/util/BoundedStack.scala b/util-core/src/main/scala/com/twitter/util/BoundedStack.scala deleted file mode 100644 index 5d4a9b0fed..0000000000 --- a/util-core/src/main/scala/com/twitter/util/BoundedStack.scala +++ /dev/null @@ -1,96 +0,0 @@ -package com.twitter.util - -import scala.reflect.ClassTag -import scala.collection.Seq - -/** - * A "stack" with a bounded size. If you push a new element on the top - * when the stack is full, the oldest element gets dropped off the bottom. - */ -class BoundedStack[A: ClassTag](val maxSize: Int) extends Seq[A] { - private val array = new Array[A](maxSize) - private var top = 0 - private var count_ = 0 - - override def length: Int = count_ - - def clear(): Unit = { - top = 0 - count_ = 0 - } - - /** - * Gets the element from the specified index in constant time. - */ - def apply(i: Int): A = { - if (i >= count_) throw new IndexOutOfBoundsException(i.toString) - else array((top + i) % maxSize) - } - - /** - * Pushes an element, possibly forcing out the oldest element in the stack. - */ - def +=(elem: A): Unit = { - top = if (top == 0) maxSize - 1 else top - 1 - array(top) = elem - if (count_ < maxSize) count_ += 1 - } - - /** - * Inserts an element 'i' positions down in the stack. An 'i' value - * of 0 is the same as calling this += elem. This is a O(n) operation - * as elements need to be shifted around. - */ - def insert(i: Int, elem: A): Unit = { - if (i == 0) this += elem - else if (i > count_) throw new IndexOutOfBoundsException(i.toString) - else if (i == count_) { - array((top + i) % maxSize) = elem - count_ += 1 - } else { - val swapped = this(i) - this(i) = elem - insert(i - 1, swapped) - } - } - - /** - * Replaces an element in the stack. - */ - def update(index: Int, elem: A): Unit = { - array((top + index) % maxSize) = elem - } - - /** - * Adds multiple elements, possibly overwriting the oldest elements in - * the stack. If the given iterable contains more elements that this - * stack can hold, then only the last maxSize elements will end up in - * the stack. - */ - def ++=(iter: Iterable[A]): Unit = { - for (elem <- iter) this += elem - } - - /** - * Removes the top element in the stack. - */ - def pop: A = { - if (count_ == 0) throw new NoSuchElementException - else { - val res = array(top) - top = (top + 1) % maxSize - count_ -= 1 - res - } - } - - override def iterator: Iterator[A] = new Iterator[A] { - var idx = 0 - def hasNext = idx != count_ - def next = { - val res = apply(idx) - idx += 1 - res - } - } -} diff --git a/util-core/src/test/scala/com/twitter/util/BoundedStackTest.scala b/util-core/src/test/scala/com/twitter/util/BoundedStackTest.scala deleted file mode 100644 index fa469eb13d..0000000000 --- a/util-core/src/test/scala/com/twitter/util/BoundedStackTest.scala +++ /dev/null @@ -1,102 +0,0 @@ -package com.twitter.util - -import org.scalatest.WordSpec - -class BoundedStackTest extends WordSpec { - "BoundedStack" should { - "empty" in { - val buf = new BoundedStack[String](4) - assert(buf.length == 0) - assert(buf.size == 0) - assert(buf.isEmpty == true) - intercept[IndexOutOfBoundsException] { - buf(0) - } - intercept[NoSuchElementException] { - buf.pop - } - assert(buf.iterator.hasNext == false) - } - - "handle single element" in { - val buf = new BoundedStack[String](4) - buf += "a" - assert(buf.size == 1) - assert(buf(0) == "a") - assert(buf.toList == List("a")) - } - - "handle multiple element" in { - val buf = new BoundedStack[String](4) - buf ++= List("a", "b", "c") - assert(buf.size == 3) - assert(buf(0) == "c") - assert(buf(1) == "b") - assert(buf(2) == "a") - assert(buf.toList == List("c", "b", "a")) - assert(buf.pop == "c") - assert(buf.size == 2) - assert(buf.pop == "b") - assert(buf.size == 1) - assert(buf.pop == "a") - assert(buf.size == 0) - } - - "handle overwrite/rollover" in { - val buf = new BoundedStack[String](4) - buf ++= List("a", "b", "c", "d", "e", "f") - assert(buf.size == 4) - assert(buf(0) == "f") - assert(buf.toList == List("f", "e", "d", "c")) - } - - "handle update" in { - val buf = new BoundedStack[String](4) - buf ++= List("a", "b", "c", "d", "e", "f") - for (i <- 0 until buf.size) { - val old = buf(i) - val updated = old + "2" - buf(i) = updated - assert(buf(i) == updated) - } - assert(buf.toList == List("f2", "e2", "d2", "c2")) - } - - "insert at 0 is same as +=" in { - val buf = new BoundedStack[String](3) - buf.insert(0, "a") - assert(buf.size == 1) - assert(buf(0) == "a") - buf.insert(0, "b") - assert(buf.size == 2) - assert(buf(0) == "b") - assert(buf(1) == "a") - buf.insert(0, "c") - assert(buf(0) == "c") - assert(buf(1) == "b") - assert(buf(2) == "a") - buf.insert(0, "d") - } - - "insert at count pushes to bottom" in { - val buf = new BoundedStack[String](3) - buf.insert(0, "a") - buf.insert(1, "b") - buf.insert(2, "c") - assert(buf(0) == "a") - assert(buf(1) == "b") - assert(buf(2) == "c") - } - - "insert > count throws exception" in { - val buf = new BoundedStack[String](3) - intercept[IndexOutOfBoundsException] { - buf.insert(1, "a") - } - buf.insert(0, "a") - intercept[IndexOutOfBoundsException] { - buf.insert(2, "b") - } - } - } -} From f4e565993b4bbc6e458674fd332259a9b6ac4e48 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Mon, 19 Aug 2019 20:35:33 +0000 Subject: [PATCH 200/796] util-logging: Remove ScribeHandler and ScribeHandlers Problem / Solution ScribeHandler is an historically poor representation of how we build clients and we would like to prevent further open source use. Additionally, its tie in directly to util-logging is problematic as the use of util-logging is discouraged and will eventually be removed. Let's remove `ScribeHandler` and `ScribeHandlers` from util-logging. Result `ScribeHandler` and `ScribeHandlers` are removed from util-logging. Users are encouraged to use slf4j for logging. However, if a util-logging integrated `ScribeHandler` is still required, users can either build their own Finagle-based scribe client as in `ScribeRawZipkinTracer` in finagle-zipkin-scribe, or copy the old `ScribeHandler` implementation directly into their code. JIRA Issues: CSL-8696 Differential Revision: https://phabricator.twitter.biz/D357008 --- CHANGELOG.rst | 6 + .../com/twitter/logging/ScribeHandler.scala | 598 ------------------ .../com/twitter/logging/ScribeHandlers.scala | 26 - .../twitter/logging/ThrottledHandler.scala | 1 - .../ScribeHandlersCompilationTest.java | 14 - .../com/twitter/logging/LoggerTest.scala | 15 - .../com/twitter/logging/LoggingTest.scala | 59 -- .../twitter/logging/ScribeHandlerTest.scala | 325 ---------- 8 files changed, 6 insertions(+), 1038 deletions(-) delete mode 100644 util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala delete mode 100644 util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala delete mode 100644 util-logging/src/test/java/com/twitter/logging/ScribeHandlersCompilationTest.java delete mode 100644 util-logging/src/test/scala/com/twitter/logging/LoggingTest.scala delete mode 100644 util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9ed2df688a..7eda8339a3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,12 @@ Breaking API Changes * util-core: BoundedStack is unused and really old code. Delete it. ``PHAB_ID=D357338`` +* util-logging: `com.twitter.logging.ScribeHandler` and `com.twitter.logging.ScribeHandlers` have + been removed. Users are encouraged to use slf4j for logging. However, if a util-logging integrated + ScribeHandler is still required, users can either build their own Finagle-based scribe client as + in `ScribeRawZipkinTracer` in finagle-zipkin-scribe, or copy the old `ScribeHandler` + implementation directly into their code. ``PHAB_ID=D357008`` + 19.8.1 ------ diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala deleted file mode 100644 index 3e6ce9e203..0000000000 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandler.scala +++ /dev/null @@ -1,598 +0,0 @@ -/* - * Copyright 2010 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter.logging - -import java.io.IOException -import java.net._ -import java.nio.{ByteBuffer, ByteOrder} -import java.nio.charset.StandardCharsets.ISO_8859_1 -import java.util.concurrent.atomic.AtomicLong -import java.util.concurrent.{ArrayBlockingQueue, LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit} -import java.util.{Arrays, logging => javalog} -import com.twitter.concurrent.NamedPoolThreadFactory -import com.twitter.conversions.StringOps._ -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.{NullStatsReceiver, StatsReceiver} -import com.twitter.util.{Duration, Stopwatch, Time} -import scala.util.control.NonFatal - -object ScribeHandler { - private sealed trait ServerType - private case object Unknown extends ServerType - private case object Archaic extends ServerType - private case object Modern extends ServerType - - val OK = 0 - val TRY_LATER = 1 - - val DefaultHostname = "localhost" - val DefaultPort = 1463 - val DefaultCategory = "scala" - val DefaultBufferTime = 100.milliseconds - val DefaultConnectBackoff = 15.seconds - val DefaultMaxMessagesPerTransaction = 1000 - val DefaultMaxMessagesToBuffer = 10000 - val DefaultStatsReportPeriod = 5.minutes - val log = Logger.get(getClass) - - /** - * Generates a HandlerFactory that returns a ScribeHandler. - * - * @note ScribeHandler is usually used to write structured binary data. - * When used in this way, wrapping this in other handlers, such as ThrottledHandler, - * which emit plain-text messages into the log, will corrupt the resulting data. - * - * @note Java users of this library can use one of the methods in [[ScribeHandlers]], - * so they need not provide all the default arguments. - * - * @param bufferTime - * send a scribe message no more frequently than this: - * - * @param connectBackoff - * don't connect more frequently than this (when the scribe server is down): - */ - def apply( - hostname: String = DefaultHostname, - port: Int = DefaultPort, - category: String = DefaultCategory, - bufferTime: Duration = DefaultBufferTime, - connectBackoff: Duration = DefaultConnectBackoff, - maxMessagesPerTransaction: Int = DefaultMaxMessagesPerTransaction, - maxMessagesToBuffer: Int = DefaultMaxMessagesToBuffer, - formatter: Formatter = new Formatter(), - level: Option[Level] = None, - statsReceiver: StatsReceiver = NullStatsReceiver - ): () => ScribeHandler = - () => - new ScribeHandler( - hostname, - port, - category, - bufferTime, - connectBackoff, - maxMessagesPerTransaction, - maxMessagesToBuffer, - formatter, - level, - statsReceiver - ) - - def apply( - hostname: String, - port: Int, - category: String, - bufferTime: Duration, - connectBackoff: Duration, - maxMessagesPerTransaction: Int, - maxMessagesToBuffer: Int, - formatter: Formatter, - level: Option[Level] - ): () => ScribeHandler = - apply( - hostname, - port, - category, - bufferTime, - connectBackoff, - maxMessagesPerTransaction, - maxMessagesToBuffer, - formatter, - level, - NullStatsReceiver - ) - - private[logging] val ScribePrefix: Array[Byte] = Array[Byte]( - // version 1, call, "Log", reqid=0 - 0x80.toByte, - 1, - 0, - 1, - 0, - 0, - 0, - 3, - 'L'.toByte, - 'o'.toByte, - 'g'.toByte, - 0, - 0, - 0, - 0, - // list of structs - 15, - 0, - 1, - 12 - ) - private[logging] val OldScribePrefix: Array[Byte] = Array[Byte]( - // (no version), "Log", reply, reqid=0 - 0, - 0, - 0, - 3, - 'L'.toByte, - 'o'.toByte, - 'g'.toByte, - 1, - 0, - 0, - 0, - 0, - // list of structs - 15, - 0, - 1, - 12 - ) - - private[logging] val ScribeReply: Array[Byte] = Array[Byte]( - // version 1, reply, "Log", reqid=0 - 0x80.toByte, - 1, - 0, - 2, - 0, - 0, - 0, - 3, - 'L'.toByte, - 'o'.toByte, - 'g'.toByte, - 0, - 0, - 0, - 0, - // int, fid 0, 0=ok - 8, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ) - private[logging] val OldScribeReply: Array[Byte] = Array[Byte]( - 0, - 0, - 0, - 20, - // (no version), "Log", reply, reqid=0 - 0, - 0, - 0, - 3, - 'L'.toByte, - 'o'.toByte, - 'g'.toByte, - 2, - 0, - 0, - 0, - 0, - // int, fid 0, 0=ok - 8, - 0, - 0, - 0, - 0, - 0, - 0, - 0 - ) -} - -/** - * NOTE: ScribeHandler is usually used to write structured binary data. - * When used in this way, wrapping this in other handlers, such as ThrottledHandler, - * which emit plain-text messages into the log, will corrupt the resulting data. - */ -class ScribeHandler( - hostname: String, - port: Int, - category: String, - bufferTime: Duration, - connectBackoff: Duration, - maxMessagesPerTransaction: Int, - maxMessagesToBuffer: Int, - formatter: Formatter, - level: Option[Level], - statsReceiver: StatsReceiver) - extends Handler(formatter, level) { - import ScribeHandler._ - - def this( - hostname: String, - port: Int, - category: String, - bufferTime: Duration, - connectBackoff: Duration, - maxMessagesPerTransaction: Int, - maxMessagesToBuffer: Int, - formatter: Formatter, - level: Option[Level] - ) = - this( - hostname, - port, - category, - bufferTime, - connectBackoff, - maxMessagesPerTransaction, - maxMessagesToBuffer, - formatter, - level, - NullStatsReceiver - ) - - private[this] val stats = new ScribeHandlerStats(statsReceiver) - - // it may be necessary to log errors here if scribe is down: - private val loggerName = getClass.getName - - private var lastConnectAttempt = Time.epoch - - @volatile private var _lastTransmission = Time.epoch - // visible for testing - private[logging] def updateLastTransmission(): Unit = _lastTransmission = Time.now - - private var socket: Option[Socket] = None - // visible for testing - private[logging] def setSocket(newSocket: Option[Socket]) = synchronized { socket = newSocket } - - private var serverType: ServerType = Unknown - - private def isArchaicServer() = { serverType == Archaic } - - // Could be rewritten using a simple Condition (await/notify) or producer/consumer - // with timed batching - private[logging] val flusher = { - val threadFactory = new NamedPoolThreadFactory("ScribeFlusher-" + category, true) - // should be 1, but this is a crude form of retry - val queue = new ArrayBlockingQueue[Runnable](5) - val rejectionHandler = new ThreadPoolExecutor.DiscardPolicy() - new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, queue, threadFactory, rejectionHandler) - } - - private[logging] val queue = new LinkedBlockingQueue[Array[Byte]](maxMessagesToBuffer) - - def queueSize: Int = queue.size() - - override def flush(): Unit = { - flusher.execute(new Runnable { - def run(): Unit = { sendBatch() } - }) - } - - private def connect(): Unit = { - if (!socket.isDefined) { - if (Time.now.since(lastConnectAttempt) > connectBackoff) { - try { - lastConnectAttempt = Time.now - socket = Some(new Socket(hostname, port)) - - stats.incrConnection() - serverType = Unknown - } catch { - case e: Exception => - log.error("Unable to open socket to scribe server at %s:%d: %s", hostname, port, e) - stats.incrConnectionFailure() - } - } else { - stats.incrConnectionSkipped() - } - } - } - - private def detectArchaicServer(): Unit = { - if (serverType != Unknown) return - - serverType = socket match { - case None => Unknown - case Some(s) => { - val outStream = s.getOutputStream() - - try { - val fakeMessageWithOldScribePrefix: Array[Byte] = { - val prefix = OldScribePrefix - val messageSize = prefix.length + 5 - val buffer = ByteBuffer.wrap(new Array[Byte](messageSize + 4)) - buffer.order(ByteOrder.BIG_ENDIAN) - buffer.putInt(messageSize) - buffer.put(prefix) - buffer.putInt(0) - buffer.put(0: Byte) - buffer.array - } - - outStream.write(fakeMessageWithOldScribePrefix) - readResponseExpecting(s, OldScribeReply) - - // Didn't get exception, so the server must be archaic. - log.debug("Scribe server is archaic; changing to old protocol for future requests.") - Archaic - } catch { - case NonFatal(_) => Modern - } - } - } - } - - // TODO we should send any remaining messages before the app terminates - // visible for tests - private[logging] def sendBatch(): Unit = { - synchronized { - connect() - detectArchaicServer() - socket.foreach { s => - val outStream = s.getOutputStream() - var remaining = queue.size - // try to send the log records in batch - // need to check if socket is closed due to exception - while (remaining > 0 && socket.isDefined) { - val count = maxMessagesPerTransaction min remaining - val buffer = makeBuffer(count) - stats.statBatchSize(count, buffer.capacity) - - val elapsed = Stopwatch.start() - try { - outStream.write(buffer.array) - val expectedReply = if (isArchaicServer()) OldScribeReply else ScribeReply - - readResponseExpecting(s, expectedReply) - - stats.sendRecordSuccess(count, elapsed().inMillis) - remaining -= count - } catch { - case e: Exception => - stats.sendRecordFailure(count, elapsed().inMillis) - log.error( - e, - "Failed to send %s %d log entries to scribe server at %s:%d", - category, - count, - hostname, - port - ) - closeSocket() - } - } - updateLastTransmission() - } - stats.log() - } - } - - private def readResponseExpecting(socket: Socket, expectedReply: Array[Byte]): Unit = { - var offset = 0 - - val inStream = socket.getInputStream() - - // read response: - val response = new Array[Byte](expectedReply.length) - while (offset < response.length) { - val n = inStream.read(response, offset, response.length - offset) - if (n < 0) { - throw new IOException("End of stream") - } - offset += n - } - if (!Arrays.equals(response, expectedReply)) { - throw new IOException("Error response from scribe server: " + response.hexlify) - } - } - - // should be private, make it visible to tests - private[logging] def makeBuffer(count: Int): ByteBuffer = { - val texts = for (i <- 0 until count) yield queue.poll() - val recordHeader = ByteBuffer.wrap(new Array[Byte](10 + category.length)) - recordHeader.order(ByteOrder.BIG_ENDIAN) - recordHeader.put(11: Byte) - recordHeader.putShort(1) - recordHeader.putInt(category.length) - recordHeader.put(category.getBytes(ISO_8859_1)) - recordHeader.put(11: Byte) - recordHeader.putShort(2) - - val prefix = if (isArchaicServer()) OldScribePrefix else ScribePrefix - val messageSize = (count * (recordHeader.capacity + 5)) + texts.foldLeft(0) { _ + _.length } + prefix.length + 5 - val buffer = ByteBuffer.wrap(new Array[Byte](messageSize + 4)) - buffer.order(ByteOrder.BIG_ENDIAN) - // "framing": - buffer.putInt(messageSize) - buffer.put(prefix) - buffer.putInt(count) - for (text <- texts) { - buffer.put(recordHeader.array) - buffer.putInt(text.length) - buffer.put(text) - buffer.put(0: Byte) - } - buffer.put(0: Byte) - buffer - } - - private def closeSocket(): Unit = { - synchronized { - socket.foreach { s => - try { - s.close() - } catch { - case _: Throwable => - } - } - socket = None - } - } - - override def close(): Unit = { - stats.incrCloses() - // TODO consider draining the flusher queue before returning - // nothing stops a pending flush from opening a new socket - closeSocket() - flusher.shutdown() - } - - override def publish(record: javalog.LogRecord): Unit = { - if (record.getLoggerName == loggerName) return - publish(getFormatter.format(record).getBytes("UTF-8")) - } - - def publish(record: Array[Byte]): Unit = { - stats.incrPublished() - if (!queue.offer(record)) stats.incrDroppedRecords() - if (Time.now.since(_lastTransmission) >= bufferTime) flush() - } - - override def toString = { - ("<%s level=%s hostname=%s port=%d scribe_buffer=%s " + - "scribe_backoff=%s scribe_max_packet_size=%d formatter=%s>").format( - getClass.getName, - getLevel, - hostname, - port, - bufferTime, - connectBackoff, - maxMessagesPerTransaction, - formatter.toString - ) - } - - private class ScribeHandlerStats(statsReceiver: StatsReceiver) { - private var _lastLogStats = Time.epoch - - private val sentRecords = new AtomicLong() - private val droppedRecords = new AtomicLong() - private val connectionFailure = new AtomicLong() - private val connectionSkipped = new AtomicLong() - - val totalSentRecords = statsReceiver.counter("sent_records") - val totalDroppedRecords = statsReceiver.counter("dropped_records") - val totalConnectionFailure = statsReceiver.counter("connection_failed") - val totalConnectionSkipped = statsReceiver.counter("connection_skipped") - val totalConnects = statsReceiver.counter("connects") - val totalPublished = statsReceiver.counter("published") - val totalCloses = statsReceiver.counter("closes") - val instances = statsReceiver.addGauge("instances") { 1 } - val unsentQueue = statsReceiver.addGauge("unsent_queue") { queueSize } - - val batchSizeBytesStat = statsReceiver.stat("batch_size_bytes") - val batchSizeCountStat = statsReceiver.stat("batch_size_messages") - - val successLatencyStat = statsReceiver.stat("success_latency_ms") - val failureLatencyStat = statsReceiver.stat("failure_latency_ms") - - /** - * Update stats for the number of records sent and the histogram of latency to scribe. - * @param count - * The number of new messages sent to scribe. - * @param latencyMs - * The latency of the successful send to scribe. - */ - def sendRecordSuccess(count: Int, latencyMs: Long): Unit = { - sentRecords.addAndGet(count) - totalSentRecords.incr(count) - successLatencyStat.add(latencyMs) - } - - /** - * Update stats for the number of records dropped and the latency histogram for failed requests - * to the scribe server. - * @param count - * The number of messages that failed to send to scribe and were dropped. - * @param latencyMs - * The latency of the failed request to scribe. - */ - def sendRecordFailure(count: Int, latencyMs: Long): Unit = { - incrDroppedRecords(count) - failureLatencyStat.add(latencyMs) - } - - /** - * Update stats for the number of records dropped. - * @param count - * The number of messages that were dropped. - */ - def incrDroppedRecords(count: Int = 1): Unit = { - droppedRecords.addAndGet(count) - totalDroppedRecords.incr(count) - } - - def incrConnectionFailure(): Unit = { - connectionFailure.incrementAndGet() - totalConnectionFailure.incr() - } - - def incrConnectionSkipped(): Unit = { - connectionSkipped.incrementAndGet() - totalConnectionSkipped.incr() - } - - def incrPublished(): Unit = totalPublished.incr() - - def incrConnection(): Unit = totalConnects.incr() - - def incrCloses(): Unit = totalCloses.incr() - - def statBatchSize(numMessages: Int, bytes: Int): Unit = { - batchSizeCountStat.add(numMessages) - batchSizeBytesStat.add(bytes) - } - - def log(): Unit = { - synchronized { - val period = Time.now.since(_lastLogStats) - if (period > ScribeHandler.DefaultStatsReportPeriod) { - val sent = sentRecords.getAndSet(0) - val dropped = droppedRecords.getAndSet(0) - val failed = connectionFailure.getAndSet(0) - val skipped = connectionSkipped.getAndSet(0) - ScribeHandler.log.debug( - "sent records: %d, per second: %d, dropped records: %d, reconnection failures: %d, reconnection skipped: %d", - sent, - sent / period.inSeconds, - dropped, - failed, - skipped - ) - - _lastLogStats = Time.now - } - } - } - } -} diff --git a/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala b/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala deleted file mode 100644 index 06a1a2fe98..0000000000 --- a/util-logging/src/main/scala/com/twitter/logging/ScribeHandlers.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.logging - -import com.twitter.finagle.stats.NullStatsReceiver -import com.twitter.logging.ScribeHandler._ - -object ScribeHandlers { - - /** - * For Java compatibility. Calls [[ScribeHandler.apply()]] method with - * the provided arguments, so Java users don't need to provide the - * other default arguments in the call. - */ - def apply(category: String, formatter: Formatter): () => ScribeHandler = - ScribeHandler.apply( - hostname = DefaultHostname, - port = DefaultPort, - category = category, - bufferTime = DefaultBufferTime, - connectBackoff = DefaultConnectBackoff, - maxMessagesPerTransaction = DefaultMaxMessagesPerTransaction, - maxMessagesToBuffer = DefaultMaxMessagesToBuffer, - formatter = formatter, - level = None, - statsReceiver = NullStatsReceiver - ) -} diff --git a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala index f649545779..3d72256cc1 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala @@ -32,7 +32,6 @@ object ThrottledHandler { * NOTE: ThrottledHandler emits plain-text messages regarding any throttling it does. * This means that using it to wrap any logger which you expect to produce easily parseable, * well-structured logs (as opposed to just plain text logs) will break your format. - * Specifically, wrapping ScribeHandler with ThrottledHandler is usually a bug. * * @param handler * Wrapped handler. diff --git a/util-logging/src/test/java/com/twitter/logging/ScribeHandlersCompilationTest.java b/util-logging/src/test/java/com/twitter/logging/ScribeHandlersCompilationTest.java deleted file mode 100644 index bb316b84cc..0000000000 --- a/util-logging/src/test/java/com/twitter/logging/ScribeHandlersCompilationTest.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.twitter.logging; - -import scala.Function0; - -/** - * Basic compilation test for calling {{{@link ScribeHandlers}}} - * methods from java - */ -public class ScribeHandlersCompilationTest { - public void testScribeHandlers() { - Function0 scribeHandlerFunc = - ScribeHandlers.apply("category", BareFormatter$.MODULE$); - } -} diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index ef93b3b6c4..354b3c01f6 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -351,15 +351,6 @@ class LoggerTest extends WordSpec with TempFolder with BeforeAndAfter { node = "w3c", level = Some(Level.OFF), useParents = false - ) :: LoggerFactory( - node = "stats", - level = Some(Level.INFO), - useParents = false, - handlers = ScribeHandler( - formatter = BareFormatter, - maxMessagesToBuffer = 100, - category = "cuckoo_json" - ) :: Nil ) :: LoggerFactory( node = "bad_jobs", level = Some(Level.INFO), @@ -373,7 +364,6 @@ class LoggerTest extends WordSpec with TempFolder with BeforeAndAfter { Logger.configure(factories) assert(Logger.get("").getLevel == Level.INFO) assert(Logger.get("w3c").getLevel == Level.OFF) - assert(Logger.get("stats").getLevel == Level.INFO) assert(Logger.get("bad_jobs").getLevel == Level.INFO) try { Logger.get("").getHandlers()(0).asInstanceOf[ThrottledHandler] @@ -391,11 +381,6 @@ class LoggerTest extends WordSpec with TempFolder with BeforeAndAfter { case _: ClassCastException => fail("not a FileHandler") } assert(Logger.get("w3c").getHandlers().size == 0) - try { - Logger.get("stats").getHandlers()(0).asInstanceOf[ScribeHandler] - } catch { - case _: ClassCastException => fail("not a ScribeHandler") - } try { Logger.get("bad_jobs").getHandlers()(0).asInstanceOf[FileHandler] } catch { diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggingTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggingTest.scala deleted file mode 100644 index d020deb837..0000000000 --- a/util-logging/src/test/scala/com/twitter/logging/LoggingTest.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2010 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter.logging - -import com.twitter.app.App -import org.scalatest.{BeforeAndAfterEach, BeforeAndAfterAll, FunSuite} - -class LoggingTest extends FunSuite with BeforeAndAfterEach with BeforeAndAfterAll { - // ensure this is initialized--we need to do something with it or scala - // complains about purity - require(ScribeHandler.log != null) - - object TestLoggingApp extends App with Logging { - override def handlers = ScribeHandler() :: super.handlers - } - - object TestLoggingWithConfigureApp extends App with Logging { - override def handlers = ScribeHandler() :: super.handlers - override def configureLoggerFactories(): Unit = {} - } - - test("TestLoggingApp should have one factory with two log handlers") { - TestLoggingApp.main(Array.empty) - assert(TestLoggingApp.loggerFactories.size == 1) - assert(TestLoggingApp.loggerFactories.head.handlers.size == 2) - // Logger is configured with two handlers, and has one logger - assert(Logger.iterator.size == 1) - } - - test("TestLoggingWithConfigureApp should set up a Logger with no Loggers") { - TestLoggingWithConfigureApp.main(Array.empty) - assert(TestLoggingApp.loggerFactories.size == 1) - assert(TestLoggingWithConfigureApp.loggerFactories.head.handlers.size == 2) - // Logger is not configured with any handler/logger - assert(Logger.iterator.isEmpty) - } - - override protected def afterAll(): Unit = { - Logger.reset() - } - - override protected def beforeEach(): Unit = { - Logger.reset() - } -} diff --git a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala deleted file mode 100644 index 8b7672f781..0000000000 --- a/util-logging/src/test/scala/com/twitter/logging/ScribeHandlerTest.scala +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright 2010 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter -package logging - -import com.twitter.conversions.StringOps._ -import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.InMemoryStatsReceiver -import com.twitter.util.{RandomSocket, Time} -import java.io.{ByteArrayInputStream, ByteArrayOutputStream} -import java.net.Socket -import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.{ - ArrayBlockingQueue, - RejectedExecutionHandler, - ThreadPoolExecutor, - TimeUnit -} -import java.util.{logging => javalog} -import org.mockito.Mockito._ -import org.mockito.invocation.InvocationOnMock -import org.mockito.stubbing.Answer -import org.scalatest.{BeforeAndAfter, WordSpec} -import org.scalatest.concurrent.Eventually -import org.scalatest.time.{Millis, Minutes, Span} -import org.scalatestplus.mockito.MockitoSugar - -class ScribeHandlerTest extends WordSpec with BeforeAndAfter with Eventually with MockitoSugar { - val record1 = new javalog.LogRecord(Level.INFO, "This is a message.") - record1.setMillis(1206769996722L) - record1.setLoggerName("hello") - val record2 = new javalog.LogRecord(Level.INFO, "This is another message.") - record2.setLoggerName("hello") - record2.setMillis(1206769996722L) - - // Ensure we get a free port. This assumes this port won't be reallocated - // while test is running even though SO_REUSEADDR is true - val portWithoutListener = RandomSocket.nextPort() - - "ScribeHandler" should { - - "build a scribe RPC call" in { - Time.withCurrentTimeFrozen { _ => - val scribe = ScribeHandler( - // Hack to make sure that the buffer doesn't get flushed. - port = portWithoutListener, - bufferTime = 100.milliseconds, - maxMessagesToBuffer = 10000, - formatter = new Formatter(timezone = Some("UTC")), - category = "test", - level = Some(Level.DEBUG) - ).apply() - - scribe.updateLastTransmission() - scribe.publish(record1) - scribe.publish(record2) - - assert(scribe.queue.size == 2) - assert( - scribe.makeBuffer(2).array.hexlify == ("000000b080010001000000034c6f67000000000f0001" + - "0c000000020b000100000004746573740b0002000000" + - "36494e46205b32303038303332392d30353a35333a31" + - "362e3732325d2068656c6c6f3a205468697320697320" + - "61206d6573736167652e0a000b000100000004746573" + - "740b00020000003c494e46205b32303038303332392d" + - "30353a35333a31362e3732325d2068656c6c6f3a2054" + - "68697320697320616e6f74686572206d657373616765" + - "2e0a0000") - ) - } - } - - "be able to log binary data" in { - Time.withCurrentTimeFrozen { _ => - val scribe = ScribeHandler( - port = portWithoutListener, - bufferTime = 100.milliseconds, - maxMessagesToBuffer = 10000, - formatter = new Formatter(timezone = Some("UTC")), - category = "test", - level = Some(Level.DEBUG) - ).apply() - - val bytes = Array[Byte](1, 2, 3, 4, 5) - - scribe.updateLastTransmission() - scribe.publish(bytes) - - assert(scribe.queue.peek() == bytes) - } - } - - "throw away log messages if scribe is too busy" in { - val statsReceiver = new InMemoryStatsReceiver - - val scribe = ScribeHandler( - port = portWithoutListener, - bufferTime = 5.seconds, - maxMessagesToBuffer = 1, - formatter = BareFormatter, - category = "test", - statsReceiver = statsReceiver - ).apply() - - scribe.updateLastTransmission() - scribe.publish(record1) - scribe.publish(record2) - - assert(statsReceiver.counter("dropped_records")() == 1l) - // Failures that never attempt to write to scribe do not have a latency - assert(statsReceiver.stat("failure_latency_ms")().size == 0) - assert(statsReceiver.counter("sent_records")() == 0l) - assert(statsReceiver.stat("failure_latency_ms")().size == 0) - } - - "have backoff on connection errors" in { - val statsReceiver = new InMemoryStatsReceiver - - // Use a mock ThreadPoolExecutor to force syncronize execution, - // to ensure reliably test connection backoff. - class MockThreadPoolExecutor - extends ThreadPoolExecutor( - 1, - 1, - 0L, - TimeUnit.MILLISECONDS, - new ArrayBlockingQueue[Runnable](5) - ) { - override def execute(command: Runnable): Unit = command.run() - } - - // In a loaded test environment, we can't reliably assume the port is not used, - // although the test just checked a short moment ago. - // Solution: try multiple times until it gets a connection failure. - def scribeWithConnectionFailure(retries: Int): ScribeHandler = { - val scribe = new ScribeHandler( - hostname = "localhost", - port = portWithoutListener, - category = "test", - bufferTime = 5.seconds, - connectBackoff = 15.seconds, - maxMessagesPerTransaction = 1, - maxMessagesToBuffer = 4, - formatter = BareFormatter, - level = None, - statsReceiver = statsReceiver - ) { - override private[logging] val flusher = new MockThreadPoolExecutor - } - scribe.publish(record1) - if (retries > 0 && statsReceiver.counter("connection_failed")() < 1l) - scribeWithConnectionFailure(retries - 1) - else - scribe - } - val scribe = scribeWithConnectionFailure(2) - - assert(statsReceiver.counter("connection_failed")() == 1l) - - scribe.publish(record2) - scribe.publish(record1) - scribe.publish(record2) - - scribe.flusher.shutdown() - scribe.flusher.awaitTermination(5, TimeUnit.SECONDS) - - assert(statsReceiver.counter("connection_skipped")() == 3l) - } - - // TODO rewrite deterministically when we rewrite ScribeHandler - "avoid unbounded flush queue growth" in { - val statsReceiver = new InMemoryStatsReceiver - - val scribe = ScribeHandler( - port = portWithoutListener, - bufferTime = 5.seconds, - maxMessagesToBuffer = 1, - formatter = BareFormatter, - category = "test", - statsReceiver = statsReceiver - ).apply() - - // Set up a rejectedExecutionHandler to count number of rejected tasks. - val rejected = new AtomicInteger(0) - scribe.flusher.setRejectedExecutionHandler(new RejectedExecutionHandler { - val inner = scribe.flusher.getRejectedExecutionHandler() - def rejectedExecution(r: Runnable, executor: ThreadPoolExecutor): Unit = { - rejected.getAndIncrement() - inner.rejectedExecution(r, executor) - } - }) - - // crude form to allow all 100 submission and get predictable dropping of tasks - scribe.synchronized { - scribe.publish(record1) - // wait till the ThreadPoolExecutor dequeue the first tast. - eventually(timeout(Span(2, Minutes)), interval(Span(100, Millis))) { - assert(scribe.flusher.getQueue().size() == 0) - } - // publish the rest. - for (i <- 1 until 100) { - scribe.publish(record1) - } - } - scribe.flusher.shutdown() - scribe.flusher.awaitTermination(15, TimeUnit.SECONDS) - - assert(statsReceiver.counter("published")() == 100l) - // 100 publish requests, each creates a flush task enqueued to ThreadPoolExecutor - // There will be 5 tasks stay in the threadpool's task queue. - // There will be 1 task dequeued by the executor. - // That leaves either 94 rejected tasks. - assert(rejected.get() == 94) - } - - "report scribe send stats" in { - Time.withCurrentTimeFrozen { timeControl => - val statsReceiver = new InMemoryStatsReceiver - val mockSocket = mock[Socket] - when(mockSocket.getOutputStream).thenReturn(new ByteArrayOutputStream()) - when(mockSocket.getInputStream) - .thenReturn(null) // First do not respond to attempts to use the old protocol - .thenAnswer(new Answer[ByteArrayInputStream] { - def answer(invocation: InvocationOnMock): ByteArrayInputStream = { - timeControl.advance(707.millis) - new ByteArrayInputStream(ScribeHandler.ScribeReply) - } - }) - - val scribe = ScribeHandler( - port = portWithoutListener, - category = "test", - bufferTime = 5.seconds, - formatter = BareFormatter, - level = None, - statsReceiver = statsReceiver - ).apply() - - scribe.setSocket(Some(mockSocket)) - - // Get the buffer first to know the size before flushing - scribe.updateLastTransmission() - scribe.publish(record2) - scribe.publish(record1) - val buffer = scribe.makeBuffer(2) - - // Make sure multiple records get flushed together - scribe.updateLastTransmission() - scribe.publish(record2) - scribe.publish(record1) - - scribe.sendBatch() - - scribe.flusher.shutdown() - scribe.flusher.awaitTermination(15, TimeUnit.SECONDS) - - assert(statsReceiver.stat("batch_size_bytes")() == Seq(buffer.capacity())) - assert(statsReceiver.stat("batch_size_messages")() == Seq(2)) - - assert(statsReceiver.counter("sent_records")() == 2L) - assert(statsReceiver.stat("success_latency_ms")() == Seq(707)) - assert(statsReceiver.counter("dropped_records")() == 0L) - assert(statsReceiver.stat("failure_latency_ms")() == Seq.empty) - } - } - - "handle batch write errors" in { - Time.withCurrentTimeFrozen { timeControl => - val statsReceiver = new InMemoryStatsReceiver - val mockSocket = mock[Socket] - when(mockSocket.getOutputStream).thenReturn(new ByteArrayOutputStream()) - when(mockSocket.getInputStream) - .thenReturn(null) // First do not respond to attempts to use the old protocol - .thenAnswer(new Answer[ByteArrayInputStream] { - def answer(invocation: InvocationOnMock): ByteArrayInputStream = { - timeControl.advance(1738.millis) - // An empty byte array ensures failure - new ByteArrayInputStream(Array.emptyByteArray) - } - }) - - val scribe = ScribeHandler( - port = portWithoutListener, - category = "test", - bufferTime = 5.seconds, - formatter = BareFormatter, - level = None, - statsReceiver = statsReceiver - ).apply() - - scribe.setSocket(Some(mockSocket)) - - // Make sure multiple records get flushed together - scribe.updateLastTransmission() - scribe.publish(record2) - scribe.publish(record1) - - // Throws exception reading an empty response - scribe.sendBatch() - - scribe.flusher.shutdown() - scribe.flusher.awaitTermination(15, TimeUnit.SECONDS) - - assert(statsReceiver.counter("dropped_records")() == 2L) - assert(statsReceiver.stat("failure_latency_ms")() == Seq(1738)) - assert(statsReceiver.counter("sent_records")() == 0L) - assert(statsReceiver.stat("success_latency_ms")() == Seq.empty) - } - } - } -} From dd8f7da01858c9566fba72a7aa3c287324458b06 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 20 Aug 2019 20:23:50 +0000 Subject: [PATCH 201/796] Remove console formatting Problem When we moved the environment variables to the sbt build scripts we accidentally moved the sbt log formatting variables as well. Solution Move them back. JIRA Issues: CSL-8718 Differential Revision: https://phabricator.twitter.biz/D358876 --- .travis.yml | 3 +++ build.sbt | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b8d907f92..e0b264a139 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,9 @@ git: depth: 3 quiet: true +env: + - JAVA_OPTS="-Dsbt.log.noformat=true" + # These directories are cached to S3 at the end of the build cache: directories: diff --git a/build.sbt b/build.sbt index 3aba50711d..8f4d694878 100644 --- a/build.sbt +++ b/build.sbt @@ -27,8 +27,7 @@ def travisTestJavaOptions: Seq[String] = { if (travisBuild) { Seq( "-DSKIP_FLAKY=true", - "-DSKIP_FLAKY_TRAVIS=true", - "-Dsbt.log.noformat=true" + "-DSKIP_FLAKY_TRAVIS=true" ) } else { Seq( From 262451cc288896852c0e421ec79aca8fef0a1faf Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Wed, 21 Aug 2019 14:17:30 +0000 Subject: [PATCH 202/796] util-core: add a test for long streams and AsyncStream.flatMap Problem / Solution We have a test for long streams generated by ++ but we don't explicitly test long streams generated by flatMap. JIRA Issues: CSL-8643 Differential Revision: https://phabricator.twitter.biz/D354363 --- .../twitter/concurrent/AsyncStreamTest.scala | 55 +++++++++++++------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 12dfc8dda9..7acdb373c5 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -10,6 +10,43 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { import AsyncStream.{mk, of} import AsyncStreamTest._ + test("++ with a long stream") { + var count = 0 + def genLongStream(len: Int): AsyncStream[Int] = + if (len == 0) { + AsyncStream.of(1) + } else { + count = count + 1 + 1 +:: genLongStream(len - 1) + } + // concat a long stream does not stack overflow + val s = genLongStream(1000000) ++ genLongStream(3) + s.foreach(_ => ()) + val first = count + s.foreach(_ => ()) + // the values are evaluated once + assert(count == first) + } + + test("flatMap with a long stream") { + var count = 0 + def genLongStream(len: Int): AsyncStream[Int] = + if (len == 0) { + AsyncStream.of(1) + } else AsyncStream.of(1).flatMap { i => + count = count + 1 + i +:: genLongStream(len - 1) + } + + // a long stream created via flatMap does not stack overflow + val s = genLongStream(1000000) ++ genLongStream(3) + s.foreach(_ => ()) + val first = count + s.foreach(_ => ()) + // the values are evaluated once + assert(count == first) + } + // Test all AsyncStream constructors: Empty, FromFuture, Cons, Embed. seqImpl.foreach { impl => def fromSeq[A](seq: Seq[A]): AsyncStream[A] = impl.apply(seq) @@ -224,24 +261,6 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } } - test(s"$impl: ++ with a long stream") { - var count = 0 - def genLongStream(len: Int): AsyncStream[Int] = - if (len == 0) { - AsyncStream.of(1) - } else { - count = count + 1 - 1 +:: genLongStream(len - 1) - } - // concat a long stream does not stack overflow - val s = genLongStream(1000000) ++ genLongStream(3) - s.foreach(_ => ()) - val first = count - s.foreach(_ => ()) - // the values are evaluated once - assert(count == first) - } - test(s"$impl: foldRight") { forAll { (a: List[Int]) => def f(n: Int, s: String) = (s.toLong + n).toString From b17297c731d7f693f0284a2d0d11e23635eff7e2 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 22 Aug 2019 02:54:15 +0000 Subject: [PATCH 203/796] util-app: Address `Await` exceptions in app closing Problem There is the potential that the final `Await.ready` call in `c.t.app.App#nonExitingMain` when closing emits a `TimeoutException` which is swallowed by the current handling of all `NonFatal` exceptions. In this case, swallowing the TimeoutException means that there's no good signal on if the JVM should exit which potentially leaves non-deamon threads running hanging the main thread of the application. Solution Make sure that we get signal when awaiting triggers a `TimeoutException` in a way that can be handled appropriately by the caller of `nonExitingMain`, which here is `App#main`. Result If we hang on awaiting during close which results in a TimeoutException, then `exitOnError` will be properly invoked which should exit the JVM. This is inspired by the work done in this PR and the conversation therein: https://github.com/twitter/util/pull/251 JIRA Issues: CSL-8627 Differential Revision: https://phabricator.twitter.biz/D356846 --- CHANGELOG.rst | 9 +++ .../src/main/scala/com/twitter/app/App.scala | 10 ++- .../test/scala/com/twitter/app/AppTest.scala | 64 ++++++++++++++++++- 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7eda8339a3..6944eb8a86 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,15 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-app: Better handling of exceptions when awaiting on the `c.t.app.App` to close at + the end of the main function. We `Await.ready` on `this` as the last step of + `App#nonExitingMain` which can potentially throw a `TimeoutException` which was previously + unhandled. We have updated the logic to ensure that `TimeoutException`s are handled accordingly. + ``PHAB_ID=D356846`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index b2ec4dcac5..bb4775447a 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -93,7 +93,13 @@ trait App extends Closable with CloseAwaitably { // want to ensure "reason" is written before attempting to write any details System.err.flush() System.err.println(details) - Await.ready(close(), closeDeadline - Time.now) + // if we have gotten here we may not yet have attempted to close, ensure we do. + try { + Await.ready(close(), closeDeadline - Time.now) + } catch { + case NonFatal(exc) => + exitOnError(newCloseException(Seq(exc))) + } System.exit(1) } @@ -378,6 +384,8 @@ trait App extends Closable with CloseAwaitably { try { // even if we suppress shutdown errors, we give the resources time to close Await.ready(this, closeDeadline - Time.now) } catch { + case e: TimeoutException => + throw e // we want TimeoutExceptions to propagate case NonFatal(_) => () } } diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 6ed8c5e3b3..817924c769 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -26,6 +26,32 @@ object VeryBadApp extends App { def main(): Unit = {} } +class WeNeverClose extends App { + override def exitOnError(throwable: Throwable): Unit = { + throwable match { + case _: CloseException => + throw throwable + case _ => + exitOnError("Exception thrown in main on startup") + } + } + + private[this] val unClosable = new Closable { + override def close(deadline: Time): Future[Unit] = Future.never + } + + closeOnExit(unClosable) + + // and we're never ready + override def isReady(implicit permit: Awaitable.CanAwait): Boolean = false + + def main(): Unit = {} +} + +class WeNeverCloseButWeDoNotCare extends WeNeverClose { + override protected val suppressGracefulShutdownErrors: Boolean = true +} + trait ErrorOnExitApp extends App { override val defaultCloseGracePeriod: Duration = 2.seconds @@ -75,6 +101,8 @@ class AppTest extends FunSuite { } test("App: order of hooks") { + import scala.collection.JavaConverters._ + val q = new ConcurrentLinkedQueue[Int] class Test1 extends App { onExit(q.add(4)) @@ -84,7 +112,7 @@ class AppTest extends FunSuite { init(q.add(0)) } new Test1().main(Array.empty) - assert(q.toArray.toSeq == Seq(0, 1, 2, 3, 4)) + assert(q.asScala.toSeq == Seq(0, 1, 2, 3, 4)) } test("loadServiceBinds") { @@ -376,4 +404,38 @@ class AppTest extends FunSuite { app.main(Array.empty) } + + test("never closing exits with exception") { + val t = new MockTimer + val app = new WeNeverClose { + override lazy val shutdownTimer: Timer = t + } + + Time.withTimeAt(Time.epoch) { _ => + val exc = intercept[CloseException] { + app.main(Array.empty) + } + + // the CloseException should be suppressing a com.twitter.util.TimeoutException + assert(exc.getSuppressed.length == 1) + assert(exc.getSuppressed.head.getClass == classOf[com.twitter.util.TimeoutException]) + } + } + + test("never closing exits with a timeout exception") { + val t = new MockTimer + val app = new WeNeverCloseButWeDoNotCare { + override lazy val shutdownTimer: Timer = t + } + + Time.withTimeAt(Time.epoch) { _ => + val exc = intercept[CloseException] { + app.main(Array.empty) + } + + // the CloseException should be suppressing a com.twitter.util.TimeoutException + assert(exc.getSuppressed.length == 1) + assert(exc.getSuppressed.head.getClass == classOf[com.twitter.util.TimeoutException]) + } + } } From cd4e36482d28b5b30d59a104b58b1fa6288eae4c Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 27 Aug 2019 23:29:21 +0000 Subject: [PATCH 204/796] dodo: update travis config to use trust dist, update scala/jdk Problem The dodo .travis.yml configuration is using the default travis build image, which has been moved to dist: xenial and does not have the oraclejdk8 available. Solution Explicitly list dist: trusty, as the other Twitter ecosystem projects do. Modify the Scala version to match our other projects. Move all projects to `openjdk8`. Result Dodo build should pass. JIRA Issues: CSL-8640 Differential Revision: https://phabricator.twitter.biz/D362555 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e0b264a139..3ec5aaa998 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ scala: - 2.13.0 jdk: - - oraclejdk8 + - openjdk8 notifications: slack: From 2ef366fcdc3a9204abf1bac34e3f90ee6766dc68 Mon Sep 17 00:00:00 2001 From: Seth Tisue Date: Wed, 28 Aug 2019 18:38:54 +0000 Subject: [PATCH 205/796] util-stats: remove bad varargs annotations @varargs only makes sense when the varargs are the last thing in the last parameter list. that isn't true here. this fixes a problem where the code failed to compile on JDK 11, with errors such as /Users/tisue/community.211-11/target-0.9.16/project-builds/twitter-util-715c1151a337e4a30659436b6cafc90bf8f35c6f/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java:21:1: cannot access com.twitter.finagle.stats.StatsReceiver bad class file: /Users/tisue/community.211-11/target-0.9.16/project-builds/twitter-util-715c1151a337e4a30659436b6cafc90bf8f35c6f/util-stats/target/scala-2.11/classes/com/twitter/finagle/stats/StatsReceiver.class class file contains malformed variable arity method: provideGauge(java.lang.String[],scala.Function0) whether the problem exists elsewhere in the codebase, I don't know; I just fixed the sites that were actually cropping up in the Scala community build /cc @martijnhoekstra the Scala compiler ought to catch this earlier; I'll open a scala/bug ticket on that Signed-off-by: Moses Nakamura Differential Revision: https://phabricator.twitter.biz/D362586 --- .../main/scala/com/twitter/finagle/stats/StatsReceiver.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index d032fefc16..0a08b9d429 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -150,7 +150,6 @@ trait StatsReceiver { * @see [[StatsReceiver.addGauge]] if you can properly control the lifecycle * of the returned [[Gauge gauge]]. */ - @varargs def provideGauge(name: String*)(f: => Float): Unit = { val gauge = addGauge(name: _*)(f) StatsReceiver.synchronized { @@ -175,7 +174,6 @@ trait StatsReceiver { * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ - @varargs def addGauge(name: String*)(f: => Float): Gauge = addGauge(Verbosity.Default, name: _*)(f) /** @@ -195,7 +193,6 @@ trait StatsReceiver { * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ - @varargs def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge /** From ab36eb698551a1aa50f13ac91b9829b688cd3997 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 28 Aug 2019 22:38:02 +0000 Subject: [PATCH 206/796] csl: bump the sbt version across the board Problem We have mismatched sbt versions, but 1.2.8 seems to work better for 2.13. Solution Get consistent sbt versions across the board. JIRA Issues: CSL-8768 Differential Revision: https://phabricator.twitter.biz/D363311 --- sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbt b/sbt index d18b842cd2..e7f538ee13 100755 --- a/sbt +++ b/sbt @@ -1,8 +1,8 @@ #!/bin/bash -sbtver=1.1.4 +sbtver=1.2.8 sbtjar=sbt-launch.jar -sbtsha128=ac6b13b709e3c0e3a8b2b3bf3031ec57660cb813 +sbtsha128=073c169c6e1a47b8ae78a7a718b907424dedab30 sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From d411716220b4c8a747fbd779201922e4f01ac599 Mon Sep 17 00:00:00 2001 From: Martijn Hoekstra Date: Thu, 5 Sep 2019 16:58:07 +0000 Subject: [PATCH 207/796] finagle-core: -> 2.13 Problem There is no 2.13 release for finagle core Solution Make a 2.13 build Result A 2.13 build Signed-off-by: Moses Nakamura Differential Revision: https://phabricator.twitter.biz/D364013 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6944eb8a86..8afa281213 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,8 @@ Runtime Behavior Changes unhandled. We have updated the logic to ensure that `TimeoutException`s are handled accordingly. ``PHAB_ID=D356846`` +* util: Upgrade to Scala Collections Compat 2.1.2. ``PHAB_ID=D364013`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index 8f4d694878..6a841399c4 100644 --- a/build.sbt +++ b/build.sbt @@ -62,7 +62,7 @@ val baseSettings = Seq( // Workaround for a scaladoc bug which causes it to choke on empty classpaths. unmanagedClasspath in Compile += Attributed.blank(new java.io.File("doesnotexist")), libraryDependencies ++= Seq( - "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.1", + "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.2", // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.10.19" % "test", From 66d76b8dcb8467df4cd614ff516b951a6c40161d Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 10 Sep 2019 19:56:19 +0000 Subject: [PATCH 208/796] Mid-September OSS release Problem We want to release the next versions of our Twitter OSS libraries 19.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-8794 Differential Revision: https://phabricator.twitter.biz/D368464 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8afa281213..d1edb089b7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.9.0 +------ + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 3d84a810f7..6631566679 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-collection` tools would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.8.1" +val collUtils = "com.twitter" %% "util-collection" % "19.9.0" ``` # Units diff --git a/build.sbt b/build.sbt index 6a841399c4..06f760c1a0 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.9.0-SNAPSHOT" +val releaseVersion = "19.9.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 34cd0dca0dc5cd413737c5dc259326e2dc019898 Mon Sep 17 00:00:00 2001 From: Jonathan Simms Date: Tue, 10 Sep 2019 22:04:17 +0000 Subject: [PATCH 209/796] finagle-redis: Add hashed-key based partitioning/routing support to redis Problem: It would be advantageous in some circumstances to be able to use a cluster of redis servers without requiring the use of a central coordinating service. Solution: Take advantage of the new finagle-partitioning module to implement a partitioned client for redis. Result: A new redis client stack that uses the (somewhat ill-named) KetamaPartitioningService to provide key-based routing to a number of backend servers. The standard and transactional clients have not changed. This review only adds a new configuration, and handles an appropriate subset of redis commands (ones that do not have an implicit assumption that they're talking to a single backend node). Note that the hashing algorithm is configurable, and the code is hash-implementation agnostic. Differential Revision: https://phabricator.twitter.biz/D362798 --- .../com/twitter/hashing/Distributor.scala | 2 ++ .../twitter/hashing/KetamaDistributor.scala | 18 +++++++++++++++--- .../hashing/KetamaDistributorTest.scala | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/util-hashing/src/main/scala/com/twitter/hashing/Distributor.scala b/util-hashing/src/main/scala/com/twitter/hashing/Distributor.scala index c90e0774d2..378187b488 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/Distributor.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/Distributor.scala @@ -2,6 +2,7 @@ package com.twitter.hashing trait Distributor[A] { def entryForHash(hash: Long): (Long, A) + def partitionIdForHash(hash: Long): Long def nodeForHash(hash: Long): A def nodeCount: Int def nodes: Seq[A] @@ -9,6 +10,7 @@ trait Distributor[A] { class SingletonDistributor[A](node: A) extends Distributor[A] { def entryForHash(hash: Long) = (hash, node) + def partitionIdForHash(hash: Long): Long = hash def nodeForHash(hash: Long) = node def nodeCount = 1 def nodes = Seq(node) diff --git a/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala b/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala index 35ccf6a985..574f4bef1f 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala @@ -104,10 +104,12 @@ class KetamaDistributor[A]( def nodes: Seq[A] = ketamaNodes.map(_.handle) def nodeCount: Int = ketamaNodes.size + // hashes are 32-bit because they are 32-bit on the libmemcached and + // we need to maintain compatibility with libmemcached + private[this] def truncateHash(hash: Long): Long = hash & 0xffffffffL + private def mapEntryForHash(hash: Long): java.util.Map.Entry[Long, KetamaNode[A]] = { - // hashes are 32-bit because they are 32-bit on the libmemcached and - // we need to maintain compatibility with libmemcached - val truncatedHash = hash & 0xffffffffL + val truncatedHash = truncateHash(hash) val entry = continuum.ceilingEntry(truncatedHash) if (entry == null) @@ -116,6 +118,16 @@ class KetamaDistributor[A]( entry } + def partitionIdForHash(hash: Long): Long = { + val truncatedHash = truncateHash(hash) + + val key: java.lang.Long = continuum.ceilingKey(truncatedHash) + if (key == null) + continuum.firstKey() + else + key + } + def entryForHash(hash: Long): (Long, A) = { val entry = mapEntryForHash(hash) (entry.getKey, entry.getValue.handle) diff --git a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala index 313985ff67..fc97b8e63e 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala @@ -25,6 +25,7 @@ class KetamaDistributorTest extends WordSpec with ScalaCheckDrivenPropertyChecks val ketamaDistributor = new KetamaDistributor(nodes, 160) val ketamaDistributorInoldLibMemcachedVersionComplianceMode = new KetamaDistributor(nodes, 160, true) + "pick the correct node with ketama hash function" in { // Test from Smile's KetamaNodeLocatorSpec.scala @@ -97,5 +98,21 @@ class KetamaDistributorTest extends WordSpec with ScalaCheckDrivenPropertyChecks assert(ketama.byteArrayToLE(ba, 12) == bb.getInt(12)) } } + + // these two methods use slightly different code paths to determine + // the partitionId for a given hash, but should return equal values + // for the same input. + + "partitionIdForHash is the same as entryForHash 1st tuple element" in { + val keyHasher = KeyHasher.MURMUR3 + + forAll { s: String => + val hash = keyHasher.hashKey(s.getBytes("UTF-8")) + val (pid1, _) = ketamaDistributor.entryForHash(hash) + val pid2 = ketamaDistributor.partitionIdForHash(hash) + + assert(pid1 == pid2) + } + } } } From 553522ccefe2f04100d8ea1dbc488d54694d12a5 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 11 Sep 2019 18:24:37 +0000 Subject: [PATCH 210/796] util: Remove util-collection from Readme Example Problem / Solution util-collection was removed in February 2019. We should stop using it as an example for the readme. Let's switch it to use util-core instead. Differential Revision: https://phabricator.twitter.biz/D368641 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6631566679..4ca525b07f 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,10 @@ for more details about how to contribute. # Using in your project -An example SBT dependency string for the `util-collection` tools would look like this: +An example SBT dependency string for the `util-core` library would look like this: ```scala -val collUtils = "com.twitter" %% "util-collection" % "19.9.0" +val utilCore = "com.twitter" %% "util-core" % "19.9.0" ``` # Units From f39f72408eab172202cf8b36d7194f4255ac553d Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 12 Sep 2019 20:08:11 +0000 Subject: [PATCH 211/796] util-core: Add Scaladoc Example to `Time.at` Problem / Solution GitHub issue #256 was opened due to a misunderstanding of how the `Time.at` method works. Let's add an example to give a demonstration of how to use it. Differential Revision: https://phabricator.twitter.biz/D369452 --- util-core/src/main/scala/com/twitter/util/Time.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index f4f89420ee..b0a09b8096 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -415,6 +415,11 @@ object Time extends TimeLikeOps[Time] { /** * Creates a [[Time]] instance at the given `datetime` string in the * "yyyy-MM-dd HH:mm:ss Z" format. + * + * {{{ + * val time = Time.at("2019-05-12 00:00:00 -0800") + * time.toString // 2019-05-12 08:00:00 +0000 + * }}} */ def at(datetime: String): Time = defaultFormat.parse(datetime) From 6f64b3cf6f3461eb4db503d6d3dadcdd6de7f041 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Mon, 16 Sep 2019 18:37:33 +0000 Subject: [PATCH 212/796] OSS Mid-September post-release Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.10.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-8794 Differential Revision: https://phabricator.twitter.biz/D369069 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4ca525b07f..ebc0ed1cda 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 06f760c1a0..daeebc98dc 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.9.0" +val releaseVersion = "19.10.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From d0a23efea502896d01c27571b57f043fe29f9c94 Mon Sep 17 00:00:00 2001 From: Olafur Pall Geirsson Date: Fri, 27 Sep 2019 15:21:08 +0000 Subject: [PATCH 213/796] util-core: Add undeclared dependency on scala-reflect. Problem/Solution The scala-reflect module is needed on the classpath to successfully compile the util-core-util module. By default, Pants automatically includes this module on the classpath so the lack of this dependency hasn't been a problem so far. However, this dependency needs to be explicitly declared in order for util-core-util to compile after exporting the Pants build it to Bloop, see https://github.com/pantsbuild/pants/pull/7961. Differential Revision: https://phabricator.twitter.biz/D372472 --- util-core/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-core/BUILD b/util-core/BUILD index 0c435d187b..5fcc7c3bd3 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -30,6 +30,7 @@ scala_library( repo = artifactory, ), dependencies = [ + "3rdparty/jvm/org/scala-lang:scala-reflect", "3rdparty/jvm/org/scala-lang/modules:scala-collection-compat", "3rdparty/jvm/org/scala-lang/modules:scala-parser-combinators", "util/util-function/src/main/java", From aedc943eb2b53944c6371d2e7a6428260d0def9f Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 1 Oct 2019 16:46:09 +0000 Subject: [PATCH 214/796] util-core: Fix Reader.framed to work with non-List sequences Problem When using Reader.framed, if the framer doesn't emit a `List` the framer will never emit any of that data. This is because it attempts to match on the `::` class which is the cons cell of the type `List`. Solution Refactor the read loop to work for any `Seq`. JIRA Issues: CSL-8868 Differential Revision: https://phabricator.twitter.biz/D378048 --- CHANGELOG.rst | 6 ++++++ .../src/main/scala/com/twitter/io/Reader.scala | 16 ++++++++-------- .../test/scala/com/twitter/io/ReaderTest.scala | 5 +++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d1edb089b7..0dbb380aea 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Bug Fixes +~~~~~~~~~ + +* util-core: Fixed bug in `c.t.io.Reader.framed` where if the `framer` didn't emit a `List` the + emitted frames were skipped. ``PHAB_ID=D378048`` + 19.9.0 ------ diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index e66ee60005..6e404c29a5 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -197,7 +197,7 @@ object Reader { extends Reader[Buf] with (Option[Buf] => Future[Option[Buf]]) { - private[this] var frames: Seq[Buf] = Seq.empty + private[this] var frames: Seq[Buf] = Nil // we only enter here when `frames` is empty. def apply(in: Option[Buf]): Future[Option[Buf]] = synchronized { @@ -211,13 +211,13 @@ object Reader { } def read(): Future[Option[Buf]] = synchronized { - frames match { - case nextFrame :: rst => - frames = rst - Future.value(Some(nextFrame)) - case _ => - // flatMap to `this` to prevent allocating - r.read().flatMap(this) + if (frames.isEmpty) { + // flatMap to `this` to prevent allocating + r.read().flatMap(this) + } else { + val nextFrame = frames.head + frames = frames.tail + Future.value(Some(nextFrame)) } } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index b2115941cc..8b756546df 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -427,6 +427,11 @@ class ReaderTest assert(await(r.read()).isEmpty) } + test("Framed works on non-list collections") { + val r = Reader.framed(BufReader(Buf.U32BE(0)), i => Vector(i)) + assert(await(r.read()).isDefined) + } + test("Reader.flatMap") { forAll { s: String => val reader1 = Reader.fromBuf(Buf.Utf8(s), 8) From 06fe819d78efd01bb43831b44ed92d89b9fb7095 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Tue, 1 Oct 2019 20:30:59 +0000 Subject: [PATCH 215/796] util-stats: create initial MetricBuilder and MetricSchema classes Problem No way to set or store metadata about a metric. StatsReceiver methods would become unwieldly if we simply added all metadata as new params. Solution Create an immutable MetricBuilder class, which allows for metric metadata to be configured. Since related metrics may share metadata, the MetricBuilder can be stored at incremental points to avoid having to reconfigure that metadata for each related metric. The MetricBuilder can then be used to generate MetricSchemas, which will later be used to create metrics in the StatsRegistry. JIRA Issues: CSL-8317 Differential Revision: https://phabricator.twitter.biz/D367752 --- .../twitter/finagle/stats/MetricBuilder.scala | 142 ++++++++++++++++++ .../twitter/finagle/stats/MetricSchema.scala | 22 +++ .../twitter/finagle/stats/MetricUnit.scala | 22 +++ .../twitter/finagle/stats/StatsReceiver.scala | 48 ++++++ 4 files changed, 234 insertions(+) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala new file mode 100644 index 0000000000..3f911f92ea --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -0,0 +1,142 @@ +package com.twitter.finagle.stats + +/** + * Represents the "role" this service plays with respect to this metric. + * + * Usually either Server (the service is processing a request) or Client (the server has sent a + * request to another service). In many cases, there is no relevant role for a metric, in which + * case NoRole should be used. + */ +sealed trait SourceRole +case object NoRoleSpecified extends SourceRole +case object Client extends SourceRole +case object Server extends SourceRole + +/** + * A builder class used to configure settings and metadata for metrics prior to instantiating them. + * Calling any of the three build methods (counter, gauge, or histogram) will cause the metric to be + * instantiated in the underlying StatsReceiver. + * + * @param keyIndicator indicates whether this metric is crucial to this service (ie, an SLO metric) + * @param description human-readable description of a metric's significance + * @param units the unit associated with the metrics value (milliseconds, megabytes, requests, etc) + * @param role whether the service is playing the part of client or server regarding this metric + * @param verbosity see StatsReceiver for details + * @param sourceClass the class pass of the module which generated this metric (ie, com.twitter...) + * @param sourceLibrary the name of the library which generated this metric (ie, finagle) + * @param name the relative metric name which will be appended to the scope of the StatsReceiver prior to long term storage + * @param identifier the service name used by obs stack for this service + * @param percentiles used to indicate buckets for histograms, to be set by the StatsReceiver + * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder + */ +class MetricBuilder( + val keyIndicator: Boolean = false, + val description: String = "", + val units: MetricUnit = Unspecified, + val role: SourceRole = NoRoleSpecified, + val verbosity: Verbosity = Verbosity.Default, + val sourceClass: String = "", + val sourceLibrary: Option[String] = None, + val name: Seq[String] = Seq(), + val identifier: String = "", + // Only persisted and relevant when building histograms. + val percentiles: IndexedSeq[Double] = IndexedSeq(), + val statsReceiver: StatsReceiver) { + + /** + * This copy method omits statsReceiver and percentiles as arguments because they should be + * set only during the initial creation of a MetricBuilder by a StatsReceiver, which should use + * itself as the value for statsReceiver. + */ + private[this] def copy( + keyIndicator: Boolean = this.keyIndicator, + description: String = this.description, + units: MetricUnit = this.units, + role: SourceRole = this.role, + verbosity: Verbosity = this.verbosity, + sourceClass: String = this.sourceClass, + sourceLibrary: Option[String] = this.sourceLibrary, + name: Seq[String] = this.name, + identifier: String = this.identifier + ): MetricBuilder = { + new MetricBuilder( + keyIndicator = keyIndicator, + description = description, + units = units, + role = role, + verbosity = verbosity, + sourceClass = sourceClass, + sourceLibrary = sourceLibrary, + name = name, + identifier = identifier, + percentiles = this.percentiles, + statsReceiver = this.statsReceiver + ) + } + + def withKeyIndicator(isKeyIndicator: Boolean = true): MetricBuilder = + this.copy(keyIndicator = isKeyIndicator) + + def withDescription(desc: String): MetricBuilder = this.copy(description = desc) + + def withVerbosity(verbosity: Verbosity): MetricBuilder = this.copy(verbosity = verbosity) + + def withSourceClass(sourceClass: String): MetricBuilder = this.copy(sourceClass = sourceClass) + + def withSourceLibrary(sourceLibrary: Option[String]): MetricBuilder = + this.copy(sourceLibrary = sourceLibrary) + + def withIdentifier(identifier: String): MetricBuilder = this.copy(identifier = identifier) + + def withUnits(units: MetricUnit): MetricBuilder = this.copy(units = units) + + def withRole(role: SourceRole): MetricBuilder = this.copy(role = role) + + /** + * Generates a CounterSchema which can be used to create a counter in a StatsReceiver. + * Used to test that builder class correctly propagates configured metadata. + * @return a CounterSchema describing a counter. + */ + private[MetricBuilder] def counterSchema: CounterSchema = CounterSchema(this) + + /** + * Generates a GaugeSchema which can be used to create a gauge in a StatsReceiver. + * Used to test that builder class correctly propagates configured metadata. + * @return a GaugeSchema describing a gauge. + */ + private[MetricBuilder] def gaugeSchema: GaugeSchema = GaugeSchema(this) + + /** + * Generates a HistogramSchema which can be used to create a histogram in a StatsReceiver. + * Used to test that builder class correctly propagates configured metadata. + * @return a HistogramSchema describing a histogram. + */ + private[MetricBuilder] def histogramSchema: HistogramSchema = HistogramSchema(this) + + /** + * Produce a counter as described by the builder inside the underlying StatsReceiver. + * @return the counter created. + */ + def counter(name: String*): Counter = { + val schema = this.copy(name = name).counterSchema + this.statsReceiver.counter(schema) + } + + /** + * Produce a gauge as described by the builder inside the underlying StatsReceiver. + * @return the gauge created. + */ + def gauge(name: String*)(f: => Float): Gauge = { + val schema = this.copy(name = name).gaugeSchema + this.statsReceiver.addGauge(schema)(f) + } + + /** + * Produce a histogram as described by the builder inside the underlying StatsReceiver. + * @return the histogram created. + */ + def histogram(name: String*): Stat = { + val schema = this.copy(name = name).histogramSchema + this.statsReceiver.stat(schema) + } +} diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala new file mode 100644 index 0000000000..89e8b7baf2 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala @@ -0,0 +1,22 @@ +package com.twitter.finagle.stats + +/** + * This trait represents a metric with the full set of its metadata. + */ +sealed abstract class MetricSchema(metricBuilder: MetricBuilder) + +/** + * Represents a counter with its assorted metadata. + */ +case class CounterSchema private (metricBuilder: MetricBuilder) extends MetricSchema(metricBuilder) + +/** + * Represents a gauge with its assorted metadata. + */ +case class GaugeSchema private (metricBuilder: MetricBuilder) extends MetricSchema(metricBuilder) + +/** + * Represents a histogram with its assorted metadata. + */ +case class HistogramSchema private (metricBuilder: MetricBuilder) + extends MetricSchema(metricBuilder) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala new file mode 100644 index 0000000000..de72ba0a56 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala @@ -0,0 +1,22 @@ +package com.twitter.finagle.stats + +/** + * Represents the units this metric is measured in. + * + * Common units for metrics are: + * Bytes/Kilobytes/Megabytes (for payload size, data written to disk) + * Milliseconds (for latency, GC durations) + * Requests (for successes, failures, and requests) + * Percentage (for CPU Util, Memory Usage) + */ +sealed trait MetricUnit +case object Unspecified extends MetricUnit +case object Bytes extends MetricUnit +case object Kilobytes extends MetricUnit +case object Megabytes extends MetricUnit +case object Seconds extends MetricUnit +case object Milliseconds extends MetricUnit +case object Microseconds extends MetricUnit +case object Requests extends MetricUnit +case object Percentage extends MetricUnit +case class CustomUnit(name: String) extends MetricUnit diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 0a08b9d429..8db7b754c2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -98,6 +98,11 @@ trait StatsReceiver { */ def isNull: Boolean = false + /** + * Get a [[MetricBuilder metricBuilder]] for this StatsReceiver. + */ + def metricBuilder(): MetricBuilder = new MetricBuilder(statsReceiver = this) + /** * Get a [[Counter counter]] with the given `name`. */ @@ -110,6 +115,16 @@ trait StatsReceiver { @varargs def counter(verbosity: Verbosity, name: String*): Counter + /** + * Get a [[Counter counter]] with the given schema. + * + * This default implementation passes the relevant fields to the pre-schema/pre-metadata interface. + * StatsReceivers with support for schemas/metadata will override this to make better use of that + * information. + */ + private[stats] def counter(schema: CounterSchema): Counter = + counter(schema.metricBuilder.verbosity, schema.metricBuilder.name: _*) + /** * Get a [[Counter counter]] with the given `name`. * @@ -131,6 +146,16 @@ trait StatsReceiver { @varargs def stat(verbosity: Verbosity, name: String*): Stat + /** + * Get a [[Stat stat]] with the given schema. + * + * This default implementation passes the relevant fields to the pre-schema/pre-metadata interface. + * StatsReceivers with support for schemas/metadata will override this to make better use of that + * information. + */ + private[stats] def stat(schema: HistogramSchema): Stat = + stat(schema.metricBuilder.verbosity, schema.metricBuilder.name: _*) + /** * Get a [[Stat stat]] with the given name. This method is a convenience for Java * programs, but is no longer needed because [[StatsReceivers.counter]] is @@ -195,6 +220,29 @@ trait StatsReceiver { */ def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge + /** + * Add the function `f` as a [[Gauge gauge]] with the given name. + * + * The returned [[Gauge gauge]] value is only weakly referenced by the + * [[StatsReceiver]], and if garbage collected will eventually cease to + * be a part of this measurement: thus, it needs to be retained by the + * caller. Or put another way, the measurement is only guaranteed to exist + * as long as there exists a strong reference to the returned + * [[Gauge gauge]] and typically should be stored in a member variable. + * + * Measurements under the same name are added together. + * + * This default implementation passes the relevant fields to the pre-schema/pre-metadata interface. + * StatsReceivers with support for schemas/metadata will override this to make better use of that + * information. + * + * @see [[StatsReceiver.provideGauge]] when there is not a good location + * to store the returned [[Gauge gauge]] that can give the desired lifecycle. + * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] + */ + private[stats] def addGauge(schema: GaugeSchema)(f: => Float): Gauge = + addGauge(schema.metricBuilder.verbosity, schema.metricBuilder.name: _*)(f) + /** * Prepend `namespace` to the names of the returned [[StatsReceiver]]. * From 9daad3d9ef7de09d52578fd3e131dea24bd1ce60 Mon Sep 17 00:00:00 2001 From: Neuman Vong Date: Tue, 1 Oct 2019 21:21:59 +0000 Subject: [PATCH 216/796] util-core: use the interruption cause directly Problem Because FuturePool wraps interrupts with j.u.c.CancellationException, it obscures failure flags for cancellations and ignorable failures. Solution Fail the promise with the raised exception directly; don't wrap it in a CancellationException. JIRA Issues: CSL-8839 Differential Revision: https://phabricator.twitter.biz/D371872 --- CHANGELOG.rst | 12 ++++++++++++ .../src/main/scala/com/twitter/util/FuturePool.scala | 4 +--- .../test/scala/com/twitter/util/FuturePoolTest.scala | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0dbb380aea..4837aa0bb7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,18 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-core: When a computation from FuturePool is interrupted, its promise is + set to the interrupt, wrapped in a j.u.c.CancellationException. This wrapper + was introduced because, all interrupts were once CancellationExceptions. In + RB_ID=98612, this changed to allow the user to raise specific exceptions as + interrupts, and in the aid of compatibility, we wrapped this raised exception + in a CancellationException. This change removes the wrapper and fails the + promise directly with the raised exception. This will affect users that + explicitly handle CancellationException. ``PHAB_ID=D371872`` + Bug Fixes ~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index b14a419c9a..ab730ef3d7 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -164,9 +164,7 @@ class ExecutorServiceFuturePool protected[this] ( p.setInterruptHandler { case cause => if (interruptible || runOk.compareAndSet(true, false)) { - val exc = new CancellationException - exc.initCause(cause) - if (p.updateIfEmpty(Throw(exc))) + if (p.updateIfEmpty(Throw(cause))) javaFuture.cancel(true) } } diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index c8eac5fcad..44191ee195 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -55,7 +55,8 @@ class FuturePoolTest extends FunSuite with Eventually { val result1 = pool { runCount.incrementAndGet(); Await.result(source1) } val result2 = pool { runCount.incrementAndGet(); Await.result(source2) } - result2.raise(new Exception) + val exc = new CancellationException + result2.raise(exc) source1.setValue(1) // The executor will run the task for result 2, but the wrapper From 36b47eae01fcfb6eb4d9fc73e2f5ee32c3392d9d Mon Sep 17 00:00:00 2001 From: Brad Scheppler Date: Thu, 3 Oct 2019 22:16:54 +0000 Subject: [PATCH 217/796] Python 3: Futurize Except Differential Revision: https://phabricator.twitter.biz/D378861 --- doc/src/sphinx/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/sphinx/conf.py b/doc/src/sphinx/conf.py index 90f5ba19d8..6369da3243 100644 --- a/doc/src/sphinx/conf.py +++ b/doc/src/sphinx/conf.py @@ -40,7 +40,7 @@ # fall back if theme is not there try: __import__('flask_theme_support') -except ImportError, e: +except ImportError as e: print '-' * 74 print 'Warning: Flask themes unavailable. Building with default theme' print 'If you want the Flask themes, run this command and build again:' From c66080b665d25334bcf6cca75ea3f02402342511 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 8 Oct 2019 18:38:09 +0000 Subject: [PATCH 218/796] util-hashing: implement `partitionIdForHash` in terms of `mapEntryForHash` Problem `partitionIdForHash` used a slightly different code path for finding the entry key in the ketama hash ring. This caused differences in the output for certain strings. Solution Unify the implementation behind `mapEntryForHash` like the other `*ForHash` methods. JIRA Issues: CSL-8858 Differential Revision: https://phabricator.twitter.biz/D381128 --- CHANGELOG.rst | 9 ++++++--- .../com/twitter/hashing/KetamaDistributor.scala | 17 +++-------------- .../twitter/hashing/KetamaDistributorTest.scala | 15 ++++++++++----- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4837aa0bb7..252dca4f3a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,9 @@ Bug Fixes * util-core: Fixed bug in `c.t.io.Reader.framed` where if the `framer` didn't emit a `List` the emitted frames were skipped. ``PHAB_ID=D378048`` +* util-hashing: Fix a bug where `partitionIdForHash` was returning incosistent values w.r.t + `entryForHash` in `KetamaDistributor`. ``PHAB_ID=D381128`` + 19.9.0 ------ @@ -32,9 +35,9 @@ Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ * util-app: Better handling of exceptions when awaiting on the `c.t.app.App` to close at - the end of the main function. We `Await.ready` on `this` as the last step of - `App#nonExitingMain` which can potentially throw a `TimeoutException` which was previously - unhandled. We have updated the logic to ensure that `TimeoutException`s are handled accordingly. + the end of the main function. We `Await.ready` on `this` as the last step of + `App#nonExitingMain` which can potentially throw a `TimeoutException` which was previously + unhandled. We have updated the logic to ensure that `TimeoutException`s are handled accordingly. ``PHAB_ID=D356846`` * util: Upgrade to Scala Collections Compat 2.1.2. ``PHAB_ID=D364013`` diff --git a/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala b/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala index 574f4bef1f..f59dc5dc9c 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala @@ -110,23 +110,12 @@ class KetamaDistributor[A]( private def mapEntryForHash(hash: Long): java.util.Map.Entry[Long, KetamaNode[A]] = { val truncatedHash = truncateHash(hash) - val entry = continuum.ceilingEntry(truncatedHash) - if (entry == null) - continuum.firstEntry - else - entry + if (entry == null) continuum.firstEntry else entry } - def partitionIdForHash(hash: Long): Long = { - val truncatedHash = truncateHash(hash) - - val key: java.lang.Long = continuum.ceilingKey(truncatedHash) - if (key == null) - continuum.firstKey() - else - key - } + def partitionIdForHash(hash: Long): Long = + mapEntryForHash(hash).getKey def entryForHash(hash: Long): (Long, A) = { val entry = mapEntryForHash(hash) diff --git a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala index fc97b8e63e..bac185e0fe 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala @@ -99,18 +99,23 @@ class KetamaDistributorTest extends WordSpec with ScalaCheckDrivenPropertyChecks } } - // these two methods use slightly different code paths to determine - // the partitionId for a given hash, but should return equal values - // for the same input. - "partitionIdForHash is the same as entryForHash 1st tuple element" in { val keyHasher = KeyHasher.MURMUR3 - forAll { s: String => + { + // Special case this unicode string since this test has failed + // on it in the past. + val s = "ퟲ狂✟풻棇埲蟂덧➀缘็滀佳黟韕숻᩿볯箿䜐㫌홻ᖛ磌ᡎ油" val hash = keyHasher.hashKey(s.getBytes("UTF-8")) val (pid1, _) = ketamaDistributor.entryForHash(hash) val pid2 = ketamaDistributor.partitionIdForHash(hash) + assert(pid1 == pid2) + } + forAll { s: String => + val hash = keyHasher.hashKey(s.getBytes("UTF-8")) + val (pid1, _) = ketamaDistributor.entryForHash(hash) + val pid2 = ketamaDistributor.partitionIdForHash(hash) assert(pid1 == pid2) } } From 0293c7d9207e46a002ecddbf0cce6230687bc1e9 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Tue, 8 Oct 2019 21:36:34 +0000 Subject: [PATCH 219/796] Twitter-oss: Prepare OSS libraries for release 19.10.0 Problem We want to release the next versions of our Twitter OSS libraries 19.10.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-8918 Differential Revision: https://phabricator.twitter.biz/D381391 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 252dca4f3a..4d43263513 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.10.0 +------- + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index ebc0ed1cda..427f7e6581 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "19.9.0" +val utilCore = "com.twitter" %% "util-core" % "19.10.0" ``` # Units diff --git a/build.sbt b/build.sbt index daeebc98dc..2bb9569cc5 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.10.0-SNAPSHOT" +val releaseVersion = "19.10.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 9414324ef1aa256cb1f1debca538c893e3246796 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Tue, 8 Oct 2019 23:15:41 +0000 Subject: [PATCH 220/796] Twitter-oss: update OSS libraries post-release to 19.11.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.11.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D381937 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 427f7e6581..0bbbade258 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 2bb9569cc5..ce882021c8 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.10.0" +val releaseVersion = "19.11.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From a8260998518a9efc32ece0c5114694674c63072c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Charczy=C5=84ski?= Date: Thu, 10 Oct 2019 19:23:13 +0000 Subject: [PATCH 221/796] util-core: new Closable.stopCollectClosablesThread method to stop CollectClosables thread Problem No possibility to stop CollectClosables thread. Problem described here [[ https://github.com/twitter/finagle/issues/758 | twitter/finagle#758 ]] Solution Created public method Closable.stopCollectClosablesThread that stops CollectClosables thread Result New API that makes possible to stop CollectClosables thread. Signed-off-by: Jing Yan Differential Revision: https://phabricator.twitter.biz/D382800 --- CHANGELOG.rst | 6 ++++++ .../src/main/scala/com/twitter/util/Closable.scala | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4d43263513..b3a5e4c9b1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-core: Created public method Closable.stopCollectClosablesThread that stops CollectClosables +thread. ``PHAB_ID=D382800`` + 19.10.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/Closable.scala b/util-core/src/main/scala/com/twitter/util/Closable.scala index cd8ade8fe2..7c06def274 100644 --- a/util-core/src/main/scala/com/twitter/util/Closable.scala +++ b/util-core/src/main/scala/com/twitter/util/Closable.scala @@ -2,7 +2,7 @@ package com.twitter.util import java.lang.ref.{PhantomReference, Reference, ReferenceQueue} import java.{util => ju} -import java.util.concurrent.atomic.AtomicReference +import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} import java.util.logging.{Level, Logger} import scala.annotation.{tailrec, varargs} import scala.util.control.{NonFatal => NF} @@ -45,6 +45,16 @@ abstract class AbstractClosable extends Closable */ object Closable { private[this] val logger = Logger.getLogger("") + private[this] val collectorThreadEnabled = new AtomicBoolean(true) + + /** Stop CollectClosables thread. */ + def stopCollectClosablesThread(): Unit = { + if (collectorThreadEnabled.compareAndSet(true, false)) + Try(collectorThread.interrupt).onFailure( + fatal => + logger + .log(Level.SEVERE, "Current thread cannot interrupt CollectClosables thread", fatal))() + } /** Provide Java access to the [[com.twitter.util.Closable]] mixin. */ def close(o: AnyRef): Future[Unit] = o match { @@ -161,7 +171,7 @@ object Closable { private val collectorThread = new Thread("CollectClosables") { override def run(): Unit = { - while (true) { + while (collectorThreadEnabled.get()) { try { val ref = refq.remove() val closable = refs.synchronized(refs.remove(ref)) From ad42642a70789eb81438de699b5051f37c03e973 Mon Sep 17 00:00:00 2001 From: Shane Delmore Date: Tue, 15 Oct 2019 21:00:09 +0000 Subject: [PATCH 222/796] Add type annotations using scalafix rule ExplicitResultTypes Differential Revision: https://phabricator.twitter.biz/D384887 --- .../scala/com/twitter/app/ClassPath.scala | 2 +- .../scala/com/twitter/app/GlobalFlag.scala | 5 +- .../twitter/concurrent/OfferBenchmark.scala | 6 +- .../twitter/concurrent/OnceBenchmark.scala | 2 +- .../hashing/KetamaDistributorBench.scala | 4 +- .../scala/com/twitter/jvm/CpuProfile.scala | 5 +- .../string/StringConcatenationBenchmark.scala | 4 +- .../com/twitter/util/FutureBenchmark.scala | 12 +- .../com/twitter/util/StopwatchBenchmark.scala | 2 +- .../com/twitter/util/StringEncoder.scala | 2 +- .../scala/com/twitter/concurrent/Spool.scala | 4 +- .../com/twitter/concurrent/SpoolSource.scala | 2 +- .../main/scala/com/twitter/util/Diff.scala | 2 +- .../com/twitter/hashing/Distributor.scala | 6 +- .../scala/com/twitter/jvm/Estimator.scala | 8 +- .../main/scala/com/twitter/jvm/Heapster.scala | 6 +- .../com/twitter/logging/FileHandler.scala | 9 +- .../scala/com/twitter/logging/Formatter.scala | 9 +- .../scala/com/twitter/logging/Handler.scala | 60 +++++---- .../com/twitter/logging/LazyLogRecord.scala | 4 +- .../scala/com/twitter/logging/Logger.scala | 125 +++++++++--------- .../scala/com/twitter/logging/Logging.scala | 2 +- .../com/twitter/logging/SyslogHandler.scala | 18 +-- .../com/twitter/util/reflect/Proxy.scala | 51 +++---- .../stats/BroadcastStatsReceiver.scala | 4 +- .../finagle/stats/NullStatsReceiver.scala | 2 +- .../com/twitter/logging/TestLogging.scala | 4 +- .../com/twitter/util/ThriftSerializer.scala | 6 +- .../twitter/util/tunable/Deserializers.scala | 26 ++-- .../main/scala/com/twitter/zk/AuthInfo.scala | 2 +- .../main/scala/com/twitter/zk/Connector.scala | 6 +- .../src/main/scala/com/twitter/zk/Event.scala | 35 ++--- .../scala/com/twitter/zk/LiftableFuture.scala | 13 +- .../com/twitter/zk/NativeConnector.scala | 10 +- .../scala/com/twitter/zk/RetryPolicy.scala | 4 +- .../src/main/scala/com/twitter/zk/ZNode.scala | 36 ++--- .../main/scala/com/twitter/zk/ZkClient.scala | 6 +- .../zk/coordination/ShardCoordinator.scala | 2 +- .../zk/coordination/ZkAsyncSemaphore.scala | 4 +- 39 files changed, 274 insertions(+), 236 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/ClassPath.scala b/util-app/src/main/scala/com/twitter/app/ClassPath.scala index 7f7a254b0d..c7fa568f63 100644 --- a/util-app/src/main/scala/com/twitter/app/ClassPath.scala +++ b/util-app/src/main/scala/com/twitter/app/ClassPath.scala @@ -12,7 +12,7 @@ import scala.io.Source private[app] object ClassPath { - val IgnoredPackages = Set( + val IgnoredPackages: Set[String] = Set( "apple/", "ch/epfl/", "com/apple/", diff --git a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala index 88683c0d0d..89c537a65e 100644 --- a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala +++ b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala @@ -51,7 +51,8 @@ abstract class GlobalFlag[T] private[app] ( extends Flag[T](null, help, defaultOrUsage, false) { // Flags defined inside package objects have unexpected names. - require(!GlobalFlag.isEnclosedInPackageObject(getClass), + require( + !GlobalFlag.isEnclosedInPackageObject(getClass), s"package object encloses flag definition: $getClass") override protected[this] def parsingDone: Boolean = true @@ -121,7 +122,7 @@ object GlobalFlag { private def isEnclosedInPackageObject(klass: Class[_]): Boolean = klass.getEnclosingClass != null && - klass.getEnclosingClass.getSimpleName == "package" + klass.getEnclosingClass.getSimpleName == "package" private[app] def get(flagName: String): Option[Flag[_]] = { val className = if (!flagName.endsWith("$")) flagName + "$" else flagName diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala index a798fe7f55..1e9e952616 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/OfferBenchmark.scala @@ -22,7 +22,7 @@ object OfferBenchmark { // to focus on the benchmarking the impl of `choose`, use dumbed-down impls // of the offer and tx classes. private class SimpleOffer[T](fut: Future[Tx[T]]) extends Offer[T] { - override def prepare() = fut + override def prepare(): Future[Tx[T]] = fut } private class SimpleTx[T](t: T) extends Tx[T] { @@ -36,8 +36,8 @@ object OfferBenchmark { @Param(Array("3", "10", "100")) var numToChooseFrom: Int = 3 - val rng = new Random(3102957159L) - val rngOpt = Some(rng) + val rng: Random = new Random(3102957159L) + val rngOpt: Some[Random] = Some(rng) // N.B. Traditionally you'd put this type of setup inside of a benchmark setup method per invocation // but because the runtime of each invocation is so small, the overhead is high JMH advises against diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/OnceBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/OnceBenchmark.scala index 86b0424a9f..1d1d2945bc 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/OnceBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/OnceBenchmark.scala @@ -17,6 +17,6 @@ class OnceBenchmark { object OnceBenchmark { @State(Scope.Benchmark) class OnceState { - val once = Once(()) + val once: () => Unit = Once(()) } } diff --git a/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala b/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala index 53933b348b..5c7ef5b75b 100644 --- a/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala +++ b/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala @@ -16,10 +16,10 @@ class KetamaDistributorBench extends StdBenchAnnotations { val numReps = 640 val numHosts = 310 - val hosts = 0.until(numHosts).map { i => + val hosts: scala.collection.immutable.IndexedSeq[String] = 0.until(numHosts).map { i => "abcd-zzz-" + i + "-ab2.qwer.twitter.com" } - val nodes = hosts map { h => + val nodes: scala.collection.immutable.IndexedSeq[KetamaNode[String]] = hosts map { h => KetamaNode[String](h + ":12021", 1, h) } diff --git a/util-benchmark/src/main/scala/com/twitter/jvm/CpuProfile.scala b/util-benchmark/src/main/scala/com/twitter/jvm/CpuProfile.scala index 710a4beb57..8e86063ccb 100644 --- a/util-benchmark/src/main/scala/com/twitter/jvm/CpuProfile.scala +++ b/util-benchmark/src/main/scala/com/twitter/jvm/CpuProfile.scala @@ -4,6 +4,7 @@ import java.lang.management.ManagementFactory import java.util.concurrent.TimeUnit import org.openjdk.jmh.annotations._ import scala.util.Random +import java.lang.management.ThreadMXBean @OutputTimeUnit(TimeUnit.NANOSECONDS) @BenchmarkMode(Array(Mode.AverageTime)) @@ -30,12 +31,12 @@ class CpuProfileBenchmark { object CpuProfileBenchmark { @State(Scope.Benchmark) class ThreadState { - val bean = ManagementFactory.getThreadMXBean() + val bean: ThreadMXBean = ManagementFactory.getThreadMXBean() // TODO: change dynamically. bounce up & down // the stack. μ and σ are for *this* stack. class Stack(rng: Random, μ: Int, σ: Int) { - def apply() = { + def apply(): Int = { val depth = math.max(1, μ + (rng.nextGaussian * σ).toInt) dive(depth) } diff --git a/util-benchmark/src/main/scala/com/twitter/string/StringConcatenationBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/string/StringConcatenationBenchmark.scala index 8e908dc244..1b3599a4f0 100644 --- a/util-benchmark/src/main/scala/com/twitter/string/StringConcatenationBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/string/StringConcatenationBenchmark.scala @@ -10,8 +10,8 @@ class StringConcatenationBenchmark extends StdBenchAnnotations { val Length = 16 val N = 10000 - val rng = new Random(1010101) - val words = rng.alphanumeric.grouped(Length).map(_.mkString).take(N).toArray + val rng: Random = new Random(1010101) + val words: Array[String] = rng.alphanumeric.grouped(Length).map(_.mkString).take(N).toArray var i = 0 private[this] def word(): String = { diff --git a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala index 07c6f3e3d9..494cc4b758 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala @@ -275,9 +275,9 @@ object FutureBenchmark { @State(Scope.Benchmark) class ByState { - val timer = Timer.Nil + val timer: Timer = Timer.Nil val now: Time = Time.now - val exc = new TimeoutException("") + val exc: TimeoutException = new TimeoutException("") } @State(Scope.Benchmark) @@ -313,12 +313,12 @@ object FutureBenchmark { @State(Scope.Benchmark) class FlattenState { - val future = Future.value(Future.Unit) + val future: Future[Future[Unit]] = Future.value(Future.Unit) } @State(Scope.Benchmark) class LowerFromTryState { - val future = Future.Unit.liftToTry + val future: Future[Try[Unit]] = Future.Unit.liftToTry } @State(Scope.Benchmark) @@ -333,7 +333,7 @@ object FutureBenchmark { @Param(Array("0", "1", "10", "100")) var numToSelect = 0 - val p = new Promise[Unit] + val p: Promise[Unit] = new Promise[Unit] val futures: Seq[Future[Unit]] = if (numToSelect == 0) IndexedSeq.empty @@ -347,7 +347,7 @@ object FutureBenchmark { @Param(Array("0", "1", "10", "100")) var numToSelect = 0 - val p = new Promise[Unit] + val p: Promise[Unit] = new Promise[Unit] val futures: IndexedSeq[Future[Unit]] = if (numToSelect == 0) IndexedSeq.empty diff --git a/util-benchmark/src/main/scala/com/twitter/util/StopwatchBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/StopwatchBenchmark.scala index 02090d1ad0..3a600bf53b 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/StopwatchBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/StopwatchBenchmark.scala @@ -32,6 +32,6 @@ class StopwatchBenchmark { object StopwatchBenchmark { @State(Scope.Benchmark) class StopwatchState { - val elapsed = Stopwatch.start() + val elapsed: Stopwatch.Elapsed = Stopwatch.start() } } diff --git a/util-codec/src/main/scala/com/twitter/util/StringEncoder.scala b/util-codec/src/main/scala/com/twitter/util/StringEncoder.scala index a7afb865b6..785abf192c 100644 --- a/util-codec/src/main/scala/com/twitter/util/StringEncoder.scala +++ b/util-codec/src/main/scala/com/twitter/util/StringEncoder.scala @@ -68,7 +68,7 @@ trait GZIPStringEncoder extends StringEncoder { Base64StringEncoder.encode(baos.toByteArray) } - def encodeString(str: String) = encode(str.getBytes("UTF-8")) + def encodeString(str: String): String = encode(str.getBytes("UTF-8")) override def decode(str: String): Array[Byte] = { val baos = new ByteArrayOutputStream diff --git a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala index 4b85196d4a..1f4c422ff2 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala @@ -296,9 +296,9 @@ object Spool { private class LazyCons[A](val head: A, next: => Future[Spool[A]]) extends Spool[A] { def isEmpty = false - lazy val tail = next + lazy val tail: Future[Spool[A]] = next // NB: not touching tail, to avoid forcing unnecessarily - override def toString = "Cons(%s, ?)".format(head) + override def toString: String = "Cons(%s, ?)".format(head) } object Empty extends Spool[Nothing] { diff --git a/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala b/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala index de04927a08..92ecdc2c70 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala @@ -9,7 +9,7 @@ import com.twitter.util.{Future, Promise, Return} object SpoolSource { private object DefaultInterruptHandler extends PartialFunction[Any, Nothing] { def isDefinedAt(x: Any) = false - def apply(x: Any) = throw new MatchError(x) + def apply(x: Any): Nothing = throw new MatchError(x) } } diff --git a/util-core/src/main/scala/com/twitter/util/Diff.scala b/util-core/src/main/scala/com/twitter/util/Diff.scala index 285fba458a..b1818c5175 100644 --- a/util-core/src/main/scala/com/twitter/util/Diff.scala +++ b/util-core/src/main/scala/com/twitter/util/Diff.scala @@ -71,7 +71,7 @@ object Diffable { def map[U](f: T => U): SetDiff[U] = SetDiff(add.map(f), remove.map(f)) - override def toString = s"Diff(+$add, -$remove)" + override def toString: String = s"Diff(+$add, -$remove)" } private case class SeqDiff[T](limit: Int, insert: Map[Int, T]) extends Diff[Seq, T] { diff --git a/util-hashing/src/main/scala/com/twitter/hashing/Distributor.scala b/util-hashing/src/main/scala/com/twitter/hashing/Distributor.scala index 378187b488..cc8f6d0bdc 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/Distributor.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/Distributor.scala @@ -9,9 +9,9 @@ trait Distributor[A] { } class SingletonDistributor[A](node: A) extends Distributor[A] { - def entryForHash(hash: Long) = (hash, node) + def entryForHash(hash: Long): (Long, A) = (hash, node) def partitionIdForHash(hash: Long): Long = hash - def nodeForHash(hash: Long) = node + def nodeForHash(hash: Long): A = node def nodeCount = 1 - def nodes = Seq(node) + def nodes: Seq[A] = Seq(node) } diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala b/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala index de10956e17..fefbae359a 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala @@ -56,7 +56,7 @@ class Kalman(N: Int) { else ebuf ) - def estimate = est + def estimate: Double = est private[this] def variance(samples: Array[Double]): Double = { if (samples.length == 1) @@ -70,7 +70,7 @@ class Kalman(N: Int) { diff / (samples.length - 1) } - override def toString = + override def toString: String = "Kalman".format(estimate, weight, mvar, evar) } @@ -123,7 +123,7 @@ class WindowedMeans(N: Int, windows: Seq[(Int, Int)]) extends Estimator[Double] n += 1 } - def estimate = { + def estimate: Double = { require(n > 0) val weightedMeans = normalized map { case (w, i) => w * mean(n, i) } weightedMeans.sum @@ -146,5 +146,5 @@ class LoadAverage(interval: Double) extends Estimator[Double] { else load * a + m * (1 - a) } - def estimate = load + def estimate: Double = load } diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala b/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala index 32b7dcca00..07f9f6f910 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Heapster.scala @@ -41,7 +41,11 @@ class Heapster(klass: Class[_]) { def dumpProfile(forceGC: java.lang.Boolean): Array[Byte] = dumpProfileM.invoke(null, forceGC).asInstanceOf[Array[Byte]] - def profile(howlong: Duration, samplingPeriod: Int = 10 << 19, forceGC: Boolean = true) = { + def profile( + howlong: Duration, + samplingPeriod: Int = 10 << 19, + forceGC: Boolean = true + ): Array[Byte] = { clearProfile() setSamplingPeriod(samplingPeriod) diff --git a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala index fb52e411d7..69afdfb70c 100644 --- a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala @@ -60,7 +60,7 @@ object Policy { } object FileHandler { - val UTF8 = Charset.forName("UTF-8") + val UTF8: Charset = Charset.forName("UTF-8") /** * Generates a HandlerFactory that returns a FileHandler @@ -84,7 +84,8 @@ object FileHandler { rotateCount: Int = -1, formatter: Formatter = new Formatter(), level: Option[Level] = None - ) = () => new FileHandler(filename, rollPolicy, append, rotateCount, formatter, level) + ): () => FileHandler = + () => new FileHandler(filename, rollPolicy, append, rotateCount, formatter, level) } /** @@ -183,7 +184,7 @@ class FileHandler( /** * Compute the suffix for a rolled logfile, based on the roll policy. */ - def timeSuffix(date: Date) = { + def timeSuffix(date: Date): String = { val dateFormat = rollPolicy match { case Policy.Never => TwitterDateFormat("yyyy") case Policy.SigHup => TwitterDateFormat("yyyy") @@ -257,7 +258,7 @@ class FileHandler( } } - def roll() = synchronized { + def roll(): Unit = synchronized { stream.close() val newFilename = filenamePrefix + "-" + timeSuffix(new Date(openTime)) + filenameSuffix new File(filename).renameTo(new File(newFilename)) diff --git a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala index 432e7a7ceb..44adbd2015 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala @@ -21,6 +21,7 @@ import java.text.{MessageFormat, DateFormat} import java.util.regex.Pattern import java.util.{Date, GregorianCalendar, TimeZone, logging => javalog} import scala.collection.mutable +import java.{util => ju} private[logging] object Formatter { // FIXME: might be nice to unmangle some scala names here. @@ -42,7 +43,7 @@ private[logging] object Formatter { out.toList } - val DateFormatRegex = Pattern.compile("<([^>]+)>") + val DateFormatRegex: Pattern = Pattern.compile("<([^>]+)>") val DefaultStackTraceSizeLimit = 30 val DefaultPrefix = "%.3s [] %s: " } @@ -108,7 +109,7 @@ class Formatter( /** * Calendar to use for time zone display in date-time formatting. */ - val calendar = if (timezone.isDefined) { + val calendar: ju.GregorianCalendar = if (timezone.isDefined) { new GregorianCalendar(TimeZone.getTimeZone(timezone.get)) } else { new GregorianCalendar @@ -219,7 +220,7 @@ class Formatter( /** * Truncates the text from a java LogRecord, if necessary */ - def truncateText(message: String) = + def truncateText(message: String): String = if ((truncateAt > 0) && (message.length > truncateAt)) message.take(truncateAt) + "..." else @@ -230,5 +231,5 @@ class Formatter( * Formatter that logs only the text of a log message, with no prefix (no date, etc). */ object BareFormatter extends Formatter { - override def format(record: javalog.LogRecord) = formatText(record) + lineTerminator + override def format(record: javalog.LogRecord): String = formatText(record) + lineTerminator } diff --git a/util-logging/src/main/scala/com/twitter/logging/Handler.scala b/util-logging/src/main/scala/com/twitter/logging/Handler.scala index 939a6b8799..d3db7599a3 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Handler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Handler.scala @@ -29,7 +29,7 @@ abstract class Handler(val formatter: Formatter, val level: Option[Level]) exten setLevel(x) } - override def toString = { + override def toString: String = { "<%s level=%s formatter=%s>".format(getClass.getName, getLevel, formatter.toString) } } @@ -40,34 +40,34 @@ abstract class Handler(val formatter: Formatter, val level: Option[Level]) exten */ abstract class ProxyHandler(val handler: Handler) extends Handler(handler.formatter, handler.level) { - override def close() = handler.close() + override def close(): Unit = handler.close() - override def flush() = handler.flush() + override def flush(): Unit = handler.flush() - override def getEncoding() = handler.getEncoding() + override def getEncoding(): String = handler.getEncoding() - override def getErrorManager() = handler.getErrorManager() + override def getErrorManager(): javalog.ErrorManager = handler.getErrorManager() - override def getFilter() = handler.getFilter() + override def getFilter(): javalog.Filter = handler.getFilter() - override def getFormatter() = handler.getFormatter() + override def getFormatter(): javalog.Formatter = handler.getFormatter() - override def getLevel() = handler.getLevel() + override def getLevel(): javalog.Level = handler.getLevel() - override def isLoggable(record: javalog.LogRecord) = handler.isLoggable(record) + override def isLoggable(record: javalog.LogRecord): Boolean = handler.isLoggable(record) - override def publish(record: javalog.LogRecord) = handler.publish(record) + override def publish(record: javalog.LogRecord): Unit = handler.publish(record) - override def setEncoding(encoding: String) = handler.setEncoding(encoding) + override def setEncoding(encoding: String): Unit = handler.setEncoding(encoding) - override def setErrorManager(errorManager: javalog.ErrorManager) = + override def setErrorManager(errorManager: javalog.ErrorManager): Unit = handler.setErrorManager(errorManager) - override def setFilter(filter: javalog.Filter) = handler.setFilter(filter) + override def setFilter(filter: javalog.Filter): Unit = handler.setFilter(filter) - override def setFormatter(formatter: javalog.Formatter) = handler.setFormatter(formatter) + override def setFormatter(formatter: javalog.Formatter): Unit = handler.setFormatter(formatter) - override def setLevel(level: javalog.Level) = handler.setLevel(level) + override def setLevel(level: javalog.Level): Unit = handler.setLevel(level) } object NullHandler extends Handler(BareFormatter, None) { @@ -84,13 +84,16 @@ object StringHandler { /** * Generates a HandlerFactory that returns a StringHandler */ - def apply(formatter: Formatter = new Formatter(), level: Option[Level] = None) = + def apply( + formatter: Formatter = new Formatter(), + level: Option[Level] = None + ): () => StringHandler = () => new StringHandler(formatter, level) /** * for java compatibility */ - def apply() = () => new StringHandler() + def apply(): () => StringHandler = () => new StringHandler() } /** @@ -102,19 +105,19 @@ class StringHandler(formatter: Formatter = new Formatter(), level: Option[Level] // thread-safe logging private val buffer = new StringBuffer() - def publish(record: javalog.LogRecord) = { + def publish(record: javalog.LogRecord): Unit = { buffer append getFormatter().format(record) } - def close() = {} + def close(): Unit = {} - def flush() = {} + def flush(): Unit = {} - def get = { + def get: String = { buffer.toString } - def clear() = { + def clear(): Unit = { buffer.setLength(0) buffer.trimToSize() } @@ -125,13 +128,16 @@ object ConsoleHandler { /** * Generates a HandlerFactory that returns a ConsoleHandler */ - def apply(formatter: Formatter = new Formatter(), level: Option[Level] = None) = + def apply( + formatter: Formatter = new Formatter(), + level: Option[Level] = None + ): () => ConsoleHandler = () => new ConsoleHandler(formatter, level) /** * for java compatibility */ - def apply() = () => new ConsoleHandler() + def apply(): () => ConsoleHandler = () => new ConsoleHandler() } /** @@ -140,11 +146,11 @@ object ConsoleHandler { class ConsoleHandler(formatter: Formatter = new Formatter(), level: Option[Level] = None) extends Handler(formatter, level) { - def publish(record: javalog.LogRecord) = { + def publish(record: javalog.LogRecord): Unit = { System.err.print(getFormatter().format(record)) } - def close() = {} + def close(): Unit = {} - def flush() = Console.flush + def flush(): Unit = Console.flush } diff --git a/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala b/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala index 63b2d26135..ebacf29499 100644 --- a/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala +++ b/util-logging/src/main/scala/com/twitter/logging/LazyLogRecord.scala @@ -21,7 +21,7 @@ import java.util.{logging => javalog} class LazyLogRecord(level: javalog.Level, messageGenerator: => AnyRef) extends LogRecord(level, "") { - override lazy val getMessage = messageGenerator.toString + override lazy val getMessage: String = messageGenerator.toString } /** @@ -30,5 +30,5 @@ class LazyLogRecord(level: javalog.Level, messageGenerator: => AnyRef) class LazyLogRecordUnformatted(level: javalog.Level, message: String, items: Any*) extends LazyLogRecord(level, { message.format(items: _*) }) { require(items.size > 0) - val preformatted = message + val preformatted: String = message } diff --git a/util-logging/src/main/scala/com/twitter/logging/Logger.scala b/util-logging/src/main/scala/com/twitter/logging/Logger.scala index 7d310cd7e6..4e703c235a 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Logger.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Logger.scala @@ -95,20 +95,20 @@ class LoggingException(reason: String) extends Exception(reason) */ class Logger protected (val name: String, private val wrapped: javalog.Logger) { // wrapped methods: - def addHandler(handler: javalog.Handler) = wrapped.addHandler(handler) - def getFilter() = wrapped.getFilter() - def getHandlers() = wrapped.getHandlers() - def getLevel() = wrapped.getLevel() - def getParent() = wrapped.getParent() - def getUseParentHandlers() = wrapped.getUseParentHandlers() - def isLoggable(level: javalog.Level) = wrapped.isLoggable(level) - def log(record: javalog.LogRecord) = wrapped.log(record) - def removeHandler(handler: javalog.Handler) = wrapped.removeHandler(handler) - def setFilter(filter: javalog.Filter) = wrapped.setFilter(filter) - def setLevel(level: javalog.Level) = wrapped.setLevel(level) - def setUseParentHandlers(use: Boolean) = wrapped.setUseParentHandlers(use) - - override def toString = { + def addHandler(handler: javalog.Handler): Unit = wrapped.addHandler(handler) + def getFilter(): javalog.Filter = wrapped.getFilter() + def getHandlers(): Array[javalog.Handler] = wrapped.getHandlers() + def getLevel(): javalog.Level = wrapped.getLevel() + def getParent(): javalog.Logger = wrapped.getParent() + def getUseParentHandlers(): Boolean = wrapped.getUseParentHandlers() + def isLoggable(level: javalog.Level): Boolean = wrapped.isLoggable(level) + def log(record: javalog.LogRecord): Unit = wrapped.log(record) + def removeHandler(handler: javalog.Handler): Unit = wrapped.removeHandler(handler) + def setFilter(filter: javalog.Filter): Unit = wrapped.setFilter(filter) + def setLevel(level: javalog.Level): Unit = wrapped.setLevel(level) + def setUseParentHandlers(use: Boolean): Unit = wrapped.setUseParentHandlers(use) + + override def toString: String = { "<%s name='%s' level=%s handlers=%s use_parent=%s>".format( getClass.getName, name, @@ -146,42 +146,47 @@ class Logger protected (val name: String, private val wrapped: javalog.Logger) { } } - final def apply(level: Level, message: String, items: Any*) = log(level, message, items: _*) + final def apply(level: Level, message: String, items: Any*): Unit = log(level, message, items: _*) - final def apply(level: Level, thrown: Throwable, message: String, items: Any*) = + final def apply(level: Level, thrown: Throwable, message: String, items: Any*): Unit = log(level, thrown, message, items) // convenience methods: @varargs - def fatal(msg: String, items: Any*) = log(Level.FATAL, msg, items: _*) + def fatal(msg: String, items: Any*): Unit = log(Level.FATAL, msg, items: _*) @varargs - def fatal(thrown: Throwable, msg: String, items: Any*) = log(Level.FATAL, thrown, msg, items: _*) + def fatal(thrown: Throwable, msg: String, items: Any*): Unit = + log(Level.FATAL, thrown, msg, items: _*) @varargs - def critical(msg: String, items: Any*) = log(Level.CRITICAL, msg, items: _*) + def critical(msg: String, items: Any*): Unit = log(Level.CRITICAL, msg, items: _*) @varargs - def critical(thrown: Throwable, msg: String, items: Any*) = + def critical(thrown: Throwable, msg: String, items: Any*): Unit = log(Level.CRITICAL, thrown, msg, items: _*) @varargs - def error(msg: String, items: Any*) = log(Level.ERROR, msg, items: _*) + def error(msg: String, items: Any*): Unit = log(Level.ERROR, msg, items: _*) @varargs - def error(thrown: Throwable, msg: String, items: Any*) = log(Level.ERROR, thrown, msg, items: _*) + def error(thrown: Throwable, msg: String, items: Any*): Unit = + log(Level.ERROR, thrown, msg, items: _*) @varargs - def warning(msg: String, items: Any*) = log(Level.WARNING, msg, items: _*) + def warning(msg: String, items: Any*): Unit = log(Level.WARNING, msg, items: _*) @varargs - def warning(thrown: Throwable, msg: String, items: Any*) = + def warning(thrown: Throwable, msg: String, items: Any*): Unit = log(Level.WARNING, thrown, msg, items: _*) @varargs - def info(msg: String, items: Any*) = log(Level.INFO, msg, items: _*) + def info(msg: String, items: Any*): Unit = log(Level.INFO, msg, items: _*) @varargs - def info(thrown: Throwable, msg: String, items: Any*) = log(Level.INFO, thrown, msg, items: _*) + def info(thrown: Throwable, msg: String, items: Any*): Unit = + log(Level.INFO, thrown, msg, items: _*) @varargs - def debug(msg: String, items: Any*) = log(Level.DEBUG, msg, items: _*) + def debug(msg: String, items: Any*): Unit = log(Level.DEBUG, msg, items: _*) @varargs - def debug(thrown: Throwable, msg: String, items: Any*) = log(Level.DEBUG, thrown, msg, items: _*) + def debug(thrown: Throwable, msg: String, items: Any*): Unit = + log(Level.DEBUG, thrown, msg, items: _*) @varargs - def trace(msg: String, items: Any*) = log(Level.TRACE, msg, items: _*) + def trace(msg: String, items: Any*): Unit = log(Level.TRACE, msg, items: _*) @varargs - def trace(thrown: Throwable, msg: String, items: Any*) = log(Level.TRACE, thrown, msg, items: _*) + def trace(thrown: Throwable, msg: String, items: Any*): Unit = + log(Level.TRACE, thrown, msg, items: _*) def debugLazy(msg: => AnyRef): Unit = logLazy(Level.DEBUG, null, msg) def traceLazy(msg: => AnyRef): Unit = logLazy(Level.TRACE, null, msg) @@ -209,20 +214,22 @@ class Logger protected (val name: String, private val wrapped: javalog.Logger) { } // convenience methods: - def ifFatal(message: => AnyRef) = logLazy(Level.FATAL, message) - def ifFatal(thrown: Throwable, message: => AnyRef) = logLazy(Level.FATAL, thrown, message) - def ifCritical(message: => AnyRef) = logLazy(Level.CRITICAL, message) - def ifCritical(thrown: Throwable, message: => AnyRef) = logLazy(Level.CRITICAL, thrown, message) - def ifError(message: => AnyRef) = logLazy(Level.ERROR, message) - def ifError(thrown: Throwable, message: => AnyRef) = logLazy(Level.ERROR, thrown, message) - def ifWarning(message: => AnyRef) = logLazy(Level.WARNING, message) - def ifWarning(thrown: Throwable, message: => AnyRef) = logLazy(Level.WARNING, thrown, message) - def ifInfo(message: => AnyRef) = logLazy(Level.INFO, message) - def ifInfo(thrown: Throwable, message: => AnyRef) = logLazy(Level.INFO, thrown, message) - def ifDebug(message: => AnyRef) = logLazy(Level.DEBUG, message) - def ifDebug(thrown: Throwable, message: => AnyRef) = logLazy(Level.DEBUG, thrown, message) - def ifTrace(message: => AnyRef) = logLazy(Level.TRACE, message) - def ifTrace(thrown: Throwable, message: => AnyRef) = logLazy(Level.TRACE, thrown, message) + def ifFatal(message: => AnyRef): Unit = logLazy(Level.FATAL, message) + def ifFatal(thrown: Throwable, message: => AnyRef): Unit = logLazy(Level.FATAL, thrown, message) + def ifCritical(message: => AnyRef): Unit = logLazy(Level.CRITICAL, message) + def ifCritical(thrown: Throwable, message: => AnyRef): Unit = + logLazy(Level.CRITICAL, thrown, message) + def ifError(message: => AnyRef): Unit = logLazy(Level.ERROR, message) + def ifError(thrown: Throwable, message: => AnyRef): Unit = logLazy(Level.ERROR, thrown, message) + def ifWarning(message: => AnyRef): Unit = logLazy(Level.WARNING, message) + def ifWarning(thrown: Throwable, message: => AnyRef): Unit = + logLazy(Level.WARNING, thrown, message) + def ifInfo(message: => AnyRef): Unit = logLazy(Level.INFO, message) + def ifInfo(thrown: Throwable, message: => AnyRef): Unit = logLazy(Level.INFO, thrown, message) + def ifDebug(message: => AnyRef): Unit = logLazy(Level.DEBUG, message) + def ifDebug(thrown: Throwable, message: => AnyRef): Unit = logLazy(Level.DEBUG, thrown, message) + def ifTrace(message: => AnyRef): Unit = logLazy(Level.TRACE, message) + def ifTrace(thrown: Throwable, message: => AnyRef): Unit = logLazy(Level.TRACE, thrown, message) /** * Lazily logs a throwable. If the throwable contains a log level (via [[HasLogLevel]]), it @@ -244,7 +251,7 @@ class Logger protected (val name: String, private val wrapped: javalog.Logger) { /** * Remove all existing log handlers. */ - def clearHandlers() = { + def clearHandlers(): Unit = { // some custom Logger implementations may return null from getHandlers val handlers = getHandlers() if (handlers ne null) { @@ -292,37 +299,37 @@ object Logger extends Iterable[Logger] { // ----- convenience methods: /** OFF is used to turn off logging entirely. */ - def OFF = Level.OFF + def OFF: Level.OFF.type = Level.OFF /** Describes an event which will cause the application to exit immediately, in failure. */ - def FATAL = Level.FATAL + def FATAL: Level.FATAL.type = Level.FATAL /** Describes an event which will cause the application to fail to work correctly, but * keep attempt to continue. The application may be unusable. */ - def CRITICAL = Level.CRITICAL + def CRITICAL: Level.CRITICAL.type = Level.CRITICAL /** Describes a user-visible error that may be transient or not affect other users. */ - def ERROR = Level.ERROR + def ERROR: Level.ERROR.type = Level.ERROR /** Describes a problem which is probably not user-visible but is notable and/or may be * an early indication of a future error. */ - def WARNING = Level.WARNING + def WARNING: Level.WARNING.type = Level.WARNING /** Describes information about the normal, functioning state of the application. */ - def INFO = Level.INFO + def INFO: Level.INFO.type = Level.INFO /** Describes information useful for general debugging, but probably not normal, * day-to-day use. */ - def DEBUG = Level.DEBUG + def DEBUG: Level.DEBUG.type = Level.DEBUG /** Describes information useful for intense debugging. */ - def TRACE = Level.TRACE + def TRACE: Level.TRACE.type = Level.TRACE /** ALL is used to log everything. */ - def ALL = Level.ALL + def ALL: Level.ALL.type = Level.ALL /** * Return a map of log level values to the corresponding Level objects. @@ -339,7 +346,7 @@ object Logger extends Iterable[Logger] { * INFO level and goes to the console (stderr). Any existing log * handlers are removed. */ - def reset() = { + def reset(): Unit = { clearHandlers() loggersCache.clear() root.addHandler(new ConsoleHandler(new Formatter(), None)) @@ -348,7 +355,7 @@ object Logger extends Iterable[Logger] { /** * Remove all existing log handlers from all existing loggers. */ - def clearHandlers() = { + def clearHandlers(): Unit = { foreach { logger => logger.clearHandlers() logger.setLevel(null) @@ -400,7 +407,7 @@ object Logger extends Iterable[Logger] { } /** An alias for `get(name)` */ - def apply(name: String) = get(name) + def apply(name: String): Logger = get(name) private def get(depth: Int): Logger = { val st = new Throwable().getStackTrace @@ -421,7 +428,7 @@ object Logger extends Iterable[Logger] { def get(): Logger = get(2) /** An alias for `get()` */ - def apply() = get(2) + def apply(): Logger = get(2) private def getForClassName(className: String) = { if (className.endsWith("$")) { @@ -437,7 +444,7 @@ object Logger extends Iterable[Logger] { def get(cls: Class[_]): Logger = getForClassName(cls.getName) /** An alias for `get(class)` */ - def apply(cls: Class[_]) = get(cls) + def apply(cls: Class[_]): Logger = get(cls) /** * Iterate the Logger objects that have been created. diff --git a/util-logging/src/main/scala/com/twitter/logging/Logging.scala b/util-logging/src/main/scala/com/twitter/logging/Logging.scala index a43d1d567f..8824d99828 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Logging.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Logging.scala @@ -24,7 +24,7 @@ object Logging { trait Logging { self: App => import Logging._ - lazy val log = Logger(name) + lazy val log: Logger = Logger(name) def defaultFormatter: Formatter = new Formatter() def defaultOutput: String = "/dev/stderr" diff --git a/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala b/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala index 7f97478949..5278e589bf 100644 --- a/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/SyslogHandler.scala @@ -22,6 +22,8 @@ import java.util.{logging => javalog} import com.twitter.concurrent.NamedPoolThreadFactory import com.twitter.util.{TwitterDateFormat, NetUtil} +import java.text.{DateFormat, SimpleDateFormat} +import java.util.concurrent.Future object SyslogHandler { val DEFAULT_PORT = 514 @@ -65,8 +67,8 @@ object SyslogHandler { } } - val ISO_DATE_FORMAT = TwitterDateFormat("yyyy-MM-dd'T'HH:mm:ss") - val OLD_SYSLOG_DATE_FORMAT = TwitterDateFormat("MMM dd HH:mm:ss") + val ISO_DATE_FORMAT: SimpleDateFormat = TwitterDateFormat("yyyy-MM-dd'T'HH:mm:ss") + val OLD_SYSLOG_DATE_FORMAT: SimpleDateFormat = TwitterDateFormat("MMM dd HH:mm:ss") /** * Generates a HandlerFactory that returns a SyslogHandler. @@ -82,7 +84,7 @@ object SyslogHandler { port: Int = SyslogHandler.DEFAULT_PORT, formatter: Formatter = new Formatter(), level: Option[Level] = None - ) = () => new SyslogHandler(server, port, formatter, level) + ): () => SyslogHandler = () => new SyslogHandler(server, port, formatter, level) } class SyslogHandler(val server: String, val port: Int, formatter: Formatter, level: Option[Level]) @@ -91,10 +93,10 @@ class SyslogHandler(val server: String, val port: Int, formatter: Formatter, lev private val socket = new DatagramSocket private[logging] val dest = new InetSocketAddress(server, port) - def flush() = {} - def close() = {} + def flush(): Unit = {} + def close(): Unit = {} - def publish(record: javalog.LogRecord) = { + def publish(record: javalog.LogRecord): Unit = { val data = formatter.format(record).getBytes val packet = new DatagramPacket(data, data.length, dest) SyslogFuture { @@ -147,7 +149,7 @@ class SyslogFormatter( prefix = "" ) { - override def dateFormat = + override def dateFormat: DateFormat = if (useIsoDateFormat) { SyslogHandler.ISO_DATE_FORMAT } else { @@ -176,7 +178,7 @@ object SyslogFuture { ) private val noop = new Runnable { def run(): Unit = {} } - def apply(action: => Unit) = + def apply(action: => Unit): Future[_] = executor.submit(new Runnable { def run(): Unit = { action } }) diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala index 37df35e0f0..50c6b164d0 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala @@ -11,12 +11,12 @@ class NonexistentTargetException extends Exception("MethodCall was invoked witho object Proxy { @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply[I <: AnyRef: Manifest](f: MethodCall[I] => AnyRef) = { + def apply[I <: AnyRef: Manifest](f: MethodCall[I] => AnyRef): I = { new ProxyFactory[I](f).apply() } @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply[I <: AnyRef: Manifest](target: I, f: MethodCall[I] => AnyRef) = { + def apply[I <: AnyRef: Manifest](target: I, f: MethodCall[I] => AnyRef): I = { new ProxyFactory[I](f).apply(target) } } @@ -25,7 +25,7 @@ object ProxyFactory { private[reflect] object NoOpInterceptor extends MethodInterceptor[AnyRef](None, m => null) private[reflect] object IgnoredMethodFilter extends CallbackFilter { - def accept(m: Method) = { + def accept(m: Method): Int = { m.getName match { case "hashCode" => 1 case "equals" => 1 @@ -39,9 +39,9 @@ object ProxyFactory { class AbstractProxyFactory[I <: AnyRef: Manifest] { import ProxyFactory._ - final val interface = implicitly[Manifest[I]].runtimeClass + final val interface: Class[_] = implicitly[Manifest[I]].runtimeClass - protected final val proto = { + protected final val proto: Factory = { val e = new Enhancer e.setCallbackFilter(IgnoredMethodFilter) e.setCallbacks(Array(NoOpInterceptor, NoOp.INSTANCE)) @@ -50,12 +50,12 @@ class AbstractProxyFactory[I <: AnyRef: Manifest] { } @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - protected final def newWithCallback(f: MethodCall[I] => AnyRef) = { + protected final def newWithCallback(f: MethodCall[I] => AnyRef): I = { proto.newInstance(Array(new MethodInterceptor(None, f), NoOp.INSTANCE)).asInstanceOf[I] } @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - protected final def newWithCallback[T <: I](target: T, f: MethodCall[I] => AnyRef) = { + protected final def newWithCallback[T <: I](target: T, f: MethodCall[I] => AnyRef): I = { proto.newInstance(Array(new MethodInterceptor(Some(target), f), NoOp.INSTANCE)).asInstanceOf[I] } } @@ -63,9 +63,9 @@ class AbstractProxyFactory[I <: AnyRef: Manifest] { class ProxyFactory[I <: AnyRef: Manifest](f: MethodCall[I] => AnyRef) extends AbstractProxyFactory[I] { @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply[T <: I](target: T) = newWithCallback(target, f) + def apply[T <: I](target: T): I = newWithCallback(target, f) @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply() = newWithCallback(f) + def apply(): I = newWithCallback(f) } private[reflect] class MethodInterceptor[I <: AnyRef]( @@ -73,10 +73,15 @@ private[reflect] class MethodInterceptor[I <: AnyRef]( callback: MethodCall[I] => AnyRef) extends CGMethodInterceptor with Serializable { - val targetRef = target.getOrElse(null).asInstanceOf[I] + val targetRef: I = target.getOrElse(null).asInstanceOf[I] @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - final def intercept(p: AnyRef, m: Method, args: Array[AnyRef], methodProxy: MethodProxy) = { + final def intercept( + p: AnyRef, + m: Method, + args: Array[AnyRef], + methodProxy: MethodProxy + ): Object = { callback(new MethodCall(targetRef, m, args, methodProxy)) } } @@ -88,34 +93,34 @@ final class MethodCall[T <: AnyRef] private[reflect] ( methodProxy: MethodProxy) extends (() => AnyRef) { - lazy val target = if (targetRef ne null) Some(targetRef) else None + lazy val target: Option[T] = if (targetRef ne null) Some(targetRef) else None @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def clazz = method.getDeclaringClass + def clazz: Class[_] = method.getDeclaringClass @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def clazzName = clazz.getName + def clazzName: String = clazz.getName @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def className = clazzName + def className: String = clazzName @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def parameterTypes = method.getParameterTypes + def parameterTypes: Array[Class[_]] = method.getParameterTypes @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def name = method.getName + def name: String = method.getName @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def returnsUnit = { + def returnsUnit: Boolean = { val rt = method.getReturnType (rt eq classOf[Unit]) || (rt eq classOf[Null]) || (rt eq java.lang.Void.TYPE) } @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def returnsFuture = classOf[Future[_]] isAssignableFrom method.getReturnType + def returnsFuture: Boolean = classOf[Future[_]] isAssignableFrom method.getReturnType private def getTarget = if (targetRef ne null) targetRef else throw new NonexistentTargetException @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply() = methodProxy.invoke(getTarget, args) + def apply(): AnyRef = methodProxy.invoke(getTarget, args) @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply(newTarget: T) = methodProxy.invoke(newTarget, args) + def apply(newTarget: T): Object = methodProxy.invoke(newTarget, args) @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply(newArgs: Array[AnyRef]) = methodProxy.invoke(getTarget, newArgs) + def apply(newArgs: Array[AnyRef]): Object = methodProxy.invoke(getTarget, newArgs) @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply(newTarget: T, newArgs: Array[AnyRef]) = methodProxy.invoke(newTarget, newArgs) + def apply(newTarget: T, newArgs: Array[AnyRef]): Object = methodProxy.invoke(newTarget, newArgs) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala index ab3f0be28b..29aa0c8e9b 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala @@ -15,7 +15,7 @@ object BroadcastStatsReceiver { private class Two(first: StatsReceiver, second: StatsReceiver) extends StatsReceiver with DelegatingStatsReceiver { - val repr = this + val repr: AnyRef = this def counter(verbosity: Verbosity, names: String*): Counter = new BroadcastCounter.Two( first.counter(verbosity, names: _*), @@ -41,7 +41,7 @@ object BroadcastStatsReceiver { } private class N(srs: Seq[StatsReceiver]) extends StatsReceiver with DelegatingStatsReceiver { - val repr = this + val repr: AnyRef = this def counter(verbosity: Verbosity, names: String*): Counter = BroadcastCounter(srs.map { _.counter(verbosity, names: _*) }) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala index 1a02801c61..3e5e3a34fb 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala @@ -1,7 +1,7 @@ package com.twitter.finagle.stats object NullStatsReceiver extends NullStatsReceiver { - def get() = this + def get(): NullStatsReceiver.type = this } /** diff --git a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala index e510314759..c1f3a21f42 100644 --- a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala +++ b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala @@ -24,7 +24,7 @@ import org.scalatest.{BeforeAndAfter, WordSpec} * Specify logging during unit tests via system property, defaulting to FATAL only. */ trait TestLogging extends BeforeAndAfter { self: WordSpec => - val logLevel = + val logLevel: Level = Logger.levelNames(Option[String](System.getenv("log")).getOrElse("FATAL").toUpperCase) private val logger = Logger.get("") @@ -72,7 +72,7 @@ trait TestLogging extends BeforeAndAfter { self: WordSpec => * Verify that the logger set up with `traceLogger` has received a log line with the given * substring somewhere inside it. */ - def mustLog(substring: String) = { + def mustLog(substring: String): org.scalatest.Assertion = { assert(logLines().filter { _ contains substring }.size > 0) } } diff --git a/util-thrift/src/main/scala/com/twitter/util/ThriftSerializer.scala b/util-thrift/src/main/scala/com/twitter/util/ThriftSerializer.scala index 4f473a11d4..2395ca0952 100644 --- a/util-thrift/src/main/scala/com/twitter/util/ThriftSerializer.scala +++ b/util-thrift/src/main/scala/com/twitter/util/ThriftSerializer.scala @@ -36,7 +36,7 @@ trait ThriftSerializer extends StringEncoder { * A thread-safe [[ThriftSerializer]] that uses [[TSimpleJSONProtocol]]. */ class JsonThriftSerializer extends ThriftSerializer { - override def protocolFactory = new TSimpleJSONProtocol.Factory + override def protocolFactory: TProtocolFactory = new TSimpleJSONProtocol.Factory override def fromBytes(obj: TBase[_, _], bytes: Array[Byte]): Unit = { val binarySerializer = new BinaryThriftSerializer @@ -52,12 +52,12 @@ class JsonThriftSerializer extends ThriftSerializer { * instead of this is recommended. */ class BinaryThriftSerializer extends ThriftSerializer with Base64StringEncoder { - override def protocolFactory = new TBinaryProtocol.Factory + override def protocolFactory: TProtocolFactory = new TBinaryProtocol.Factory } /** * A thread-safe [[ThriftSerializer]] that uses [[TCompactProtocol]]. */ class CompactThriftSerializer extends ThriftSerializer with Base64StringEncoder { - override def protocolFactory = new TCompactProtocol.Factory + override def protocolFactory: TProtocolFactory = new TCompactProtocol.Factory } diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/Deserializers.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/Deserializers.scala index 9d6a31a55a..848d5bceb9 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/Deserializers.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/Deserializers.scala @@ -14,17 +14,19 @@ private[twitter] object json { * Deserialize a String representation of a [[com.twitter.util.Duration]] using * [[com.twitter.util.Duration.parse]] */ - val DurationFromString = new StdDeserializer[util.Duration](classOf[util.Duration]) { - override def deserialize( - jsonParser: JsonParser, - deserializationContext: DeserializationContext - ): util.Duration = util.Duration.parse(jsonParser.getText) - } + val DurationFromString: StdDeserializer[util.Duration] = + new StdDeserializer[util.Duration](classOf[util.Duration]) { + override def deserialize( + jsonParser: JsonParser, + deserializationContext: DeserializationContext + ): util.Duration = util.Duration.parse(jsonParser.getText) + } - val StorageUnitFromString = new StdDeserializer[util.StorageUnit](classOf[util.StorageUnit]) { - override def deserialize( - jsonParser: JsonParser, - deserializationContext: DeserializationContext - ): util.StorageUnit = util.StorageUnit.parse(jsonParser.getText) - } + val StorageUnitFromString: StdDeserializer[util.StorageUnit] = + new StdDeserializer[util.StorageUnit](classOf[util.StorageUnit]) { + override def deserialize( + jsonParser: JsonParser, + deserializationContext: DeserializationContext + ): util.StorageUnit = util.StorageUnit.parse(jsonParser.getText) + } } diff --git a/util-zk/src/main/scala/com/twitter/zk/AuthInfo.scala b/util-zk/src/main/scala/com/twitter/zk/AuthInfo.scala index a94d6ba41d..3714f512ce 100644 --- a/util-zk/src/main/scala/com/twitter/zk/AuthInfo.scala +++ b/util-zk/src/main/scala/com/twitter/zk/AuthInfo.scala @@ -3,7 +3,7 @@ package com.twitter.zk case class AuthInfo(mode: String, data: Array[Byte]) object AuthInfo { - def digest(user: String, secret: String) = { + def digest(user: String, secret: String): AuthInfo = { val authString = "%s:%s".format(user, secret).getBytes("UTF-8") //DigestAuthenticationProvider.generateDigest("%s:%s".format(user, secret)).getBytes("UTF-8") AuthInfo("digest", authString) } diff --git a/util-zk/src/main/scala/com/twitter/zk/Connector.scala b/util-zk/src/main/scala/com/twitter/zk/Connector.scala index 8170bd1b6b..33b9432df5 100644 --- a/util-zk/src/main/scala/com/twitter/zk/Connector.scala +++ b/util-zk/src/main/scala/com/twitter/zk/Connector.scala @@ -14,7 +14,7 @@ trait Connector { import Connector.EventHandler val name = "zk-connector" - protected lazy val log = Logger.get(name) + protected lazy val log: Logger = Logger.get(name) private[this] val listeners = new AtomicReference[List[EventHandler]](Nil) @@ -60,10 +60,10 @@ object Connector { */ case class RoundRobin(connectors: Connector*) extends Connector { require(connectors.length > 0) - override val name = "round-robin-zk-connector:%d".format(connectors.length) + override val name: String = "round-robin-zk-connector:%d".format(connectors.length) private[this] var index = 0 - protected[this] def nextConnector() = { + protected[this] def nextConnector(): Connector = { val i = synchronized { if (index == Int.MaxValue) { index = 0 diff --git a/util-zk/src/main/scala/com/twitter/zk/Event.scala b/util-zk/src/main/scala/com/twitter/zk/Event.scala index 149eddebf3..72c70b1c1f 100644 --- a/util-zk/src/main/scala/com/twitter/zk/Event.scala +++ b/util-zk/src/main/scala/com/twitter/zk/Event.scala @@ -11,7 +11,8 @@ import com.twitter.util.{Promise, Return} */ object Event { - def apply(t: EventType, s: KeeperState, p: Option[String]) = new WatchedEvent(t, s, p.orNull) + def apply(t: EventType, s: KeeperState, p: Option[String]): WatchedEvent = + new WatchedEvent(t, s, p.orNull) def unapply(event: WatchedEvent): Option[(EventType, KeeperState, Option[String])] = { Some((event.getType, event.getState, Option { event.getPath })) @@ -19,10 +20,10 @@ object Event { } sealed trait StateEvent { - val eventType = EventType.None + val eventType: EventType = EventType.None val state: KeeperState - def apply() = Event(eventType, state, None) - def unapply(event: WatchedEvent) = event match { + def apply(): WatchedEvent = Event(eventType, state, None) + def unapply(event: WatchedEvent): Boolean = event match { case Event(t, s, _) => (t == eventType && s == state) case _ => false } @@ -30,27 +31,27 @@ sealed trait StateEvent { object StateEvent { object AuthFailed extends StateEvent { - val state = KeeperState.AuthFailed + val state: KeeperState = KeeperState.AuthFailed } object Connected extends StateEvent { - val state = KeeperState.SyncConnected + val state: KeeperState = KeeperState.SyncConnected } object Disconnected extends StateEvent { - val state = KeeperState.Disconnected + val state: KeeperState = KeeperState.Disconnected } object Expired extends StateEvent { - val state = KeeperState.Expired + val state: KeeperState = KeeperState.Expired } object ConnectedReadOnly extends StateEvent { - val state = KeeperState.ConnectedReadOnly + val state: KeeperState = KeeperState.ConnectedReadOnly } object SaslAuthenticated extends StateEvent { - val state = KeeperState.SaslAuthenticated + val state: KeeperState = KeeperState.SaslAuthenticated } def apply(w: WatchedEvent): StateEvent = { @@ -71,10 +72,10 @@ object StateEvent { } sealed trait NodeEvent { - val state = KeeperState.SyncConnected + val state: KeeperState = KeeperState.SyncConnected val eventType: EventType - def apply(path: String) = Event(eventType, state, Some(path)) - def unapply(event: WatchedEvent) = event match { + def apply(path: String): WatchedEvent = Event(eventType, state, Some(path)) + def unapply(event: WatchedEvent): Option[String] = event match { case Event(t, _, somePath) if (t == eventType) => somePath case _ => None } @@ -82,19 +83,19 @@ sealed trait NodeEvent { object NodeEvent { object Created extends NodeEvent { - val eventType = EventType.NodeCreated + val eventType: EventType = EventType.NodeCreated } object ChildrenChanged extends NodeEvent { - val eventType = EventType.NodeChildrenChanged + val eventType: EventType = EventType.NodeChildrenChanged } object DataChanged extends NodeEvent { - val eventType = EventType.NodeDataChanged + val eventType: EventType = EventType.NodeDataChanged } object Deleted extends NodeEvent { - val eventType = EventType.NodeDeleted + val eventType: EventType = EventType.NodeDeleted } } diff --git a/util-zk/src/main/scala/com/twitter/zk/LiftableFuture.scala b/util-zk/src/main/scala/com/twitter/zk/LiftableFuture.scala index b23867dce2..1390aa5951 100644 --- a/util-zk/src/main/scala/com/twitter/zk/LiftableFuture.scala +++ b/util-zk/src/main/scala/com/twitter/zk/LiftableFuture.scala @@ -3,6 +3,7 @@ package com.twitter.zk import com.twitter.util.{Future, Return, Throw} import org.apache.zookeeper.KeeperException import scala.language.implicitConversions +import com.twitter.util.Try protected[zk] object LiftableFuture { implicit def liftableFuture[T](f: Future[T]): LiftableFuture[T] = new LiftableFuture(f) @@ -15,14 +16,18 @@ protected[zk] object LiftableFuture { protected[zk] class LiftableFuture[T](f: Future[T]) { /** Lift a value to a Return. */ - def liftSuccess = f map { Return(_) } + def liftSuccess: Future[Return[T]] = f map { Return(_) } /** Lift all errors to a Throw */ - def liftFailure = liftSuccess handle { case e => Throw(e) } + def liftFailure: Future[Try[T]] = liftSuccess handle { case e => Throw(e) } /** Lift all KeeperExceptions to a Throw */ - def liftKeeperException = liftSuccess handle { case e: KeeperException => Throw(e) } + def liftKeeperException: Future[Try[T]] = liftSuccess handle { + case e: KeeperException => Throw(e) + } /** Lift failures when a watch would have been successfully installed */ - def liftNoNode = liftSuccess handle { case e: KeeperException.NoNodeException => Throw(e) } + def liftNoNode: Future[Try[T]] = liftSuccess handle { + case e: KeeperException.NoNodeException => Throw(e) + } } diff --git a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala index c0e6970891..8627612099 100644 --- a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala +++ b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala @@ -18,7 +18,7 @@ case class NativeConnector( with Serialized { override val name = "native-zk-connector" - protected[this] def mkConnection = { + protected[this] def mkConnection: NativeConnector.Connection = { new NativeConnector.Connection( connectString, connectTimeout, @@ -128,7 +128,7 @@ object NativeConnector { @volatile protected[this] var zookeeper: Option[ZooKeeper] = None - protected[this] val connectPromise = new Promise[ZooKeeper] + protected[this] val connectPromise: Promise[ZooKeeper] = new Promise[ZooKeeper] /** A ZooKeeper handle that will error if connectTimeout is specified and exceeded. */ lazy val connected: Future[ZooKeeper] = connectTimeout @@ -140,10 +140,10 @@ object NativeConnector { } .getOrElse(connectPromise) - protected[this] val releasePromise = new Promise[Unit] + protected[this] val releasePromise: Promise[Unit] = new Promise[Unit] val released: Future[Unit] = releasePromise - protected[this] val sessionBroker = new EventBroker + protected[this] val sessionBroker: EventBroker = new EventBroker /** * Publish session events, but intercept Connected events and use them to satisfy the pending @@ -185,7 +185,7 @@ object NativeConnector { connected } - protected[this] def mkZooKeeper = { + protected[this] def mkZooKeeper: ZooKeeper = { new ZooKeeper(connectString, sessionTimeout.inMillis.toInt, sessionBroker) } diff --git a/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala b/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala index 3246e77e91..fa2e77e1a8 100644 --- a/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala +++ b/util-zk/src/main/scala/com/twitter/zk/RetryPolicy.scala @@ -12,7 +12,7 @@ trait RetryPolicy { /** Matcher for connection-related KeeperExceptions. */ object KeeperConnectionException { - def unapply(e: KeeperException) = e match { + def unapply(e: KeeperException): Option[KeeperException] = e match { case e: KeeperException.ConnectionLossException => Some(e) case e: KeeperException.SessionExpiredException => Some(e) case e: KeeperException.SessionMovedException => Some(e) @@ -69,6 +69,6 @@ object RetryPolicy { /** A single try */ object None extends RetryPolicy { - def apply[T](op: => Future[T]) = op + def apply[T](op: => Future[T]): Future[T] = op } } diff --git a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala index de7af2606c..e04b67c7c6 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala @@ -9,6 +9,7 @@ import org.apache.zookeeper.{CreateMode, KeeperException, WatchedEvent} import com.twitter.concurrent.{Broker, Offer} import com.twitter.util.{Future, Return, Throw, Try} +import com.twitter.logging.Logger /** * A handle to a ZNode attached to a ZkClient @@ -19,14 +20,14 @@ trait ZNode { val path: String protected[zk] val zkClient: ZkClient - protected[this] lazy val log = zkClient.log + protected[this] lazy val log: Logger = zkClient.log - override def hashCode = path.hashCode + override def hashCode: Int = path.hashCode - override def toString = "ZNode(%s)".format(path) + override def toString: String = "ZNode(%s)".format(path) /** ZNodes are equal if they share a path. */ - override def equals(other: Any) = other match { + override def equals(other: Any): Boolean = other match { case z @ ZNode(_) => (z.hashCode == hashCode) case _ => false } @@ -36,7 +37,7 @@ trait ZNode { */ /** Return the ZkClient associated with this node. */ - def client = zkClient + def client: ZkClient = zkClient /** Get a child node. */ def apply(child: String): ZNode = ZNode(zkClient, childPath(child)) @@ -275,7 +276,7 @@ trait ZNode { /** AuthFailed and Expired are unmonitorable. Everything else can be resumed. */ protected[this] object MonitorableEvent { - def unapply(event: WatchedEvent) = event match { + def unapply(event: WatchedEvent): Boolean = event match { case StateEvent.AuthFailed() => false case StateEvent.Expired() => false case _ => true @@ -289,25 +290,25 @@ trait ZNode { object ZNode { /** Build a ZNode */ - def apply(zk: ZkClient, _path: String) = new ZNode { + def apply(zk: ZkClient, _path: String): ZNode = new ZNode { PathUtils.validatePath(_path) protected[zk] val zkClient = zk val path = _path } /** matcher */ - def unapply(znode: ZNode) = Some(znode.path) + def unapply(znode: ZNode): Some[String] = Some(znode.path) /** A matcher for KeeperExceptions that have a non-null path. */ object Error { - def unapply(ke: KeeperException) = Option(ke.getPath) + def unapply(ke: KeeperException): Option[String] = Option(ke.getPath) } /** A ZNode with its Stat metadata. */ trait Exists extends ZNode { val stat: Stat - override def equals(other: Any) = other match { + override def equals(other: Any): Boolean = other match { case Exists(p, s) => (p == path && s == stat) case o => super.equals(o) } @@ -317,13 +318,13 @@ object ZNode { } object Exists { - def apply(znode: ZNode, _stat: Stat) = new Exists { + def apply(znode: ZNode, _stat: Stat): Exists = new Exists { val path = znode.path protected[zk] val zkClient = znode.zkClient val stat = _stat } def apply(znode: Exists): Exists = apply(znode, znode.stat) - def unapply(znode: Exists) = Some((znode.path, znode.stat)) + def unapply(znode: Exists): Some[(String, Stat)] = Some((znode.path, znode.stat)) } /** A ZNode with its Stat metadata and children znodes. */ @@ -331,7 +332,7 @@ object ZNode { val stat: Stat val children: Seq[ZNode] - override def equals(other: Any) = other match { + override def equals(other: Any): Boolean = other match { case Children(p, s, c) => (p == path && s == stat && c == children) case o => super.equals(o) } @@ -347,7 +348,7 @@ object ZNode { def apply(znode: ZNode, stat: Stat, children: Seq[String]): Children = { apply(Exists(znode, stat), children.map(znode.apply)) } - def unapply(z: Children) = Some((z.path, z.stat, z.children)) + def unapply(z: Children): Some[(String, Stat, Seq[ZNode])] = Some((z.path, z.stat, z.children)) } /** A ZNode with its Stat metadata and data. */ @@ -355,21 +356,22 @@ object ZNode { val stat: Stat val bytes: Array[Byte] - override def equals(other: Any) = other match { + override def equals(other: Any): Boolean = other match { case Data(p, s, b) => (p == path && s == stat && b == bytes) case o => super.equals(o) } } object Data { - def apply(znode: ZNode, _stat: Stat, _bytes: Array[Byte]) = new Data { + def apply(znode: ZNode, _stat: Stat, _bytes: Array[Byte]): Data = new Data { val path = znode.path protected[zk] val zkClient = znode.zkClient val stat = _stat val bytes = _bytes } def apply(znode: Exists, bytes: Array[Byte]): Data = apply(znode, znode.stat, bytes) - def unapply(znode: Data) = Some((znode.path, znode.stat, znode.bytes)) + def unapply(znode: Data): Some[(String, Stat, Array[Byte])] = + Some((znode.path, znode.stat, znode.bytes)) } case class Watch[T <: Exists](result: Try[T], update: Future[WatchedEvent]) { diff --git a/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala b/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala index 2a97c0c1c7..7020ab361a 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala @@ -18,7 +18,7 @@ import com.twitter.util.{Duration, Future, Timer} */ trait ZkClient { val name = "zk.client" - protected[zk] val log = Logger.get(name) + protected[zk] val log: Logger = Logger.get(name) /** Build a ZNode handle using this ZkClient. */ def apply(path: String): ZNode = ZNode(this, path) @@ -71,7 +71,7 @@ trait ZkClient { _acl: Seq[ACL] = acl, _mode: CreateMode = mode, _retryPolicy: RetryPolicy = retryPolicy - ) = new ZkClient { + ): ZkClient = new ZkClient { val connector = _connector override val acl = _acl override val mode = _mode @@ -82,7 +82,7 @@ trait ZkClient { object ZkClient { /** Build a ZkClient with a provided Connector */ - def apply(_connector: Connector) = new ZkClient { + def apply(_connector: Connector): ZkClient = new ZkClient { protected[this] val connector = _connector } diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala index e259046556..6075fd1554 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala @@ -121,7 +121,7 @@ sealed trait ShardPermit { case class Shard(id: Int, private val node: ZNode, private val permit: Permit) extends ShardPermit { - def release() = { + def release(): Unit = { node.delete() ensure { permit.release() } } } diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala index 0b88ca492e..a52c68eff3 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala @@ -51,8 +51,8 @@ class ZkAsyncSemaphore( private[this] val waitq = new ConcurrentLinkedQueue[(Promise[ZkSemaphorePermit], ZNode)] private[this] class ZkSemaphorePermit(node: ZNode) extends Permit { - val zkPath = node.path - val sequenceNumber = sequenceNumberOf(zkPath) + val zkPath: String = node.path + val sequenceNumber: Int = sequenceNumberOf(zkPath) override def release(): Unit = { node.delete() } From f35ae591930e2c37a965e66f707bb704f41070d1 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 16 Oct 2019 15:52:31 +0000 Subject: [PATCH 223/796] {finagle,util}: Upgrade the version of caffeine Problem / Solution Caffeine has released some performance improvements and generally improved things, so let's update to the latest version, 2.8.0 Differential Revision: https://phabricator.twitter.biz/D384592 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b3a5e4c9b1..6ac59ed1c3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,11 @@ New Features * util-core: Created public method Closable.stopCollectClosablesThread that stops CollectClosables thread. ``PHAB_ID=D382800`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Upgrade to caffeine 2.8.0 ``PHAB_ID=D384592`` + 19.10.0 ------- diff --git a/build.sbt b/build.sbt index ce882021c8..af8378796b 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,7 @@ val slf4jVersion = "1.7.21" val jacksonVersion = "2.9.9" val guavaLib = "com.google.guava" % "guava" % "19.0" -val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.3.4" +val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion From e6970ed1749eefbc3d34bfe36c8ef33ba823210d Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Thu, 24 Oct 2019 20:35:49 +0000 Subject: [PATCH 224/796] oss: update all projects with JDK 11 support and travis builds Problem We want to be able to build and test using JDK 11, but Dodo and Scrooge do not use compatible Scala and SBT versions. Solution Update Scala and SBT versions used by dodo and scrooge from Scala 2.10.6 to 2.10.7 and SBT from 0.13.16 to 0.13.18. Each SBT file will contain GC options that are compatible with the active JDK version. Some logic has been updated to fix runtime behavior in JDK 11. Result Projects can be compiled and tested by both JDK 8 and JDK 11. JIRA Issues: CSL-8513 Differential Revision: https://phabricator.twitter.biz/D365075 --- .travis.yml | 1 + CHANGELOG.rst | 3 +++ build.sbt | 23 ++++++++++++++++++ .../scala/com/twitter/app/ClassPath.scala | 23 +++++++++++++++--- .../scala/com/twitter/util/FutureTest.scala | 24 ++++++++++--------- 5 files changed, 60 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3ec5aaa998..8acca293f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,7 @@ scala: jdk: - openjdk8 + - openjdk11 notifications: slack: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6ac59ed1c3..4e00d11572 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* util: Add initial support for JDK 11 compatibility. ``PHAB_ID=D365075`` + + New Features ~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index af8378796b..9e38597c87 100644 --- a/build.sbt +++ b/build.sbt @@ -37,6 +37,15 @@ def travisTestJavaOptions: Seq[String] = { } def gcJavaOptions: Seq[String] = { + val javaVersion = System.getProperty("java.version") + if (javaVersion.startsWith("1.8")) { + jdk8GcJavaOptions + } else { + jdk11GcJavaOptions + } +} + +def jdk8GcJavaOptions: Seq[String] = { Seq( "-XX:+UseParNewGC", "-XX:+UseConcMarkSweepGC", @@ -51,6 +60,20 @@ def gcJavaOptions: Seq[String] = { ) } +def jdk11GcJavaOptions: Seq[String] = { + Seq( + "-XX:+UseConcMarkSweepGC", + "-XX:+CMSParallelRemarkEnabled", + "-XX:+CMSClassUnloadingEnabled", + "-XX:ReservedCodeCacheSize=128m", + "-XX:SurvivorRatio=128", + "-XX:MaxTenuringThreshold=0", + "-Xss8M", + "-Xms512M", + "-Xmx2G" + ) +} + val defaultProjectSettings = Seq( scalaVersion := "2.12.8", crossScalaVersions := Seq("2.11.12", "2.12.8", "2.13.0") diff --git a/util-app/src/main/scala/com/twitter/app/ClassPath.scala b/util-app/src/main/scala/com/twitter/app/ClassPath.scala index c7fa568f63..ad0d3b3c3f 100644 --- a/util-app/src/main/scala/com/twitter/app/ClassPath.scala +++ b/util-app/src/main/scala/com/twitter/app/ClassPath.scala @@ -1,9 +1,10 @@ package com.twitter.app import com.twitter.finagle.util.loadServiceIgnoredPaths -import java.io.{IOException, File} -import java.net.{URISyntaxException, URLClassLoader, URI} +import java.io.{File, IOException} +import java.net.{URI, URISyntaxException, URL, URLClassLoader} import java.nio.charset.MalformedInputException +import java.nio.file.Paths import java.util.jar.{JarEntry, JarFile} import scala.collection.mutable import scala.collection.mutable.Builder @@ -62,12 +63,28 @@ private[app] sealed abstract class ClassPath[CpInfo <: ClassPath.Info] { buf.result } + // Note: In JDK 9+ URLClassLoader is no longer the default ClassLoader. + // This method allows us to scan URL's on the class path that can be used + // but does NOT include the ModulePath introduced in JDK9. This method is + // used as a bridge between JDK 8 and JDK 9+. + // The method used here is attributed to https://stackoverflow.com/a/49557901. + // TODO - add suppport for the ModulePath after dropping JDK 8 support. + private[this] def urlsFromClasspath(): Array[URL] = { + val classpath: String = System.getProperty("java.class.path") + classpath.split(File.pathSeparator).map { pathEntry: String => + Paths.get(pathEntry).toAbsolutePath().toUri().toURL + } + } + // package protected for testing private[app] def getEntries(loader: ClassLoader): Seq[(URI, ClassLoader)] = { val parent = Option(loader.getParent) val ownURIs: Vector[(URI, ClassLoader)] = for { - urlLoader <- Vector(loader).collect { case u: URLClassLoader => u } + urlLoader <- Vector(loader).map { + case urlClassLoader: URLClassLoader => urlClassLoader + case cl => new URLClassLoader(urlsFromClasspath(), cl) + } urls <- Option(urlLoader.getURLs()).toVector url <- urls if url != null } yield (url.toURI -> loader) diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index 8e2defd953..be81e32c72 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -961,18 +961,20 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert } } - "doesn't leak the underlying promise after completion" in { - new MonitoredHelper { - val inner1 = new Promise[String] - val inner2 = new Promise[String] - val f: Future[String] = Future.monitored { inner2.ensure(()); inner1 } - val s: String = "." * 1024 - val sSize: Long = ObjectSizeCalculator.getObjectSize(s) - inner1.setValue(s) - val inner2Size: Long = ObjectSizeCalculator.getObjectSize(inner2) - assert(inner2Size < sSize) + // we only know this works as expected when running with JDK 8 + if (System.getProperty("java.version").startsWith("1.8")) + "doesn't leak the underlying promise after completion" in { + new MonitoredHelper { + val inner1 = new Promise[String] + val inner2 = new Promise[String] + val f: Future[String] = Future.monitored { inner2.ensure(()); inner1 } + val s: String = "." * 1024 + val sSize: Long = ObjectSizeCalculator.getObjectSize(s) + inner1.setValue(s) + val inner2Size: Long = ObjectSizeCalculator.getObjectSize(inner2) + assert(inner2Size < sSize) + } } - } } } From a47a219b618e2dac2135acf5ee629d25fafe480d Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 30 Oct 2019 18:51:45 +0000 Subject: [PATCH 225/796] util-core: Introduce Reader.readAllItems for generic types Problem: Reader.readAll was Buf biased, and it should be in BufReader. Solution: Move `readAll(r: Reader[Buf])` to `BufReader`, while introduce `readAllItems[A](r: Reader[A])` to the `Reader` object. JIRA Issues: CSL-8869 Differential Revision: https://phabricator.twitter.biz/D391346 --- CHANGELOG.rst | 12 +++- .../main/java/com/twitter/io/BufReaders.java | 23 +++++++ .../src/main/java/com/twitter/io/Readers.java | 10 +-- .../main/scala/com/twitter/io/BufReader.scala | 13 ++++ .../main/scala/com/twitter/io/Reader.scala | 20 +++--- .../com/twitter/io/exp/ActivitySource.scala | 8 +-- .../com/twitter/io/ReaderCompilationTest.java | 2 +- .../scala/com/twitter/io/BufReaderTest.scala | 8 ++- .../twitter/io/InputStreamReaderTest.scala | 10 +-- .../test/scala/com/twitter/io/PipeTest.scala | 4 +- .../scala/com/twitter/io/ReaderTest.scala | 62 +++++++++++-------- 11 files changed, 113 insertions(+), 59 deletions(-) create mode 100644 util-core/src/main/java/com/twitter/io/BufReaders.java diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4e00d11572..af22c70826 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,12 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -* util: Add initial support for JDK 11 compatibility. ``PHAB_ID=D365075`` - - New Features ~~~~~~~~~~~~ +* util: Add initial support for JDK 11 compatibility. ``PHAB_ID=D365075`` + * util-core: Created public method Closable.stopCollectClosablesThread that stops CollectClosables thread. ``PHAB_ID=D382800`` @@ -21,6 +20,13 @@ Runtime Behavior Changes * util: Upgrade to caffeine 2.8.0 ``PHAB_ID=D384592`` +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: Add `c.t.io.BufReader.readAll` to consume a `Reader[Buf]` and concat values to a Buf. + Replace `c.t.io.Reader.readAll` with `Reader.readAllItems`, the new API consumes a generic Reader[T], + and return a Seq of items. ``PHAB_ID=D391346`` + 19.10.0 ------- diff --git a/util-core/src/main/java/com/twitter/io/BufReaders.java b/util-core/src/main/java/com/twitter/io/BufReaders.java new file mode 100644 index 0000000000..a6c5b53ae6 --- /dev/null +++ b/util-core/src/main/java/com/twitter/io/BufReaders.java @@ -0,0 +1,23 @@ +package com.twitter.io; + +import com.twitter.util.Future; + +/** + * Better Java APIs of BufReader for Scala 2.11. + * This will be removed when drop Scala 2.11 support. + * + * @see com.twitter.io.BufReader + */ +public final class BufReaders { + + private BufReaders() { + throw new IllegalStateException(); + } + + /** + * See {@code com.twitter.io.BufReader.readAll}. + */ + public static Future readAll(Reader r) { + return BufReader$.MODULE$.readAll(r); + } +} diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index 89ba0c2543..e746e508eb 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -14,7 +14,8 @@ import scala.runtime.BoxedUnit; /** - * Java APIs for Reader. + * Better Java APIs of Reader for Scala 2.11. + * This will be removed when drop Scala 2.11 support. * * @see com.twitter.io.Reader */ @@ -33,13 +34,6 @@ public static Reader newBufReader(Buf buf, int chunkSize) { return Reader$.MODULE$.fromBuf(buf, chunkSize); } - /** - * See {@code com.twitter.io.Reader.readAll}. - */ - public static Future readAll(Reader r) { - return Reader$.MODULE$.readAll(r); - } - /** * See {@code com.twitter.io.Reader.concat}. */ diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index 357afca546..445efebf74 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -52,4 +52,17 @@ object BufReader { */ def apply(buf: Buf, chunkSize: Int): Reader[Buf] = if (buf.isEmpty) Reader.empty[Buf] else new BufReader(buf, chunkSize) + + /** + * Read the entire bytestream presented by `r`. + */ + def readAll(r: Reader[Buf]): Future[Buf] = { + def loop(left: Buf): Future[Buf] = + r.read().flatMap { + case Some(right) => loop(left.concat(right)) + case _ => Future.value(left) + } + + loop(Buf.Empty) + } } diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 6e404c29a5..d61f8afe8d 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -248,16 +248,18 @@ object Reader { def exception[A](e: Throwable): Reader[A] = fromFuture[A](Future.exception(e)) /** - * Read the entire bytestream presented by `r`. + * Read all items from the Reader r. + * @return A Sequence of items. */ - def readAll(r: Reader[Buf]): Future[Buf] = { - def loop(left: Buf): Future[Buf] = - r.read().flatMap { - case Some(right) => loop(left concat right) - case _ => Future.value(left) - } - - loop(Buf.Empty) + def readAllItems[A](r: Reader[A]): Future[Seq[A]] = { + val acc = List.newBuilder[A] + def loop(): Future[List[A]] = r.read().flatMap { + case None => Future.value(acc.result()) + case Some(t) => + acc += t + loop() + } + loop() } /** diff --git a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala b/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala index 4c24508126..ed939121b7 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala +++ b/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala @@ -1,8 +1,8 @@ package com.twitter.io.exp -import com.twitter.io.{InputStreamReader, Buf, Reader} +import com.twitter.io.{Buf, BufReader, InputStreamReader} import com.twitter.util._ -import java.io.{FileInputStream, File} +import java.io.{File, FileInputStream} import java.lang.ref.{ReferenceQueue, WeakReference} import java.util.concurrent.atomic.AtomicBoolean import java.util.HashMap @@ -138,7 +138,7 @@ class FilePollingActivitySource private[exp] ( InputStreamReader.DefaultMaxBufferSize, pool ) - Reader.readAll(reader) respond { + BufReader.readAll(reader) respond { case Return(buf) => value() = Activity.Ok(buf) case Throw(cause) => @@ -187,7 +187,7 @@ class ClassLoaderActivitySource private[exp] (classLoader: ClassLoader, pool: Fu case stream => val reader = new InputStreamReader(stream, InputStreamReader.DefaultMaxBufferSize, pool) - Reader.readAll(reader) respond { + BufReader.readAll(reader) respond { case Return(buf) => p.setValue(Activity.Ok(buf)) case Throw(cause) => diff --git a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java b/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java index 4d1ec8db43..61c6d54180 100644 --- a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java +++ b/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java @@ -18,7 +18,7 @@ public void testNewBufReader() { @Test public void testReadAll() { - Readers.readAll(Readers.newEmptyReader()); + BufReaders.readAll(Readers.newEmptyReader()); } @Test diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index f66780c60a..bb32b91eaf 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -13,7 +13,7 @@ class BufReaderTest extends FunSuite with Checkers { check { bytes: String => val buf = Buf.Utf8(bytes) val r = Reader.fromBuf(buf, 8) - await(Reader.readAll(r)) == buf + await(BufReader.readAll(r)) == buf } } @@ -30,4 +30,10 @@ class BufReaderTest extends FunSuite with Checkers { ) } } + test("BufReader - readAll") { + check { bytes: String => + val r = Reader.fromBuf(Buf.Utf8(bytes)) + await(BufReader.readAll(r)) == Buf.Utf8(bytes) + } + } } diff --git a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala index be8458e7a7..40bd4e1102 100644 --- a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala @@ -62,16 +62,16 @@ class InputStreamReaderTest extends FunSuite { val r1 = new InputStreamReader(s1, 100) val r2 = new InputStreamReader(s2, 500) - val f1 = Reader.readAll(r1) + val f1 = BufReader.readAll(r1) assert(Await.result(f1, 5.seconds) == buf(0, 250)) - val f2 = Reader.readAll(r1) + val f2 = BufReader.readAll(r1) assert(Await.result(f2, 5.seconds).isEmpty) - val f3 = Reader.readAll(r2) + val f3 = BufReader.readAll(r2) assert(Await.result(f3, 5.seconds) == buf(0, 250)) - val f4 = Reader.readAll(r2) + val f4 = BufReader.readAll(r2) assert(Await.result(f4, 5.seconds).isEmpty) } @@ -108,7 +108,7 @@ class InputStreamReaderTest extends FunSuite { test("reading EOF closes the InputStream") { val in = spy(new ByteArrayInputStream(arr(0, 10))) val reader = new InputStreamReader(in, 100, FuturePool.immediatePool) - val f = Reader.readAll(reader) + val f = BufReader.readAll(reader) assert(Await.result(f, 5.seconds) == buf(0, 10)) verify(in, times(1)).close() } diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index d2b79646b0..4420f00776 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -72,7 +72,7 @@ class PipeTest extends FunSuite with Matchers { test("Reader.readAll") { val rw = new Pipe[Buf] - val all = Reader.readAll(rw) + val all = BufReader.readAll(rw) assert(!all.isDefined) assertWrite(rw, 0, 3) assertWrite(rw, 3, 6) @@ -430,7 +430,7 @@ class PipeTest extends FunSuite with Matchers { rw.close(1.second) assert(!writef.isDefined) - assert(await(Reader.readAll(rw)) == buf) + assert(await(BufReader.readAll(rw)) == buf) assertReadNone(rw) assert(writef.isDefined) diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 8b756546df..1093e82aef 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -3,7 +3,7 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.conversions.DurationOps._ import com.twitter.conversions.StorageUnitOps._ -import com.twitter.util.{Await, Awaitable, Future, Promise, Return, Try} +import com.twitter.util.{Await, Awaitable, Future, Promise} import java.io.{ByteArrayInputStream, ByteArrayOutputStream} import java.nio.charset.{StandardCharsets => JChar} import java.util.concurrent.atomic.AtomicBoolean @@ -55,16 +55,6 @@ class ReaderTest } } - private def readAllString(r: Reader[String]): Future[String] = { - def loop(left: StringBuilder): Future[String] = { - r.read.flatMap { - case Some(right) => loop(left.append(right)) - case _ => Future.value(left.toString()) - } - } - loop(StringBuilder.newBuilder) - } - private def writeLoop[T](from: List[T], to: Writer[T]): Future[Unit] = from match { case h :: t => to.write(h).flatMap(_ => writeLoop(t, to)) @@ -150,9 +140,10 @@ class ReaderTest test("Reader.concat") { forAll { ss: List[String] => - val readers = ss.map(s => Reader.fromBuf(Buf.Utf8(s), 16)) - val buf = Reader.readAll(Reader.concat(AsyncStream.fromSeq(readers))) - assert(await(buf) == Buf.Utf8(ss.mkString)) + val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) + val concatedReaders = Reader.concat(AsyncStream.fromSeq(readers)) + val values = Reader.readAllItems(concatedReaders) + assert(await(values) == ss) } } @@ -205,7 +196,7 @@ class ReaderTest } val combined = Reader.concat(head +:: tail) - val buf = Reader.readAll(combined) + val buf = Reader.readAllItems(combined) intercept[Exception] { await(buf) } @@ -214,17 +205,17 @@ class ReaderTest test("Reader.flatten") { forAll { ss: List[String] => - val readers = ss.map(s => Reader.fromBuf(Buf.Utf8(s), 16)) - val buf = Reader.readAll(Reader.flatten(Reader.fromSeq(readers))) - assert(await(buf) == Buf.Utf8(ss.mkString)) + val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) + val value = Reader.readAllItems(Reader.flatten(Reader.fromSeq(readers))) + assert(await(value) == ss) } } test("Reader#flatten") { forAll { ss: List[String] => - val readers = ss.map(s => Reader.fromBuf(Buf.Utf8(s), 16)) - val buf = Reader.readAll((Reader.fromSeq(readers).flatten)) - assert(await(buf) == Buf.Utf8(ss.mkString)) + val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) + val buf = Reader.readAllItems((Reader.fromSeq(readers).flatten)) + assert(await(buf) == ss) } } @@ -337,8 +328,8 @@ class ReaderTest test("Reader.fromStream closes resources on EOF read") { val in = spy(new ByteArrayInputStream(arr(0, 10))) - val r = Reader.fromStream(in, 4) - val f = Reader.readAll(r) + val r: Reader[Buf] = Reader.fromStream(in, 4) + val f = BufReader.readAll(r) assert(await(f) == buf(0, 10)) eventually { verify(in).close() @@ -362,7 +353,7 @@ class ReaderTest test("Reader.fromAsyncStream fails on exceptional stream") { val as = AsyncStream.exception(new Exception()) val r = Reader.fromAsyncStream(as) - val f = Reader.readAll(r) + val f = Reader.readAllItems(r) intercept[Exception] { await(f) } @@ -441,7 +432,7 @@ class ReaderTest pipe } - assert(Buf.decodeString(await(Reader.readAll(reader2)), JChar.UTF_8) == s) + assert(Buf.decodeString(await(BufReader.readAll(reader2)), JChar.UTF_8) == s) } } @@ -564,7 +555,7 @@ class ReaderTest val pipe = new Pipe[Int] writeLoop(l, pipe) val reader2 = pipe.map(_.toString) - assert(await(readAllString(reader2)) == l.mkString) + assert(await(Reader.readAllItems(reader2)) == l.map(_.toString)) } } @@ -586,6 +577,25 @@ class ReaderTest } assert(await(reader.onClose) == StreamTermination.Discarded) } + + test("Reader.readAll") { + val genBuf: Gen[Buf] = { + for { + // limit arrays to a few kilobytes, otherwise we may generate a very large amount of data + numBytes <- Gen.choose(0.bytes.inBytes, 2.kilobytes.inBytes) + bytes <- Gen.containerOfN[Array, Byte](numBytes.toInt, Arbitrary.arbitrary[Byte]) + } yield { + Buf.ByteArray.Owned(bytes) + } + } + + val genList = + Gen.listOf(Gen.oneOf(Gen.alphaLowerStr, Gen.alphaNumStr, Gen.choose(1, 100), genBuf)) + forAll(genList) { l => + val r = Reader.fromSeq(l) + assert(await(Reader.readAllItems(r)) == l) + } + } } object ReaderTest { From d1b42f4bc8ee281d33ef4a52e2d5295ad410250c Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 31 Oct 2019 00:23:33 +0000 Subject: [PATCH 226/796] util-core: Unify `BufReader` and `SeqReader` Problem `Reader.fromBuf` and `Reader.fromSeq` are really similar. One thing `Reader.fromBuf` provides is internal chunking, which can be mimic'd via `Reader.fromSeq`, where each element in a seq is a chunk. Solution Introduce `IteratorReader` as a standalone reader as well as the implementation of both `BufReader` and `SeqReader`. JIRA Issues: CSL-7490 Differential Revision: https://phabricator.twitter.biz/D391769 --- CHANGELOG.rst | 5 ++ .../main/scala/com/twitter/io/BufReader.scala | 60 +++++++------------ .../{SeqReader.scala => IteratorReader.scala} | 16 +++-- .../main/scala/com/twitter/io/Reader.scala | 14 ++++- .../scala/com/twitter/io/BufReaderTest.scala | 50 ++++++++++++---- .../com/twitter/io/IteratorReaderTest.scala | 30 ++++++++++ 6 files changed, 117 insertions(+), 58 deletions(-) rename util-core/src/main/scala/com/twitter/io/{SeqReader.scala => IteratorReader.scala} (84%) create mode 100644 util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index af22c70826..4a91e30b37 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,11 @@ New Features * util-core: Created public method Closable.stopCollectClosablesThread that stops CollectClosables thread. ``PHAB_ID=D382800`` +* util-core: Introduced `Reader.fromIterator` to create a Reader from an iterator. It is not +recommended to call `iterator.next()` after creating a `Reader` from it. Doing so will affect the +behavior of `Reader.read()` because it will skip the value returned from `iterator.next`. +``PHAB_ID=D391769`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index 445efebf74..dfa29f7b0c 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -1,41 +1,7 @@ package com.twitter.io -import com.twitter.util.{Future, Return, Try, Throw, ConstFuture, Promise} - -/** - * Construct a [[Reader]] from a [[Buf]]. - */ -private[io] final class BufReader(buf: Buf, chunkSize: Int) extends Reader[Buf] { - private[this] var state: Try[Buf] = Return(buf) - private[this] val closep = Promise[StreamTermination]() - - def read(): Future[Option[Buf]] = { - val result: Future[Option[Buf]] = synchronized { - state match { - case Return(Buf.Empty) => Future.None - case Return(b) => - state = Return(b.slice(chunkSize, Int.MaxValue)) - Future.value(Some(b.slice(0, chunkSize))) - - case t: Throw[_] => - new ConstFuture[Option[Buf]](t.cast[Option[Buf]]) - } - } - if (result eq Future.None) { - closep.updateIfEmpty(StreamTermination.FullyRead.Return) - } - result - } - - def discard(): Unit = { - synchronized { - state = Throw(new ReaderDiscardedException) - } - closep.updateIfEmpty(StreamTermination.Discarded.Return) - } - - def onClose: Future[StreamTermination] = closep -} +import com.twitter.util.Future +import java.util.NoSuchElementException object BufReader { @@ -50,8 +16,26 @@ object BufReader { * Creates a [[BufReader]] out of a given `buf`. The result [[Reader]] emits chunks of at most * `chunkSize`. */ - def apply(buf: Buf, chunkSize: Int): Reader[Buf] = - if (buf.isEmpty) Reader.empty[Buf] else new BufReader(buf, chunkSize) + def apply(buf: Buf, chunkSize: Int): Reader[Buf] = { + if (buf.isEmpty) Reader.empty[Buf] + else new IteratorReader[Buf](iterator(buf, chunkSize)) + } + + /** + * Returns an iterator that iterates over the given [[Buf]] by `chunkSize`. + */ + private[io] def iterator(buf: Buf, chunkSize: Int): Iterator[Buf] = new Iterator[Buf] { + require(chunkSize > 0, s"Then chunkSize should be greater than 0, but received: $chunkSize") + private[this] var remainingBuf: Buf = buf + + def hasNext: Boolean = !remainingBuf.isEmpty + def next(): Buf = { + if (!hasNext) throw new NoSuchElementException("next() on empty iterator") + val nextChuck = remainingBuf.slice(0, chunkSize) + remainingBuf = remainingBuf.slice(chunkSize, Int.MaxValue) + nextChuck + } + } /** * Read the entire bytestream presented by `r`. diff --git a/util-core/src/main/scala/com/twitter/io/SeqReader.scala b/util-core/src/main/scala/com/twitter/io/IteratorReader.scala similarity index 84% rename from util-core/src/main/scala/com/twitter/io/SeqReader.scala rename to util-core/src/main/scala/com/twitter/io/IteratorReader.scala index 50407a0b9d..b3c41be987 100644 --- a/util-core/src/main/scala/com/twitter/io/SeqReader.scala +++ b/util-core/src/main/scala/com/twitter/io/IteratorReader.scala @@ -1,7 +1,6 @@ package com.twitter.io import com.twitter.util.{Future, Promise} -import scala.collection.Iterable /** * We want to ensure that this reader always satisfies these invariants: @@ -11,19 +10,18 @@ import scala.collection.Iterable * We achieved this with a state machine where any access to the state is synchronized, * and by preventing changing the state when the reader is fully read or discarded. */ -private[io] final class SeqReader[A](it: Iterable[A]) extends Reader[A] { - import SeqReader._ +private[io] final class IteratorReader[A](it: Iterator[A]) extends Reader[A] { + import IteratorReader._ private[this] val closep = Promise[StreamTermination]() - private[this] val value: Iterator[A] = synchronized { it.iterator } private[this] var state: State = State.Idle def read(): Future[Option[A]] = { val result = synchronized { state match { case State.Idle => - if (value.hasNext) { - val n = value.next() + if (it.hasNext) { + val n = it.next() Future.value(Some(n)) } else { state = State.FullyRead @@ -58,13 +56,13 @@ private[io] final class SeqReader[A](it: Iterable[A]) extends Reader[A] { def onClose: Future[StreamTermination] = closep } -object SeqReader { +private object IteratorReader { /** * Indicates reader state when the reader is created via SeqReader */ - private sealed trait State - private object State { + sealed trait State + object State { /** Indicates the reader is ready to be read. */ case object Idle extends State diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index d61f8afe8d..ff2766ec5d 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -305,6 +305,18 @@ object Reader { */ def fromFile(f: File, chunkSize: Int): Reader[Buf] = fromStream(new FileInputStream(f), chunkSize) + /** + * Create a new [[Reader]] from a given [[Iterator]]. + * + * The resources held by the returned [[Reader]] are released on reading of EOF and + * [[Reader.discard()]]. + * + * @note It is not recommended to call `it.next()` after creating a `Reader` from it. + * Doing so will affect the behavior of `Reader.read()` because it will skip + * the value returned from `it.next`. + */ + def fromIterator[A](it: Iterator[A]): Reader[A] = new IteratorReader[A](it) + /** * Create a new [[Reader]] from a given [[InputStream]]. The output of a returned reader is * chunked by at most `chunkSize` (bytes). @@ -334,7 +346,7 @@ object Reader { * * @note Multiple outstanding reads are not allowed on this reader. */ - def fromSeq[A](seq: Seq[A]): Reader[A] = new SeqReader(seq) + def fromSeq[A](seq: Seq[A]): Reader[A] = fromIterator(seq.iterator) /** * Allow [[AsyncStream]] to be consumed as a [[Reader]] diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index bb32b91eaf..d797b725a1 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -2,6 +2,8 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} +import org.scalacheck.Prop.forAll +import org.scalacheck.Gen import org.scalatest.FunSuite import org.scalatestplus.scalacheck.Checkers @@ -18,16 +20,44 @@ class BufReaderTest extends FunSuite with Checkers { } test("BufReader - discard") { - check { (bytes: String, n: Int) => - val r = Reader.fromBuf(Buf.Utf8(bytes), n) - r.discard() - - n < 0 || - bytes.length == 0 || - Await - .ready(r.read(), 5.seconds).poll.exists( - _.throwable.isInstanceOf[ReaderDiscardedException] - ) + val alphaString = for { + s <- Gen.alphaStr + } yield s + val positiveInt = for { + n <- Gen.posNum[Int] + } yield n + + check { + forAll(alphaString, positiveInt) { (bytes, n) => + val r = Reader.fromBuf(Buf.Utf8(bytes), n) + r.discard() + + bytes.length == 0 || + Await + .ready(r.read(), 5.seconds).poll.exists( + _.throwable.isInstanceOf[ReaderDiscardedException] + ) + } + } + } + + test("BufReader - iterator") { + val alphaString = for { + s <- Gen.alphaStr + } yield s + val positiveInt = for { + n <- Gen.posNum[Int] + } yield n + + check { + forAll(alphaString, positiveInt) { (bytes, n) => + var result = Buf.Empty + val iterator = BufReader.iterator(Buf.Utf8(bytes), n) + while (iterator.hasNext) { + result = result.concat(iterator.next) + } + Buf.Utf8(bytes) == result + } } } test("BufReader - readAll") { diff --git a/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala b/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala new file mode 100644 index 0000000000..7e898389e0 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala @@ -0,0 +1,30 @@ +package com.twitter.io + +import com.twitter.conversions.DurationOps._ +import com.twitter.util.{Await, Future} +import org.scalatest.FunSuite +import org.scalatestplus.scalacheck.Checkers + +class IteratorReaderTest extends FunSuite with Checkers { + + private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) + + test("IteratorReader") { + check { list: List[String] => + val r = Reader.fromIterator(list.iterator) + await(Reader.readAllItems(r)).mkString == list.mkString + } + } + + test("IteratorReader - discard") { + check { list: List[String] => + val r = Reader.fromIterator(list.iterator) + r.discard() + + Await + .ready(r.read(), 5.seconds).poll.exists( + _.throwable.isInstanceOf[ReaderDiscardedException] + ) + } + } +} From 5e73766036b77be982fa67a124920669032f1a1c Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 31 Oct 2019 04:19:02 +0000 Subject: [PATCH 227/796] util-stats: Add notes about scope performance Problem We don't provide users with guidance on whether or not to `scope` their stats receivers. Solution Add notes to both scaladoc and metrics user guide about the scope efficiency. JIRA Issues: CSL-7064 Differential Revision: https://phabricator.twitter.biz/D392650 --- doc/src/sphinx/util-stats/user_guide.rst | 25 ++++++++++++++++ .../twitter/finagle/stats/StatsReceiver.scala | 30 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/doc/src/sphinx/util-stats/user_guide.rst b/doc/src/sphinx/util-stats/user_guide.rst index 37deb280fb..559bd12172 100644 --- a/doc/src/sphinx/util-stats/user_guide.rst +++ b/doc/src/sphinx/util-stats/user_guide.rst @@ -88,6 +88,31 @@ along with code that holds a strong reference to the gauge in a global linked li such you should only rely on ``provideGauge`` when you do not have a place to keep a strong reference. + +Prefer fully-qualified names over scoping +----------------------------------------- +There is a convenient API method (``scope``) allowing to spawn a "scoped" version of a given +``StatsReceiver``. Although it enables effortless code reuse, scoping comes at the cost of extra +allocations. Inlining a fully-qualified metric name directly into the constructing method yields +the same outcome yet avoids the overhead needed to accommodate interim structures. + +Put this way, if possible, prefer this + +.. code-block:: scala + + statsReceiver.counter("foo", "bar", "baz") + +over this + +.. code-block:: scala + + statsReceiver.scope("foo").scope("bar").counter("baz") + +It's important to note that while this optimization is appealing like an easy win, it shouldn't +be universally applied. There are many legitimate use-cases when scoping a ``StatsReceiver`` +is very reasonable thing to do, and otherwise, would require an alternative channel for the scope +to be propagated between components, presumably introducing the overhead somewhere else. + .. _testing_code: Testing code that use StatsReceivers diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 8db7b754c2..5f43d6451b 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -254,6 +254,21 @@ trait StatsReceiver { * }}} * * will generate [[Counter counters]] named `/client/adds` and `/client/backend/adds`. + * + * Note it's recommended to be mindful with usage of the `scope` method as it's almost always + * more efficient to pass a full metric name directly to a constructing method. + * + * Put this way, whenever possible prefer + * + * {{{ + * statsReceiver.counter("client", "adds") + * }}} + * + * to + * + * {{{ + * statsReceiver.scope("client").counter("adds") + * }}} */ def scope(namespace: String): StatsReceiver = { if (namespace == "") this @@ -274,6 +289,21 @@ trait StatsReceiver { * }}} * * will generate a [[Counter counter]] named `/client/backend/pool/adds`. + * + * Note it's recommended to be mindful with usage of the `scope` method as it's almost always + * more efficient to pass a full metric name directly to a constructing method. + * + * Put this way, whenever possible prefer + * + * {{{ + * statsReceiver.counter("client", "backend", "pool", "adds") + * }}} + * + * to + * + * {{{ + * statsReceiver.scope("client", "backend", "pool").counter("adds") + * }}} */ @varargs final def scope(namespaces: String*): StatsReceiver = From 459038f9533a5c38c6e1f4359aee1227ad772170 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 31 Oct 2019 19:27:50 +0000 Subject: [PATCH 228/796] util-core: Move chunked and framed to BufReader Problem/Solution: Reader.chunked and Reader.framed are Buf specific, move them to BufReader. JIRA Issues: CSL-6756 Differential Revision: https://phabricator.twitter.biz/D392198 --- CHANGELOG.rst | 3 + .../main/scala/com/twitter/io/BufReader.scala | 74 ++++++++++ .../main/scala/com/twitter/io/Reader.scala | 74 ---------- .../scala/com/twitter/io/BufReaderTest.scala | 139 ++++++++++++++---- .../scala/com/twitter/io/ReaderTest.scala | 88 +---------- 5 files changed, 188 insertions(+), 190 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4a91e30b37..fcabd4ecc6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -32,6 +32,9 @@ Breaking API Changes Replace `c.t.io.Reader.readAll` with `Reader.readAllItems`, the new API consumes a generic Reader[T], and return a Seq of items. ``PHAB_ID=D391346`` +* util-core: Moved `c.t.io.Reader.chunked` to `c.t.io.BufReader.chunked`, and `Reader.framed` to + `BufReader.framed`. ``PHAB_ID=D392198`` + 19.10.0 ------- diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index dfa29f7b0c..223b5e982a 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -2,6 +2,8 @@ package com.twitter.io import com.twitter.util.Future import java.util.NoSuchElementException +import scala.annotation.tailrec +import scala.collection.mutable.ListBuffer object BufReader { @@ -49,4 +51,76 @@ object BufReader { loop(Buf.Empty) } + + // see BufReader.chunked + private final class ChunkedFramer(chunkSize: Int) extends (Buf => Seq[Buf]) { + require(chunkSize > 0, s"chunkSize should be > 0 but was $chunkSize") + + @tailrec + private def loop(acc: ListBuffer[Buf], in: Buf): Seq[Buf] = { + if (in.length < chunkSize) (acc :+ in).toSeq + else { + loop( + acc :+ in.slice(0, chunkSize), + in.slice(chunkSize, in.length) + ) + } + } + + def apply(in: Buf): Seq[Buf] = { + loop(ListBuffer(), in) + } + } + + // see BufReader.framed + private final class Framed(r: Reader[Buf], framer: Buf => Seq[Buf]) + extends Reader[Buf] + with (Option[Buf] => Future[Option[Buf]]) { + + private[this] var frames: Seq[Buf] = Nil + + // we only enter here when `frames` is empty. + def apply(in: Option[Buf]): Future[Option[Buf]] = synchronized { + in match { + case Some(data) => + frames = framer(data) + read() + case None => + Future.None + } + } + + def read(): Future[Option[Buf]] = synchronized { + if (frames.isEmpty) { + // flatMap to `this` to prevent allocating + r.read().flatMap(this) + } else { + val nextFrame = frames.head + frames = frames.tail + Future.value(Some(nextFrame)) + } + } + + def discard(): Unit = synchronized { + frames = Seq.empty + r.discard() + } + + def onClose: Future[StreamTermination] = r.onClose + } + + /** + * Chunk the output of a given [[Reader]] by at most `chunkSize` (bytes). This consumes the + * reader. + */ + def chunked(r: Reader[Buf], chunkSize: Int): Reader[Buf] = + new Framed(r, new ChunkedFramer(chunkSize)) + + /** + * Wraps a [[ Reader[Buf] ]] and emits frames as decided by `framer`. + * + * @note The returned `Reader` may not be thread safe depending on the behavior + * of the framer. + */ + def framed(r: Reader[Buf], framer: Buf => Seq[Buf]): Reader[Buf] = new Framed(r, framer) } diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index ff2766ec5d..724b624eca 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -4,8 +4,6 @@ import com.twitter.concurrent.AsyncStream import com.twitter.util.Promise.Detachable import com.twitter.util._ import java.io.{File, FileInputStream, InputStream} -import scala.annotation.tailrec -import scala.collection.mutable.ListBuffer /** * A reader exposes a pull-based API to model a potentially infinite stream of arbitrary elements. @@ -172,63 +170,6 @@ object Reader { def onClose: Future[StreamTermination] = closep } - // see Reader.chunked - private final class ChunkedFramer(chunkSize: Int) extends (Buf => Seq[Buf]) { - require(chunkSize > 0, s"chunkSize should be > 0 but was $chunkSize") - - @tailrec - private def loop(acc: ListBuffer[Buf], in: Buf): Seq[Buf] = { - if (in.length < chunkSize) (acc :+ in).toSeq - else { - loop( - acc :+ in.slice(0, chunkSize), - in.slice(chunkSize, in.length) - ) - } - } - - def apply(in: Buf): Seq[Buf] = { - loop(ListBuffer(), in) - } - } - - // see Reader.framed - private final class Framed(r: Reader[Buf], framer: Buf => Seq[Buf]) - extends Reader[Buf] - with (Option[Buf] => Future[Option[Buf]]) { - - private[this] var frames: Seq[Buf] = Nil - - // we only enter here when `frames` is empty. - def apply(in: Option[Buf]): Future[Option[Buf]] = synchronized { - in match { - case Some(data) => - frames = framer(data) - read() - case None => - Future.None - } - } - - def read(): Future[Option[Buf]] = synchronized { - if (frames.isEmpty) { - // flatMap to `this` to prevent allocating - r.read().flatMap(this) - } else { - val nextFrame = frames.head - frames = frames.tail - Future.value(Some(nextFrame)) - } - } - - def discard(): Unit = synchronized { - frames = Seq.empty - r.discard() - } - - def onClose: Future[StreamTermination] = r.onClose - } - /** * Construct a `Reader` from a `Future` * @@ -262,13 +203,6 @@ object Reader { loop() } - /** - * Chunk the output of a given [[Reader]] by at most `chunkSize` (bytes). This consumes the - * reader. - */ - def chunked(r: Reader[Buf], chunkSize: Int): Reader[Buf] = - new Framed(r, new ChunkedFramer(chunkSize)) - /** * Create a new [[Reader]] from a given [[Buf]]. The output of a returned reader is chunked by * a least `chunkSize` (bytes). @@ -504,12 +438,4 @@ object Reader { p.setInterruptHandler { case _ => r.discard() } p } - - /** - * Wraps a [[ Reader[Buf] ]] and emits frames as decided by `framer`. - * - * @note The returned `Reader` may not be thread safe depending on the behavior - * of the framer. - */ - def framed(r: Reader[Buf], framer: Buf => Seq[Buf]): Reader[Buf] = new Framed(r, framer) } diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index d797b725a1..a533a8a08c 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -1,69 +1,148 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ +import com.twitter.conversions.StorageUnitOps._ import com.twitter.util.{Await, Future} -import org.scalacheck.Prop.forAll -import org.scalacheck.Gen +import org.scalacheck.{Arbitrary, Gen} import org.scalatest.FunSuite -import org.scalatestplus.scalacheck.Checkers +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import scala.annotation.tailrec -class BufReaderTest extends FunSuite with Checkers { +class BufReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) test("BufReader") { - check { bytes: String => + forAll { bytes: String => val buf = Buf.Utf8(bytes) val r = Reader.fromBuf(buf, 8) - await(BufReader.readAll(r)) == buf + assert(await(BufReader.readAll(r)) == buf) } } test("BufReader - discard") { - val alphaString = for { + val stringAndChunk = for { s <- Gen.alphaStr - } yield s - val positiveInt = for { - n <- Gen.posNum[Int] - } yield n + i <- Gen.posNum[Int].suchThat(_ <= s.length) + } yield (s, i) - check { - forAll(alphaString, positiveInt) { (bytes, n) => + forAll(stringAndChunk) { + case (bytes, n) => val r = Reader.fromBuf(Buf.Utf8(bytes), n) r.discard() - bytes.length == 0 || - Await - .ready(r.read(), 5.seconds).poll.exists( - _.throwable.isInstanceOf[ReaderDiscardedException] - ) - } + assert( + bytes.length == 0 || + Await + .ready(r.read(), 5.seconds).poll + .exists(_.throwable.isInstanceOf[ReaderDiscardedException])) } } test("BufReader - iterator") { - val alphaString = for { + val stringAndChunk = for { s <- Gen.alphaStr - } yield s - val positiveInt = for { - n <- Gen.posNum[Int] - } yield n + i <- Gen.posNum[Int].suchThat(_ <= s.length) + } yield (s, i) - check { - forAll(alphaString, positiveInt) { (bytes, n) => + forAll(stringAndChunk) { + case (bytes, n) => var result = Buf.Empty val iterator = BufReader.iterator(Buf.Utf8(bytes), n) while (iterator.hasNext) { result = result.concat(iterator.next) } - Buf.Utf8(bytes) == result - } + assert(Buf.Utf8(bytes) == result) } } + test("BufReader - readAll") { - check { bytes: String => + forAll { bytes: String => val r = Reader.fromBuf(Buf.Utf8(bytes)) - await(BufReader.readAll(r)) == Buf.Utf8(bytes) + assert(await(BufReader.readAll(r)) == Buf.Utf8(bytes)) + } + } + + test("BufReader.chunked") { + val stringAndChunk = for { + s <- Gen.alphaStr + i <- Gen.posNum[Int].suchThat(_ <= s.length) + } yield (s, i) + + forAll(stringAndChunk) { + case (s, i) => + val r = BufReader.chunked(Reader.fromBuf(Buf.Utf8(s), 32), i) + + def readLoop(): Unit = await(r.read()) match { + case Some(b) => + assert(b.length <= i) + readLoop() + case None => () + } + + readLoop() + } + } + + test("BufReader.framed reads framed data") { + val getByteArrays: Gen[Seq[Buf]] = Gen.listOf( + for { + // limit arrays to a few kilobytes, otherwise we may generate a very large amount of data + numBytes <- Gen.choose(0.bytes.inBytes, 2.kilobytes.inBytes) + bytes <- Gen.containerOfN[Array, Byte](numBytes.toInt, Arbitrary.arbitrary[Byte]) + } yield Buf.ByteArray.Owned(bytes) + ) + + forAll(getByteArrays) { buffers: Seq[Buf] => + val buffersWithLength = buffers.map(buf => Buf.U32BE(buf.length).concat(buf)) + + val r = BufReader.framed(BufReader(Buf(buffersWithLength)), new BufReaderTest.U32BEFramer()) + + // read all of the frames + buffers.foreach { buf => + assert(await(r.read()).contains(buf)) + } + + // make sure the reader signals EOF + assert(await(r.read()).isEmpty) + } + } + + test("BufReader.framed reads empty frames") { + val r = BufReader.framed(BufReader(Buf.U32BE(0)), new BufReaderTest.U32BEFramer()) + assert(await(r.read()).contains(Buf.Empty)) + assert(await(r.read()).isEmpty) + } + + test("Framed works on non-list collections") { + val r = BufReader.framed(BufReader(Buf.U32BE(0)), i => Vector(i)) + assert(await(r.read()).isDefined) + } +} + +object BufReaderTest { + + /** + * Used to test BufReader.framed, extract fields in terms of + * frames, signified by a 32-bit BE value preceding + * each frame. + */ + private class U32BEFramer() extends (Buf => Seq[Buf]) { + var state: Buf = Buf.Empty + + @tailrec + private def loop(acc: Seq[Buf], buf: Buf): Seq[Buf] = { + buf match { + case Buf.U32BE(l, d) if d.length >= l => + loop(acc :+ d.slice(0, l), d.slice(l, d.length)) + case _ => + state = buf + acc + } + } + + def apply(buf: Buf): Seq[Buf] = synchronized { + loop(Seq.empty, state concat buf) } } } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 1093e82aef..e49b1c6201 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -10,9 +10,8 @@ import java.util.concurrent.atomic.AtomicBoolean import org.mockito.Mockito._ import org.scalacheck.{Arbitrary, Gen} import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.scalatest.{FunSuite, Matchers} -import scala.annotation.tailrec +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable.ArrayBuffer class ReaderTest @@ -117,27 +116,6 @@ class ReaderTest assert(cancelled) } - test("Reader.chunked") { - val stringAndChunk = for { - s <- Gen.alphaStr - i <- Gen.posNum[Int].suchThat(_ <= s.length) - } yield (s, i) - - forAll(stringAndChunk) { - case (s, i) => - val r = Reader.chunked(Reader.fromBuf(Buf.Utf8(s), 32), i) - - def readLoop(): Unit = await(r.read()) match { - case Some(b) => - assert(b.length <= i) - readLoop() - case None => () - } - - readLoop() - } - } - test("Reader.concat") { forAll { ss: List[String] => val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) @@ -388,41 +366,6 @@ class ReaderTest } } - test("Reader.framed reads framed data") { - val getByteArrays: Gen[Seq[Buf]] = Gen.listOf( - for { - // limit arrays to a few kilobytes, otherwise we may generate a very large amount of data - numBytes <- Gen.choose(0.bytes.inBytes, 2.kilobytes.inBytes) - bytes <- Gen.containerOfN[Array, Byte](numBytes.toInt, Arbitrary.arbitrary[Byte]) - } yield Buf.ByteArray.Owned(bytes) - ) - - forAll(getByteArrays) { buffers: Seq[Buf] => - val buffersWithLength = buffers.map(buf => Buf.U32BE(buf.length).concat(buf)) - - val r = Reader.framed(BufReader(Buf(buffersWithLength)), new ReaderTest.U32BEFramer()) - - // read all of the frames - buffers.foreach { buf => - assert(await(r.read()).contains(buf)) - } - - // make sure the reader signals EOF - assert(await(r.read()).isEmpty) - } - } - - test("Reader.framed reads empty frames") { - val r = Reader.framed(BufReader(Buf.U32BE(0)), new ReaderTest.U32BEFramer()) - assert(await(r.read()).contains(Buf.Empty)) - assert(await(r.read()).isEmpty) - } - - test("Framed works on non-list collections") { - val r = Reader.framed(BufReader(Buf.U32BE(0)), i => Vector(i)) - assert(await(r.read()).isDefined) - } - test("Reader.flatMap") { forAll { s: String => val reader1 = Reader.fromBuf(Buf.Utf8(s), 8) @@ -578,7 +521,7 @@ class ReaderTest assert(await(reader.onClose) == StreamTermination.Discarded) } - test("Reader.readAll") { + test("Reader.readAllItems") { val genBuf: Gen[Buf] = { for { // limit arrays to a few kilobytes, otherwise we may generate a very large amount of data @@ -597,30 +540,3 @@ class ReaderTest } } } - -object ReaderTest { - - /** - * Used to test Reader.framed, extract fields in terms of - * frames, signified by a 32-bit BE value preceding - * each frame. - */ - private class U32BEFramer() extends (Buf => Seq[Buf]) { - var state: Buf = Buf.Empty - - @tailrec - private def loop(acc: Seq[Buf], buf: Buf): Seq[Buf] = { - buf match { - case Buf.U32BE(l, d) if d.length >= l => - loop(acc :+ d.slice(0, l), d.slice(l, d.length)) - case _ => - state = buf - acc - } - } - - def apply(buf: Buf): Seq[Buf] = synchronized { - loop(Seq.empty, state concat buf) - } - } -} From 9e0bdeefc8a81d50f00397a991c9395bf7db9a01 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 31 Oct 2019 23:16:22 +0000 Subject: [PATCH 229/796] util-core: Make ChunkedFramer more efficient Problem: ChunkedFramer used to impletement Buf => Seq[Buf] by creating new ListBuffer and copy the value every interation. Solution: ListBuffer is mutable, we can append new items to it. JIRA Issues: CSL-9021 Differential Revision: https://phabricator.twitter.biz/D393451 --- .../main/scala/com/twitter/io/BufReader.scala | 22 +++++++++---------- .../scala/com/twitter/io/BufReaderTest.scala | 13 +++++++++++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index 223b5e982a..056de58975 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -56,19 +56,19 @@ object BufReader { private final class ChunkedFramer(chunkSize: Int) extends (Buf => Seq[Buf]) { require(chunkSize > 0, s"chunkSize should be > 0 but was $chunkSize") - @tailrec - private def loop(acc: ListBuffer[Buf], in: Buf): Seq[Buf] = { - if (in.length < chunkSize) (acc :+ in).toSeq - else { - loop( - acc :+ in.slice(0, chunkSize), - in.slice(chunkSize, in.length) - ) + def apply(input: Buf): Seq[Buf] = { + val acc = ListBuffer[Buf]() + + @tailrec + def loop(buf: Buf): Seq[Buf] = { + if (buf.length < chunkSize) (acc += buf).result() + else { + acc += buf.slice(0, chunkSize) + loop(buf.slice(chunkSize, buf.length)) + } } - } - def apply(in: Buf): Seq[Buf] = { - loop(ListBuffer(), in) + loop(input) } } diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index a533a8a08c..7529477022 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -69,6 +69,19 @@ class BufReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { i <- Gen.posNum[Int].suchThat(_ <= s.length) } yield (s, i) + forAll(stringAndChunk) { + case (s, i) => + val r = BufReader.chunked(Reader.fromBuf(Buf.Utf8(s), 32), i) + assert(await(BufReader.readAll(r)) == Buf.Utf8(s)) + } + } + + test("BufReader.chunked by the chunkSize") { + val stringAndChunk = for { + s <- Gen.alphaStr + i <- Gen.posNum[Int].suchThat(_ <= s.length) + } yield (s, i) + forAll(stringAndChunk) { case (s, i) => val r = BufReader.chunked(Reader.fromBuf(Buf.Utf8(s), 32), i) From 5119e65c5c947893b15a70b0af05b22b910fc3de Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Sat, 2 Nov 2019 22:51:30 +0000 Subject: [PATCH 230/796] util-stats: Remove deprecated stat0 and counter0 methods from StatsReciever Problem These stats receiver methods were deprecated since 2017. Solution Remove deprecated methods and move users over to supported APIs. JIRA Issues: CSL-4710 Differential Revision: https://phabricator.twitter.biz/D393063 --- CHANGELOG.rst | 2 + .../twitter/finagle/stats/StatsReceiver.scala | 37 +++---------------- .../stats/StatsReceiverCompilationTest.java | 4 +- 3 files changed, 9 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fcabd4ecc6..bd86f5b24a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,6 +35,8 @@ Breaking API Changes * util-core: Moved `c.t.io.Reader.chunked` to `c.t.io.BufReader.chunked`, and `Reader.framed` to `BufReader.framed`. ``PHAB_ID=D392198`` +* util-stats: Removed deprecated methods `stat0` and `counter0` from `StatsReceiver`. ``PHAB_ID=D393063`` + 19.10.0 ------- diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 5f43d6451b..6e08289e8f 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -31,14 +31,6 @@ object StatsReceiver { */ object StatsReceivers { - /** - * Java compatible version of [[StatsReceiver.counter]]. - */ - @varargs - @deprecated("Use vararg StatsReceiver.counter() instead", "2017-6-16") - def counter(statsReceiver: StatsReceiver, name: String*): Counter = - statsReceiver.counter(name: _*) - /** * Java compatible version of [[StatsReceiver.addGauge]]. */ @@ -55,13 +47,6 @@ object StatsReceivers { @varargs def provideGauge(statsReceiver: StatsReceiver, callable: Callable[JFloat], name: String*): Unit = statsReceiver.provideGauge(name: _*)(callable.call()) - - /** - * Java compatible version of [[StatsReceiver.stat]]. - */ - @varargs - @deprecated("Use vararg StatsReceiver.stat() instead", "2017-6-16") - def stat(statsReceiver: StatsReceiver, name: String*): Stat = statsReceiver.stat(name: _*) } /** @@ -125,15 +110,6 @@ trait StatsReceiver { private[stats] def counter(schema: CounterSchema): Counter = counter(schema.metricBuilder.verbosity, schema.metricBuilder.name: _*) - /** - * Get a [[Counter counter]] with the given `name`. - * - * This method is a convenience for Java programs, but is no longer needed because - * [[StatsReceivers.counter]] is usable from java. - */ - @deprecated("Use vararg counter() instead", "2017-6-16") - def counter0(name: String): Counter = counter(name) - /** * Get a [[Stat stat]] with the given name. */ @@ -156,14 +132,6 @@ trait StatsReceiver { private[stats] def stat(schema: HistogramSchema): Stat = stat(schema.metricBuilder.verbosity, schema.metricBuilder.name: _*) - /** - * Get a [[Stat stat]] with the given name. This method is a convenience for Java - * programs, but is no longer needed because [[StatsReceivers.counter]] is - * usable from java. - */ - @deprecated("Use vararg stat() instead", "2017-6-16") - def stat0(name: String): Stat = stat(name) - /** * Register a function `f` as a [[Gauge gauge]] with the given name that has * a lifecycle with no end. @@ -329,6 +297,11 @@ trait StatsReceiver { override def scope(namespace: String): StatsReceiver = self.scope(namespace).scope(suffix) } } + + // These two are needed to accommodate Zookeper's circular dependency. + // We'll remove them once zk is upgraded: CSL-4710. + private[stats] def counter0(name: String): Counter = counter(name) + private[stats] def stat0(name: String): Stat = stat(name) } abstract class AbstractStatsReceiver extends StatsReceiver { diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java index 91e83c0e1e..4a059be386 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java @@ -24,7 +24,7 @@ public void testNullStatsReceiver() { @Test public void testStatMethods() { InMemoryStatsReceiver sr = new InMemoryStatsReceiver(); - Stat stat = StatsReceivers.stat(sr, "hello", "world"); + Stat stat = sr.stat("hello", "world"); Callable callable = new Callable() { public Integer call() { return 3; @@ -61,7 +61,7 @@ public Float call() { public void testCounterMethods() { InMemoryStatsReceiver sr = new InMemoryStatsReceiver(); sr.isNull(); - Counter counter = StatsReceivers.counter(sr, "hello", "world"); + Counter counter = sr.counter("hello", "world"); counter.incr(); counter.incr(100); } From 844fe24d0e190baaa976971221c88d8b4b7f9ad5 Mon Sep 17 00:00:00 2001 From: David Rusek Date: Mon, 4 Nov 2019 17:26:18 +0000 Subject: [PATCH 231/796] util-core: mark 2.11 java compat classes as Deprecated Problem/Solution/Result The io Java compatibility helper classes will no longer be necessary once we drop support for Scala 2.11. In preparation we should deprecate them so users have time to plan migration. JIRA Issues: CSL-7915 Differential Revision: https://phabricator.twitter.biz/D393913 --- CHANGELOG.rst | 7 +++++++ util-core/src/main/java/com/twitter/io/BufReaders.java | 2 ++ util-core/src/main/java/com/twitter/io/Bufs.java | 3 +++ util-core/src/main/java/com/twitter/io/Readers.java | 2 ++ util-core/src/main/java/com/twitter/io/Writers.java | 2 ++ 5 files changed, 16 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bd86f5b24a..73170ec928 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,6 +35,13 @@ Breaking API Changes * util-core: Moved `c.t.io.Reader.chunked` to `c.t.io.BufReader.chunked`, and `Reader.framed` to `BufReader.framed`. ``PHAB_ID=D392198`` +Deprecations +~~~~~~~~~~~~ + +* util-core: Mark `c.t.io.BufReaders`, `c.t.io.Bufs`, `c.t.io.Readers`, and `c.t.io.Writers` as + Deprecated. These classes will no longer be needed, and will be removed, after 2.11 support is + dropped. ``PHAB_ID=D393913`` + * util-stats: Removed deprecated methods `stat0` and `counter0` from `StatsReceiver`. ``PHAB_ID=D393063`` 19.10.0 diff --git a/util-core/src/main/java/com/twitter/io/BufReaders.java b/util-core/src/main/java/com/twitter/io/BufReaders.java index a6c5b53ae6..04f9b6c8fe 100644 --- a/util-core/src/main/java/com/twitter/io/BufReaders.java +++ b/util-core/src/main/java/com/twitter/io/BufReaders.java @@ -7,7 +7,9 @@ * This will be removed when drop Scala 2.11 support. * * @see com.twitter.io.BufReader + * @deprecated This will no longer be necessary when 2.11 support is dropped. 2019-10-04 */ +@Deprecated() public final class BufReaders { private BufReaders() { diff --git a/util-core/src/main/java/com/twitter/io/Bufs.java b/util-core/src/main/java/com/twitter/io/Bufs.java index 12997d51f5..5ca30866eb 100644 --- a/util-core/src/main/java/com/twitter/io/Bufs.java +++ b/util-core/src/main/java/com/twitter/io/Bufs.java @@ -4,7 +4,10 @@ /** * A Java adaptation of the {@link com.twitter.io.Buf} companion object. + * + * @deprecated This will no longer be necessary when 2.11 support is dropped. 2019-10-04 */ +@Deprecated public final class Bufs { private Bufs() { } diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index e746e508eb..9ea6475ff7 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -18,7 +18,9 @@ * This will be removed when drop Scala 2.11 support. * * @see com.twitter.io.Reader + * @deprecated This will no longer be necessary when 2.11 support is dropped. 2019-10-04 */ +@Deprecated public final class Readers { private Readers() { throw new IllegalStateException(); } diff --git a/util-core/src/main/java/com/twitter/io/Writers.java b/util-core/src/main/java/com/twitter/io/Writers.java index 53bb484020..0a1895240d 100644 --- a/util-core/src/main/java/com/twitter/io/Writers.java +++ b/util-core/src/main/java/com/twitter/io/Writers.java @@ -7,7 +7,9 @@ * Java APIs for Writer. * * @see com.twitter.io.Writer + * @deprecated This will no longer be necessary when 2.11 support is dropped. 2019-10-04 */ +@Deprecated public final class Writers { private Writers() { throw new IllegalStateException(); } From f42a017c15a0218cecf6a32e5550ebd8d95e9906 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Mon, 4 Nov 2019 21:29:52 +0000 Subject: [PATCH 232/796] util-core: `BufReader` clean-ups Problem/Solution As a follow up on the work to unify `BufReader` and `SeqReader` with a new `IteratorReader`, move the `BufReader#iterator` to `BufReader#apply`, and clean up tests for old `BufReader`, and added tests for the 2 `apply` methods in `BufReader` object. JIRA Issues: CSL-7490 Differential Revision: https://phabricator.twitter.biz/D393969 --- .../main/scala/com/twitter/io/BufReader.scala | 30 +++++++-------- .../scala/com/twitter/io/BufReaderTest.scala | 37 ++++--------------- 2 files changed, 22 insertions(+), 45 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index 056de58975..75d513c6e9 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -19,26 +19,24 @@ object BufReader { * `chunkSize`. */ def apply(buf: Buf, chunkSize: Int): Reader[Buf] = { + // Returns an iterator that iterates over the given [[Buf]] by `chunkSize`. + def iterator(buf: Buf, chunkSize: Int): Iterator[Buf] = new Iterator[Buf] { + require(chunkSize > 0, s"Then chunkSize should be greater than 0, but received: $chunkSize") + private[this] var remainingBuf: Buf = buf + + def hasNext: Boolean = !remainingBuf.isEmpty + def next(): Buf = { + if (!hasNext) throw new NoSuchElementException("next() on empty iterator") + val nextChuck = remainingBuf.slice(0, chunkSize) + remainingBuf = remainingBuf.slice(chunkSize, Int.MaxValue) + nextChuck + } + } + if (buf.isEmpty) Reader.empty[Buf] else new IteratorReader[Buf](iterator(buf, chunkSize)) } - /** - * Returns an iterator that iterates over the given [[Buf]] by `chunkSize`. - */ - private[io] def iterator(buf: Buf, chunkSize: Int): Iterator[Buf] = new Iterator[Buf] { - require(chunkSize > 0, s"Then chunkSize should be greater than 0, but received: $chunkSize") - private[this] var remainingBuf: Buf = buf - - def hasNext: Boolean = !remainingBuf.isEmpty - def next(): Buf = { - if (!hasNext) throw new NoSuchElementException("next() on empty iterator") - val nextChuck = remainingBuf.slice(0, chunkSize) - remainingBuf = remainingBuf.slice(chunkSize, Int.MaxValue) - nextChuck - } - } - /** * Read the entire bytestream presented by `r`. */ diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index 7529477022..d149709d91 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -12,47 +12,25 @@ class BufReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) - test("BufReader") { + test("BufReader - apply(buf)") { forAll { bytes: String => val buf = Buf.Utf8(bytes) - val r = Reader.fromBuf(buf, 8) + val r = BufReader(buf) assert(await(BufReader.readAll(r)) == buf) } } - test("BufReader - discard") { + test("BufReader - apply(buf, chunkSize)") { val stringAndChunk = for { s <- Gen.alphaStr i <- Gen.posNum[Int].suchThat(_ <= s.length) } yield (s, i) forAll(stringAndChunk) { - case (bytes, n) => - val r = Reader.fromBuf(Buf.Utf8(bytes), n) - r.discard() - - assert( - bytes.length == 0 || - Await - .ready(r.read(), 5.seconds).poll - .exists(_.throwable.isInstanceOf[ReaderDiscardedException])) - } - } - - test("BufReader - iterator") { - val stringAndChunk = for { - s <- Gen.alphaStr - i <- Gen.posNum[Int].suchThat(_ <= s.length) - } yield (s, i) - - forAll(stringAndChunk) { - case (bytes, n) => - var result = Buf.Empty - val iterator = BufReader.iterator(Buf.Utf8(bytes), n) - while (iterator.hasNext) { - result = result.concat(iterator.next) - } - assert(Buf.Utf8(bytes) == result) + case (bytes, chunkSize) => + val buf = Buf.Utf8(bytes) + val r = BufReader(buf, chunkSize) + assert(await(BufReader.readAll(r)) == buf) } } @@ -86,6 +64,7 @@ class BufReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { case (s, i) => val r = BufReader.chunked(Reader.fromBuf(Buf.Utf8(s), 32), i) + @tailrec def readLoop(): Unit = await(r.read()) match { case Some(b) => assert(b.length <= i) From 5562ebf36a728bef09e6a9104c49c41526538fa2 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 4 Nov 2019 22:00:27 +0000 Subject: [PATCH 233/796] util-core: Pipe handles copying Problem/Solution: `copy` methods that copy from Readers to Writers should not be hosted under Reader. they are equally related to both parties. Moving them to `Pipe`. Result: Scala 2.12 supports Java calling companion object methods directly without many `$` and no users are using `copy`. I don't feel we should add Pipes.java for these two method temporarily until we drop 2.11 support. Having them still under Readers.java before removal. JIRA Issues: CSL-6756 Differential Revision: https://phabricator.twitter.biz/D393650 --- CHANGELOG.rst | 3 + .../src/main/java/com/twitter/io/Readers.java | 8 +-- .../scala/com/twitter/io/IteratorReader.scala | 2 +- .../src/main/scala/com/twitter/io/Pipe.scala | 42 +++++++++++++ .../main/scala/com/twitter/io/Reader.scala | 43 +------------ .../test/scala/com/twitter/io/PipeTest.scala | 60 ++++++++++++++++++- .../scala/com/twitter/io/ReaderTest.scala | 58 +----------------- 7 files changed, 111 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 73170ec928..2bd8ce12e7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,6 +35,9 @@ Breaking API Changes * util-core: Moved `c.t.io.Reader.chunked` to `c.t.io.BufReader.chunked`, and `Reader.framed` to `BufReader.framed`. ``PHAB_ID=D392198`` +* util-core: Moved `c.t.io.Reader.copy` to `c.t.io.Pipe.copy`, and `Reader.copyMany` to + `Pipe.copyMany`. ``PHAB_ID=D393650`` + Deprecations ~~~~~~~~~~~~ diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index 9ea6475ff7..a7e7653b29 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -44,17 +44,17 @@ public static Reader concat(AsyncStream> readers) { } /** - * See {@code com.twitter.io.Reader.copy}. + * See {@code com.twitter.io.Pipe.copy}. */ public static Future copy(Reader r, Writer w) { - return Reader$.MODULE$.copy(r, w); + return Pipe$.MODULE$.copy(r, w); } /** - * See {@code com.twitter.io.Reader.copyMany}. + * See {@code com.twitter.io.Pipe.copyMany}. */ public static Future copyMany(AsyncStream> readers, Writer w) { - return Reader$.MODULE$.copyMany(readers, w); + return Pipe$.MODULE$.copyMany(readers, w); } /** diff --git a/util-core/src/main/scala/com/twitter/io/IteratorReader.scala b/util-core/src/main/scala/com/twitter/io/IteratorReader.scala index b3c41be987..8acc4e7bf9 100644 --- a/util-core/src/main/scala/com/twitter/io/IteratorReader.scala +++ b/util-core/src/main/scala/com/twitter/io/IteratorReader.scala @@ -59,7 +59,7 @@ private[io] final class IteratorReader[A](it: Iterator[A]) extends Reader[A] { private object IteratorReader { /** - * Indicates reader state when the reader is created via SeqReader + * Indicates reader state when the reader is created via IteratorReader */ sealed trait State object State { diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index c0b915db6d..7ab1e2f6ac 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -1,5 +1,6 @@ package com.twitter.io +import com.twitter.concurrent.AsyncStream import com.twitter.util.{Future, Promise, Return, Throw, Time, Timer} /** @@ -297,4 +298,45 @@ object Pipe { /** Indicates the reader has seen the EOF. No more reads or writes are allowed. */ case object Closed extends State[Nothing] } + + /** + * Copy elements from many Readers to a Writer. Readers will be discarded if + * `copy` is cancelled (discarding the target). The Writer is unmanaged, the + * caller is responsible for finalization and error handling, e.g.: + * + * {{{ + * Pipe.copyMany(readers, writer) ensure writer.close() + * }}} + * @param readers An AsyncStream holds a stream of Reader[A] + */ + def copyMany[A](readers: AsyncStream[Reader[A]], target: Writer[A]): Future[Unit] = + readers.foreachF(copy(_, target)) + + /** + * Copy elements from a Reader to a Writer. The Reader will be discarded if + * `copy` is cancelled (discarding the writer). The Writer is unmanaged, the caller + * is responsible for finalization and error handling, e.g.: + * + * {{{ + * Pipe.copy(r, w, n) ensure w.close() + * }}} + */ + def copy[A](r: Reader[A], w: Writer[A]): Future[Unit] = { + def loop(): Future[Unit] = + r.read().flatMap { + case None => Future.Done + case Some(elem) => w.write(elem) before loop() + } + + w.onClose.respond { + case Return(StreamTermination.Discarded) => r.discard() + case _ => () + } + val p = new Promise[Unit] + // We have to do this because discarding the writer doesn't interrupt read + // operations, it only fails the next write operation. + loop().proxyTo(p) + p.setInterruptHandler { case _ => r.discard() } + p + } } diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 724b624eca..daf008faa0 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -312,7 +312,7 @@ object Reader { */ def concat[A](readers: AsyncStream[Reader[A]]): Reader[A] = { val target = new Pipe[A] - val copied = copyMany(readers, target).respond { + val copied = Pipe.copyMany(readers, target).respond { case Throw(exc) => target.fail(exc) case _ => target.close() } @@ -397,45 +397,4 @@ object Reader { } } } - - /** - * Copy elements from many Readers to a Writer. Readers will be discarded if - * `copy` is cancelled (discarding the target). The Writer is unmanaged, the - * caller is responsible for finalization and error handling, e.g.: - * - * {{{ - * Reader.copyMany(readers, writer) ensure writer.close() - * }}} - * @param readers An AsyncStream holds a stream of Reader[A] - */ - def copyMany[A](readers: AsyncStream[Reader[A]], target: Writer[A]): Future[Unit] = - readers.foreachF(Reader.copy(_, target)) - - /** - * Copy elements from a Reader to a Writer. The Reader will be discarded if - * `copy` is cancelled (discarding the writer). The Writer is unmanaged, the caller - * is responsible for finalization and error handling, e.g.: - * - * {{{ - * Reader.copy(r, w, n) ensure w.close() - * }}} - */ - def copy[A](r: Reader[A], w: Writer[A]): Future[Unit] = { - def loop(): Future[Unit] = - r.read().flatMap { - case None => Future.Done - case Some(elem) => w.write(elem) before loop() - } - - w.onClose.respond { - case Return(StreamTermination.Discarded) => r.discard() - case _ => () - } - val p = new Promise[Unit] - // We have to do this because discarding the writer doesn't interrupt read - // operations, it only fails the next write operation. - loop().proxyTo(p) - p.setInterruptHandler { case _ => r.discard() } - p - } } diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index 4420f00776..351acb6b0d 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -2,9 +2,11 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, MockTimer, Return, Time} +import java.io.ByteArrayOutputStream import org.scalatest.{FunSuite, Matchers} +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -class PipeTest extends FunSuite with Matchers { +class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyChecks { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) @@ -441,4 +443,60 @@ class PipeTest extends FunSuite with Matchers { } } + test("Pipe.copy - source and destination equality") { + forAll { (p: Array[Byte], q: Array[Byte], r: Array[Byte]) => + val rw = new Pipe[Buf] + val bos = new ByteArrayOutputStream + + val w = Writer.fromOutputStream(bos, 31) + val f = Pipe.copy(rw, w).ensure(w.close()) + val g = + rw.write(Buf.ByteArray.Owned(p)).before { + rw.write(Buf.ByteArray.Owned(q)).before { + rw.write(Buf.ByteArray.Owned(r)).before { + rw.close() + } + } + } + + await(Future.join(f, g)) + + val b = new ByteArrayOutputStream + b.write(p) + b.write(q) + b.write(r) + b.flush() + + bos.toByteArray should equal(b.toByteArray) + } + } + + test("Pipe.copy - discard the source if cancelled") { + var cancelled = false + val src = new Reader[Int] { + def read(): Future[Option[Int]] = Future.value(Some(1)) + def discard(): Unit = cancelled = true + def onClose: Future[StreamTermination] = Future.never + } + val dest = new Pipe[Int] + Pipe.copy(src, dest) + assert(await(dest.read()) == Some(1)) + dest.discard() + assert(cancelled) + } + + test("Pipe.copy - discard the source if interrupted") { + var cancelled = false + val src = new Reader[Int] { + def read(): Future[Option[Int]] = Future.value(Some(1)) + def discard(): Unit = cancelled = true + def onClose: Future[StreamTermination] = Future.never + } + val dest = new Pipe[Int] + val f = Pipe.copy(src, dest) + assert(await(dest.read()) == Some(1)) + f.raise(new Exception("Freeze!")) + assert(cancelled) + } + } diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index e49b1c6201..3568b68926 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -4,7 +4,7 @@ import com.twitter.concurrent.AsyncStream import com.twitter.conversions.DurationOps._ import com.twitter.conversions.StorageUnitOps._ import com.twitter.util.{Await, Awaitable, Future, Promise} -import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import java.io.ByteArrayInputStream import java.nio.charset.{StandardCharsets => JChar} import java.util.concurrent.atomic.AtomicBoolean import org.mockito.Mockito._ @@ -60,62 +60,6 @@ class ReaderTest case _ => to.close() } - test("Reader.copy - source and destination equality") { - forAll { (p: Array[Byte], q: Array[Byte], r: Array[Byte]) => - val rw = new Pipe[Buf] - val bos = new ByteArrayOutputStream - - val w = Writer.fromOutputStream(bos, 31) - val f = Reader.copy(rw, w).ensure(w.close()) - val g = - rw.write(Buf.ByteArray.Owned(p)).before { - rw.write(Buf.ByteArray.Owned(q)).before { - rw.write(Buf.ByteArray.Owned(r)).before { - rw.close() - } - } - } - - await(Future.join(f, g)) - - val b = new ByteArrayOutputStream - b.write(p) - b.write(q) - b.write(r) - b.flush() - - bos.toByteArray should equal(b.toByteArray) - } - } - - test("Reader.copy - discard the source if cancelled") { - var cancelled = false - val src = new Reader[Int] { - def read(): Future[Option[Int]] = Future.value(Some(1)) - def discard(): Unit = cancelled = true - def onClose: Future[StreamTermination] = Future.never - } - val dest = new Pipe[Int] - Reader.copy(src, dest) - assert(await(dest.read()) == Some(1)) - dest.discard() - assert(cancelled) - } - - test("Reader.copy - discard the source if interrupted") { - var cancelled = false - val src = new Reader[Int] { - def read(): Future[Option[Int]] = Future.value(Some(1)) - def discard(): Unit = cancelled = true - def onClose: Future[StreamTermination] = Future.never - } - val dest = new Pipe[Int] - val f = Reader.copy(src, dest) - assert(await(dest.read()) == Some(1)) - f.raise(new Exception("Freeze!")) - assert(cancelled) - } - test("Reader.concat") { forAll { ss: List[String] => val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) From 405f3882e6a40b8d7e0fd6808efa62ff2ec27970 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 5 Nov 2019 17:56:28 +0000 Subject: [PATCH 234/796] twitter-oss: Prepare OSS Libraries for Release 19.11.0 Problem We want to release the next versions of our Twitter OSS libraries 19.11.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9036 Differential Revision: https://phabricator.twitter.biz/D395427 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2bd8ce12e7..715dc6716c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.11.0 +------- + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 0bbbade258..2c71d58883 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "19.10.0" +val utilCore = "com.twitter" %% "util-core" % "19.11.0" ``` # Units diff --git a/build.sbt b/build.sbt index 9e38597c87..51e6294610 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.11.0-SNAPSHOT" +val releaseVersion = "19.11.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 700a9d12702dc37eaa17d0c740cb61143ee490b4 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 6 Nov 2019 18:07:51 +0000 Subject: [PATCH 235/796] twitter-oss: Update OSS Libraries Post-Release to 19.12.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 19.12.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9036 Differential Revision: https://phabricator.twitter.biz/D396071 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2c71d58883..58b901d6e8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 51e6294610..e5647f6773 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.11.0" +val releaseVersion = "19.12.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 9ce99a6e58b251dcdd2296a9ff854a215c3026ca Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 7 Nov 2019 19:22:43 +0000 Subject: [PATCH 236/796] util-core: Remove Pipe.copyMany Problem Pipe.copyMany is rarely used by our users, and copying from AsyncStream[Reader[A]] to Writer[A] isn't the way we want to advocate. Solution Remove it. JIRA Issues: CSL-9052 Differential Revision: https://phabricator.twitter.biz/D396590 --- CHANGELOG.rst | 3 +++ .../src/main/java/com/twitter/io/Readers.java | 7 ------- util-core/src/main/scala/com/twitter/io/Pipe.scala | 14 -------------- .../src/main/scala/com/twitter/io/Reader.scala | 2 +- 4 files changed, 4 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 715dc6716c..8f2e23f8d5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -31,6 +31,9 @@ Runtime Behavior Changes Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ +* util-core: Removed `c.t.io.Pipe.copyMany` (was `Reader.copyMany`). Use `AsyncStream.foreachF` + link to `Pipe.copy` for substitution. ``PHAB_ID=D396590`` + * util-core: Add `c.t.io.BufReader.readAll` to consume a `Reader[Buf]` and concat values to a Buf. Replace `c.t.io.Reader.readAll` with `Reader.readAllItems`, the new API consumes a generic Reader[T], and return a Seq of items. ``PHAB_ID=D391346`` diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index a7e7653b29..5b29bb3416 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -50,13 +50,6 @@ public static Future copy(Reader r, Writer w) { return Pipe$.MODULE$.copy(r, w); } - /** - * See {@code com.twitter.io.Pipe.copyMany}. - */ - public static Future copyMany(AsyncStream> readers, Writer w) { - return Pipe$.MODULE$.copyMany(readers, w); - } - /** * See {@code com.twitter.io.Reader.fromFile()}. */ diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 7ab1e2f6ac..e8c87202bd 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -1,6 +1,5 @@ package com.twitter.io -import com.twitter.concurrent.AsyncStream import com.twitter.util.{Future, Promise, Return, Throw, Time, Timer} /** @@ -299,19 +298,6 @@ object Pipe { case object Closed extends State[Nothing] } - /** - * Copy elements from many Readers to a Writer. Readers will be discarded if - * `copy` is cancelled (discarding the target). The Writer is unmanaged, the - * caller is responsible for finalization and error handling, e.g.: - * - * {{{ - * Pipe.copyMany(readers, writer) ensure writer.close() - * }}} - * @param readers An AsyncStream holds a stream of Reader[A] - */ - def copyMany[A](readers: AsyncStream[Reader[A]], target: Writer[A]): Future[Unit] = - readers.foreachF(copy(_, target)) - /** * Copy elements from a Reader to a Writer. The Reader will be discarded if * `copy` is cancelled (discarding the writer). The Writer is unmanaged, the caller diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index daf008faa0..0dc4ef524d 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -312,7 +312,7 @@ object Reader { */ def concat[A](readers: AsyncStream[Reader[A]]): Reader[A] = { val target = new Pipe[A] - val copied = Pipe.copyMany(readers, target).respond { + val copied = readers.foreachF(Pipe.copy(_, target)).respond { case Throw(exc) => target.fail(exc) case _ => target.close() } From 608abd8c49e88b530fe71d15bf4ef893edcb0604 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 8 Nov 2019 22:52:54 +0000 Subject: [PATCH 237/796] util-jvm: Stop exporting postGC stats twice Problem We've been exporting `postGC` stats under two different scopes (`jvm/postGC` and `jvm/mem/postGC`) for quite a while due to a historical accident. Solution Stop double-exporting `postGC` and move affected dashboards over. This change removes 4 gauges from each Twitter Server. JIRA Issues: CSL-5045 Differential Revision: https://phabricator.twitter.biz/D392230 --- CHANGELOG.rst | 3 +++ util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala | 7 ++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8f2e23f8d5..e92193663e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -28,6 +28,9 @@ Runtime Behavior Changes * util: Upgrade to caffeine 2.8.0 ``PHAB_ID=D384592`` +* util-jvm: Stop double-exporting `postGC` stats under both `jvm` and `jvm/mem`. These are now + only exported under `jvm/mem/postGC`. ``PHAB_ID=D392230`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index eeb2f35084..7b9715eecc 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -1,7 +1,7 @@ package com.twitter.jvm import com.twitter.conversions.StringOps._ -import com.twitter.finagle.stats.{BroadcastStatsReceiver, StatsReceiver} +import com.twitter.finagle.stats.StatsReceiver import java.lang.management.{ManagementFactory, BufferPoolMXBean} import scala.collection.JavaConverters._ import scala.collection.mutable @@ -75,10 +75,7 @@ object JvmStats { val memPool = ManagementFactory.getMemoryPoolMXBeans.asScala val memStats = stats.scope("mem") val currentMem = memStats.scope("current") - // TODO: Refactor postGCStats when we confirmed that no one is using this stats anymore - // val postGCStats = memStats.scope("postGC") - val postGCMem = memStats.scope("postGC") - val postGCStats = BroadcastStatsReceiver(Seq(stats.scope("postGC"), postGCMem)) + val postGCStats = memStats.scope("postGC") memPool.foreach { pool => val name = pool.getName.regexSub("""[^\w]""".r) { m => "_" From 19a6af44759e775c33106f95295954f6e6e0cd10 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 14 Nov 2019 18:27:11 +0000 Subject: [PATCH 238/796] util, finagle, twitter-server: Remove JavaSingleton Trait Problem / Solution The `JavaSingleton` trait in util is lightly used and doesn't provide functionality in the way that it's intended to work. Let's remove its single uses within Finagle and Twitter Server and encourage folks to take a different approach for better Java compatibility. JIRA Issues: CSL-9080 Differential Revision: https://phabricator.twitter.biz/D399947 --- CHANGELOG.rst | 7 +++++++ .../src/main/scala/com/twitter/util/JavaSingleton.scala | 8 -------- 2 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 util-core/src/main/scala/com/twitter/util/JavaSingleton.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e92193663e..3eb649e6b5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: The lightly used `com.twitter.util.JavaSingleton` trait has been removed. It + did not work as intended. Users should provide Java friendly objects, classes, and methods + instead. ``PHAB_ID=D399947`` + 19.11.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/JavaSingleton.scala b/util-core/src/main/scala/com/twitter/util/JavaSingleton.scala deleted file mode 100644 index af5c55b01b..0000000000 --- a/util-core/src/main/scala/com/twitter/util/JavaSingleton.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.util - -/** - * A mixin to allow scala objects to be used from java. - */ -trait JavaSingleton { - def get: JavaSingleton = this -} From 36d820712635a14d9545f2834341744e9932035d Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 15 Nov 2019 15:27:23 +0000 Subject: [PATCH 239/796] util: Add a tool to make metrics lazy Problem Sometimes we don't know in advance if we're going to be using certain metrics. However, we don't want to have to change our programming model to have to know in advance if things are lazy. Solution Provide a LazyStatsReceiver that will automatically make metrics lazy without having to rewrite where they're used. JIRA Issues: CSL-9070 Differential Revision: https://phabricator.twitter.biz/D398898 --- CHANGELOG.rst | 6 ++++ .../finagle/stats/LazyStatsReceiver.scala | 31 +++++++++++++++++++ .../finagle/stats/LazyStatsReceiverTest.scala | 23 ++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala create mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/LazyStatsReceiverTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3eb649e6b5..165d321d65 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-stats: Introduces `c.t.f.stats.LazyStatsReceiver` which ensures that counters and histograms + don't export metrics until after they have been `incr`ed or `add`ed at least once. ``PHAB_ID=D398898`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala new file mode 100644 index 0000000000..f9732263c1 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala @@ -0,0 +1,31 @@ +package com.twitter.finagle.stats + +/** + * Wraps an underlying [[StatsReceiver]] to ensure that derived counters and + * stats will not start exporting metrics until `incr` or `add` is first called + * on them. + * + * This should be used when integrating with tools that create metrics eagerly, + * but you don't know whether you're going to actually use those metrics or not. + * One example might be if you're speaking to a remote peer that exposes many + * endpoints, and you eagerly create metrics for all of those endpoints, but + * aren't going to use all of the different methods. + * + * We don't apply this very widely automatically--it can mess with caching, and + * adds an extra allocation when you construct a new counter or stat, so please + * be judicious when using it. + * + * @note does not change the way gauges are used, since there isn't a way of + * modeling whether a gauge is "used" or not. + */ +final class LazyStatsReceiver(val self: StatsReceiver) extends StatsReceiverProxy { + override def counter(verbosity: Verbosity, names: String*): Counter = new Counter { + private[this] lazy val underlying = self.counter(verbosity, names: _*) + def incr(delta: Long): Unit = underlying.incr(delta) + } + + override def stat(verbosity: Verbosity, names: String*): Stat = new Stat { + private[this] lazy val underlying = self.stat(verbosity, names: _*) + def add(value: Float): Unit = underlying.add(value) + } +} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/LazyStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/LazyStatsReceiverTest.scala new file mode 100644 index 0000000000..d59d89d84b --- /dev/null +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/LazyStatsReceiverTest.scala @@ -0,0 +1,23 @@ +package com.twitter.finagle.stats + +import org.scalatest.FunSuite + +class LazyStatsReceiverTest extends FunSuite { + test("Doesn't eagerly initialize counters") { + val underlying = new InMemoryStatsReceiver() + val wrapped = new LazyStatsReceiver(underlying) + val counter = wrapped.counter("foo") + assert(underlying.counters.isEmpty) + counter.incr() + assert(underlying.counters == Map(Seq("foo") -> 1)) + } + + test("Doesn't eagerly initialize histograms") { + val underlying = new InMemoryStatsReceiver() + val wrapped = new LazyStatsReceiver(underlying) + val histo = wrapped.stat("foo") + assert(underlying.stats.isEmpty) + histo.add(5) + assert(underlying.stats == Map(Seq("foo") -> Seq(5))) + } +} From 8989b183efdf8b7f10b363cbf969b74dc7a25df5 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Mon, 18 Nov 2019 18:55:18 +0000 Subject: [PATCH 240/796] util-logging: Split Level and HasLogLevel Into Own Files Problem / Solution util-logging's `Level` and `HasLogLevel` are useful outside of util-logging's `Logger` implementation. The `HasLogLevel` trait is mixed into Exceptions within Finagle to give an indication at what level an exception or failure should be logged at. These items will likely outlive `Logger` within Finagle, so it makes sense to split them off into their own entities. JIRA Issues: CSL-9091 Differential Revision: https://phabricator.twitter.biz/D400775 --- .../com/twitter/logging/HasLogLevel.scala | 36 ++++++++++ .../scala/com/twitter/logging/Level.scala | 36 ++++++++++ .../scala/com/twitter/logging/Logger.scala | 68 +------------------ 3 files changed, 73 insertions(+), 67 deletions(-) create mode 100644 util-logging/src/main/scala/com/twitter/logging/HasLogLevel.scala create mode 100644 util-logging/src/main/scala/com/twitter/logging/Level.scala diff --git a/util-logging/src/main/scala/com/twitter/logging/HasLogLevel.scala b/util-logging/src/main/scala/com/twitter/logging/HasLogLevel.scala new file mode 100644 index 0000000000..ace06737f9 --- /dev/null +++ b/util-logging/src/main/scala/com/twitter/logging/HasLogLevel.scala @@ -0,0 +1,36 @@ +package com.twitter.logging + +import scala.annotation.tailrec + +/** + * Typically mixed into `Exceptions` to indicate what [[Level]] + * they should be logged at. + * + * @see Finagle's `com.twitter.finagle.Failure`. + */ +trait HasLogLevel { + def logLevel: Level +} + +object HasLogLevel { + + /** + * Finds the first [[HasLogLevel]] for the given `Throwable` including + * its chain of causes and returns its `logLevel`. + * + * @note this finds the first [[HasLogLevel]], and should be there + * be multiple in the chain of causes, it '''will not use''' + * the most severe. + * + * @return `None` if neither `t` nor any of its causes are a + * [[HasLogLevel]]. Otherwise, returns `Some` of the first + * one found. + */ + @tailrec + def unapply(t: Throwable): Option[Level] = t match { + case null => None + case hll: HasLogLevel => Some(hll.logLevel) + case _ => unapply(t.getCause) + } + +} diff --git a/util-logging/src/main/scala/com/twitter/logging/Level.scala b/util-logging/src/main/scala/com/twitter/logging/Level.scala new file mode 100644 index 0000000000..8e7881ef9f --- /dev/null +++ b/util-logging/src/main/scala/com/twitter/logging/Level.scala @@ -0,0 +1,36 @@ +package com.twitter.logging + +import java.util.{logging => javalog} + +// replace java's ridiculous log levels with the standard ones. +sealed abstract class Level(val name: String, val value: Int) extends javalog.Level(name, value) { + // for java compat + def get(): Level = this +} + +object Level { + case object OFF extends Level("OFF", Int.MaxValue) + case object FATAL extends Level("FATAL", 1000) + case object CRITICAL extends Level("CRITICAL", 970) + case object ERROR extends Level("ERROR", 930) + case object WARNING extends Level("WARNING", 900) + case object INFO extends Level("INFO", 800) + case object DEBUG extends Level("DEBUG", 500) + case object TRACE extends Level("TRACE", 400) + case object ALL extends Level("ALL", Int.MinValue) + + private[logging] val AllLevels: Seq[Level] = + Seq(OFF, FATAL, CRITICAL, ERROR, WARNING, INFO, DEBUG, TRACE, ALL) + + /** + * Associate [[java.util.logging.Level]] and `Level` by their integer + * values. If there is no match, we return `None`. + */ + def fromJava(level: javalog.Level): Option[Level] = + AllLevels.find(_.value == level.intValue) + + /** + * Get a `Level` by its name, or `None` if there is no match. + */ + def parse(name: String): Option[Level] = AllLevels.find(_.name == name) +} diff --git a/util-logging/src/main/scala/com/twitter/logging/Logger.scala b/util-logging/src/main/scala/com/twitter/logging/Logger.scala index 4e703c235a..f8ff9ea224 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Logger.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Logger.scala @@ -18,76 +18,10 @@ package com.twitter.logging import java.util.concurrent.ConcurrentHashMap import java.util.{logging => javalog} -import scala.annotation.{tailrec, varargs} +import scala.annotation.varargs import scala.collection.JavaConverters._ import scala.collection.Map -// replace java's ridiculous log levels with the standard ones. -sealed abstract class Level(val name: String, val value: Int) extends javalog.Level(name, value) { - // for java compat - def get(): Level = this -} - -object Level { - case object OFF extends Level("OFF", Int.MaxValue) - case object FATAL extends Level("FATAL", 1000) - case object CRITICAL extends Level("CRITICAL", 970) - case object ERROR extends Level("ERROR", 930) - case object WARNING extends Level("WARNING", 900) - case object INFO extends Level("INFO", 800) - case object DEBUG extends Level("DEBUG", 500) - case object TRACE extends Level("TRACE", 400) - case object ALL extends Level("ALL", Int.MinValue) - - private[logging] val AllLevels: Seq[Level] = - Seq(OFF, FATAL, CRITICAL, ERROR, WARNING, INFO, DEBUG, TRACE, ALL) - - /** - * Associate [[java.util.logging.Level]] and `Level` by their integer - * values. If there is no match, we return `None`. - */ - def fromJava(level: javalog.Level): Option[Level] = - AllLevels.find(_.value == level.intValue) - - /** - * Get a `Level` by its name, or `None` if there is no match. - */ - def parse(name: String): Option[Level] = AllLevels.find(_.name == name) -} - -/** - * Typically mixed into `Exceptions` to indicate what [[Level]] - * they should be logged at. - * - * @see Finagle's `com.twitter.finagle.Failure`. - */ -trait HasLogLevel { - def logLevel: Level -} - -object HasLogLevel { - - /** - * Finds the first [[HasLogLevel]] for the given `Throwable` including - * its chain of causes and returns its `logLevel`. - * - * @note this finds the first [[HasLogLevel]], and should be there - * be multiple in the chain of causes, it '''will not use''' - * the most severe. - * - * @return `None` if neither `t` nor any of its causes are a - * [[HasLogLevel]]. Otherwise, returns `Some` of the first - * one found. - */ - @tailrec - def unapply(t: Throwable): Option[Level] = t match { - case null => None - case hll: HasLogLevel => Some(hll.logLevel) - case _ => unapply(t.getCause) - } - -} - class LoggingException(reason: String) extends Exception(reason) /** From 5b1b453f77c96b07358fd19c2f1b4ec91ff0c430 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 19 Nov 2019 17:12:24 +0000 Subject: [PATCH 241/796] util-security: Stop Testing Log Messages Problem / Solution Testing for specific log messages within util-security doesn't provide the return on value that having a concrete dependency on the legacy logging implementation requires. Let's remove the testing for specific log messages to ease the switch from the legacy util-logging to slf4j. JIRA Issues: CSL-9106 Differential Revision: https://phabricator.twitter.biz/D401991 --- .../scala/com/twitter/util/security/BUILD | 1 - .../twitter/util/security/PemFileTest.scala | 3 -- .../util/security/PemFileTestUtils.scala | 52 ------------------- .../Pkcs8EncodedKeySpecFileTest.scala | 3 -- .../security/X509CertificateFileTest.scala | 5 -- .../util/security/X509CrlFileTest.scala | 5 -- 6 files changed, 69 deletions(-) diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index a8ae77fa43..bd109b27ba 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -6,7 +6,6 @@ junit_tests( "3rdparty/jvm/org/scalatest", "util/util-core/src/main/scala/com/twitter/io", "util/util-core/src/main/scala/com/twitter/util", - "util/util-logging/src/main/scala/com/twitter/logging", "util/util-security/src/main/scala/com/twitter/util/security", "util/util-security/src/test/resources", ], diff --git a/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala index f6eb548bd0..cd0367060c 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala @@ -7,9 +7,6 @@ import org.scalatest.FunSuite class PemFileTest extends FunSuite { - private[this] val assertLogMessage = - PemFileTestUtils.assertLogMessage("PemFile") _ - private[this] def readPemFileMessage(messageType: String)(tempFile: File): Try[String] = { val pemFile = new PemFile(tempFile) pemFile.readMessage(messageType).map(new String(_)) diff --git a/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala b/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala index 21648e3653..e1dc7afb3c 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala @@ -1,6 +1,5 @@ package com.twitter.util.security -import com.twitter.logging.{BareFormatter, Logger, StringHandler} import com.twitter.util.Try import java.io.{File, IOException} import org.scalatest.Assertions._ @@ -8,21 +7,6 @@ import scala.reflect.ClassTag object PemFileTestUtils { - def newLogger(): Logger = { - val logger = Logger.get("com.twitter.util.security") - logger.clearHandlers() - logger.setLevel(Logger.WARNING) - logger.setUseParentHandlers(false) - logger - } - - def newHandler(): StringHandler = { - val logger = newLogger() - val handler = new StringHandler(BareFormatter, None) - logger.addHandler(handler) - handler - } - def assertException[A <: AnyRef, B](tryValue: Try[B])(implicit classTag: ClassTag[A]): Unit = { assert(tryValue.isThrow) intercept[A] { @@ -42,28 +26,7 @@ object PemFileTestUtils { def assertIOException[A](tryValue: Try[A]): Unit = assertException[IOException, A](tryValue) - def assertLogMessage( - name: String - )(logMessage: String, - filename: String, - expectedExMessage: String - ): Unit = { - assert(logMessage.startsWith(s"$name ($filename) failed to load: ")) - assert(logMessage.contains(expectedExMessage)) - } - - def assertAnyLogMessage( - name: String - )(logMessage: String, - filename: String, - expectedExMessages: String* - ): Unit = { - assert(logMessage.startsWith(s"$name ($filename) failed to load: ")) - assert(expectedExMessages.exists(expectedExMessage => logMessage.contains(expectedExMessage))) - } - def testFileDoesntExist[A](name: String, read: File => Try[A]): Unit = { - val handler = newHandler() val tempFile = File.createTempFile("test", "ext") tempFile.deleteOnExit() @@ -72,32 +35,20 @@ object PemFileTestUtils { assert(!tempFile.exists()) val tryReadPem: Try[A] = read(tempFile) - assertAnyLogMessage(name)( - handler.get, - tempFile.getName(), - "No such file or directory", - "The system cannot find the file specified") assertIOException(tryReadPem) } def testFilePathIsntFile[A](name: String, read: File => Try[A]): Unit = { - val handler = newHandler() val tempFile = File.createTempFile("test", "ext") tempFile.deleteOnExit() val parentFile = tempFile.getParentFile() val tryReadPem: Try[A] = read(parentFile) - assertAnyLogMessage(name)( - handler.get, - parentFile.getName(), - "Is a directory", - "Access is denied") assertIOException(tryReadPem) } def testFilePathIsntReadable[A](name: String, read: File => Try[A]): Unit = { - val handler = newHandler() val tempFile = File.createTempFile("test", "ext") tempFile.deleteOnExit() @@ -107,7 +58,6 @@ object PemFileTestUtils { val tryReadPem: Try[A] = read(tempFile) - assertLogMessage(name)(handler.get, tempFile.getName(), "Permission denied") assertIOException(tryReadPem) } @@ -117,13 +67,11 @@ object PemFileTestUtils { )( implicit classTag: ClassTag[A] ): Unit = { - val handler = newHandler() val tempFile = File.createTempFile("test", "ext") tempFile.deleteOnExit() val tryReadPem: Try[B] = read(tempFile) - assertLogMessage(name)(handler.get, tempFile.getName(), "Empty input") assertException[A, B](tryReadPem) } diff --git a/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala index 35bb97044e..fd60f05281 100644 --- a/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala @@ -8,9 +8,6 @@ import org.scalatest.FunSuite class Pkcs8EncodedKeySpecFileTest extends FunSuite { - private[this] val assertLogMessage = - PemFileTestUtils.assertLogMessage("PKCS8EncodedKeySpec") _ - private[this] val readKeySpecFromFile: File => Try[PKCS8EncodedKeySpec] = (tempFile) => { val keyFile = new Pkcs8EncodedKeySpecFile(tempFile) diff --git a/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala index d5e0a2849d..1aa3845a1d 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala @@ -13,9 +13,6 @@ class X509CertificateFileTest extends FunSuite { assert(subjectData.contains("Core Systems Libraries")) } - private[this] val assertLogMessage = - PemFileTestUtils.assertLogMessage("X509Certificate") _ - private[this] def assertCertException(tryCert: Try[X509Certificate]): Unit = PemFileTestUtils.assertException[CertificateException, X509Certificate](tryCert) @@ -45,7 +42,6 @@ class X509CertificateFileTest extends FunSuite { } test("File is garbage") { - val handler = PemFileTestUtils.newHandler() // Lines were manually deleted from a real certificate file val tempFile = TempFile.fromResourcePath("/certs/test-rsa-garbage.crt") // deleteOnExit is handled by TempFile @@ -53,7 +49,6 @@ class X509CertificateFileTest extends FunSuite { val certFile = new X509CertificateFile(tempFile) val tryCert = certFile.readX509Certificate() - assertLogMessage(handler.get, tempFile.getName, "Incomplete BER/DER data.") assertCertException(tryCert) } diff --git a/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala index 907bac34bf..251eb746b5 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala @@ -8,9 +8,6 @@ import org.scalatest.FunSuite class X509CrlFileTest extends FunSuite { - private[this] val assertLogMessage = - PemFileTestUtils.assertLogMessage("X509Crl") _ - private[this] def assertCrlException(tryCrl: Try[X509CRL]): Unit = PemFileTestUtils.assertException[CRLException, X509CRL](tryCrl) @@ -36,7 +33,6 @@ class X509CrlFileTest extends FunSuite { } test("File is garbage") { - val handler = PemFileTestUtils.newHandler() // Lines were manually deleted from a real crl file val tempFile = TempFile.fromResourcePath("/crl/csl-intermediate-garbage.crl") // deleteOnExit is handled by TempFile @@ -44,7 +40,6 @@ class X509CrlFileTest extends FunSuite { val crlFile = new X509CrlFile(tempFile) val tryCrl = crlFile.readX509Crl() - assertLogMessage(handler.get, tempFile.getName, "Incomplete BER/DER data.") assertCrlException(tryCrl) } From fa4fe43048d8b9d167f3152f4f1c44e8a0b539f7 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Wed, 20 Nov 2019 19:01:08 +0000 Subject: [PATCH 242/796] util: Add User Doc for Reader Problem We don't have a user doc for Reader Solution Created a user doc, this is mainly copied from Reader scala doc JIRA Issues: CSL-8590 Differential Revision: https://phabricator.twitter.biz/D357703 --- doc/src/sphinx/util-cookbook/index.rst | 3 +- doc/src/sphinx/util-cookbook/reader.rst | 215 ++++++++++++++++++++++++ 2 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 doc/src/sphinx/util-cookbook/reader.rst diff --git a/doc/src/sphinx/util-cookbook/index.rst b/doc/src/sphinx/util-cookbook/index.rst index 11f8334263..45d45fabf6 100644 --- a/doc/src/sphinx/util-cookbook/index.rst +++ b/doc/src/sphinx/util-cookbook/index.rst @@ -14,9 +14,10 @@ This is a cookbook with recipes on how to accomplish many common tasks. basics futures io + reader java Contact ------- -For questions email the `finaglers@twitter.com `_ group. \ No newline at end of file +For questions email the `finaglers@googlegroups.com `_ group. diff --git a/doc/src/sphinx/util-cookbook/reader.rst b/doc/src/sphinx/util-cookbook/reader.rst new file mode 100644 index 0000000000..bf8490598b --- /dev/null +++ b/doc/src/sphinx/util-cookbook/reader.rst @@ -0,0 +1,215 @@ +Reader +====== + +A reader exposes a pull-based API to model a potentially infinite stream of elements. Similar to use an +iterator to iterate through all elements from the stream, users are able to read one element at a time, +all elements are read in the order as they are fed to the reader. + +Interface +--------- + +The most frequently used APIs are: + +.. code-block:: scala + + trait Reader { + def read(): Future[Option[A]] + + def discard(): Unit + + def onClose: Future[StreamTermination] + + final def map[B](f: A => B): Reader[B] + + final def flatMap[B](f: A => Reader[B]): Reader[B] + + def flatten[B](implicit ev: A <:< Reader[B]): Reader[B] + } + +You could find a complete list of APIs from our `Reader Scaladoc `_. +The APIs give you abilities to read from the reader, discard the reader after you are done reading or +you don't want others to read from it. You can use flatMap and map on a reader in a similar way you +use them on `Future` and return a new reader after applying all transforming logic you need. Use +`onClose` to check if the stream is terminated (read until the end of stream, discarded, or encountered +an exception). + +read() +^^^^^^ + +Read the next element of this stream. The returned `Future` will resolve into `Some(e)` when the +element is available, or into `None` when the stream is fully-consumed. Reading from a discarded +reader will always return `ReaderDiscardedException`, reading from a failed reader will always resolve +in the failed Future. + +Multiple outstanding reads are not allowed on the same reader, please avoid calling `read()` again until +the future that the previous read returned has been satisfied. + +Duplicate reads are guaranteed not to happen. When properly using the Reader, it's impossible for a +subsequent read to be satisfied before the first read, since the second read will not be issued until +after the first read has been satisfied. + +discard() +^^^^^^^^^ + +Always discard the reader when its output is no longer required to avoid resource leaks. Reading from a +discarded reader will always resolve in `ReaderDiscardedException`. + +Discarding a fully-consumed reader or a failed reader won't override its stream termination to `Discarded`. +A fully-consumed reader remains at `FullyRead` and a failed reader remains at the failed future that +terminates the stream. + +onClose +^^^^^^^ + +A `Future` that resolves once this reader is fully-consumed (will resolves in `FullyRead`), discarded +(will resolve in `Discarded`), or failed (will resolve in the failed future that terminates the stream). +Once `onClose` is resolved in one stream termination, it won't be overridden to another one. + +This is useful for any extra resource cleanup that you must do once the stream is no longer being used, like +measuring the stream lifetime, counting the number of alive streams, counting the number of failed streams, +calculating stream success rate, etc. + +map +^^^ + +Construct a new Reader by lazily applying `f` to every item read from this Reader, where `f` is the function +that transforms data of type A to B. Note this operation only happens everytime a `read()` is called on the +new Reader. + +All operations of the new Reader will be in sync with self Reader. Discarding one Reader will discard +the other Reader. When one Reader's onClose resolves, the other Reader's onClose will be resolved +immediately with the same value. + +flatMap +^^^^^^^ + +Construct a new Reader (aka flat reader in the rest of the context) by lazily applying `f` to every item +read from self Reader (aka outer reader in the rest of the context), in the order as they are fed to the outer reader, +where `f` is the function that transforms data of type A to B and constructs a Reader[B] (aka inner reader in the rest +of the context). Note this operation only happens when we need to create a new inner reader(explained in the next section). + +When a `read()` operation is called on the flat reader, we will read from the latest constructed inner reader, and +return the value. If the current inner reader is fully-consumed (reading from it returns `None`), instead of +returning `None`, we will apply `f` to create the next inner reader and return the value read from the new inner reader. +If all inner readers are fully-consumed, the flat reader will be fully-consumed. If the current inner reader is failed, +reading from the flat reader will always resolve in the same exception as reading from the failed inner reader, and no new +inner reader will be created. If the current inner reader is discarded, reading from the flat reader will always resolve in +`ReaderDiscardedException` and no new inner readers will be created. + +All operations of the flat reader will be in sync with the outer reader. Discarding one Reader will discard the other Reader. +When one Reader's onClose resolves, the other Reader's onClose will be resolved immediately with the same value. + +However, after creating a flat reader via `flatMap`, the flat reader should be the only channel to interact with the stream, +please avoid manipulating the inner reader or the outer reader by reading from it, discarding it, or failing it. This will +affect the bahivor of the flat reader. + + +flatten +^^^^^^^ + +Converts a `Reader[Reader[B]]` into a `Reader[B]`, this is a convenient abstraction to read from a +stream (Reader, aka outer reader in the rest of the context) of Readers (aka inner reader in the rest of the context) as if +it were a single Reader (aka flat reader in the rest of the context). The behavior to read from the flat reader +is similar to use 2 iterators, with one outer iterator to traverse all inner readers from the stream, another +inner iterator to traverse all elements in the inner reader, and move the outer iterator after the inner iterator +finishes traversing an inner reader (the inner reader is fully-consumed). + +When a `read()` operation is called on the flat reader, we will read from the latest constructed inner reader, and +return the value. If the current inner reader is fully-consumed (reading from it returns `None`), instead of +returning `None`, we will read from the next inner reader and return the value read from the new inner reader. +If all inner readers are fully-consumed, the flat reader will be fully-consumed. If the current inner reader is failed, +reading from the flat reader will always resolve in the same exception as reading from the failed inner reader, and no new +inner reader will be created. If the current inner reader is discarded, reading from the flat reader will always resolve in +`ReaderDiscardedException` and no new inner readers will be created. + +All operations of the flat reader will be in sync with the outer reader. Discarding one Reader will discard +the other Reader. When one Reader's onClose resolves, the other Reader's onClose will be resolved immediately +with the same value. + +However, after creating a flat reader via `flatten`, the flat reader should be the only channel to interact with the stream, +please avoid manipulating the inner readers or the outer reader by reading from it, discarding it, or failing it. This will +affect the bahivor of the flat reader. + +Example Usages +-------------- + +Basic operations: read, discard, onClose +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: scala + + import com.twitter.io._ + import com.twitter.util._ + + val r1 = Reader.value(1) // Future(Some(1)) + r1.read() // Future(None) + r1.read() // Future(StreamTermination.FullyRead) + r1.onClose // Future(None) + r1.read() + + val r2 = Reader.value(2) + r2.discard() // Future(StreamTermination.Discarded) + r2.onClose // Future(ReaderDiscardedException) + r2.read() + +Construct a read-loop to consume a reader +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Given the pull-based API, the consumer is responsible for driving the computation. A very typical +code pattern to consume a reader is to use a read-loop: + +.. code-block:: scala + + def consume[A](r: Reader[A]))(process: A => Future[Unit]): Future[Unit] = { + r.read().flatMap { + case Some(a) => process(a).before(consume(r)(process)) + case None => Future.Done // reached the end of the stream; no need to discard + } + } + +The pattern above leverages `Future` recursion to exert back-pressure by allowing only one +outstanding read. Read must not be issued until the previous read finishes. This would always +ensure a finer grained back-pressure in network systems allowing consumers to artificially slow down +the producers and not rely solely on transport and IO buffering. + +.. NOTE:: + One way to reason about the read-loop idiom is to view it as a subscription to a publisher (reader) + in the Pub/Sub terminology. Perhaps the major difference between readers and traditional publishers, + is readers only allow one subscriber (read-loop). It's generally safer to assume the reader is fully + consumed (stream is exhausted) once its read-loop is run. + +Error handling +^^^^^^^^^^^^^^ + +Given the read-loop above, its returned `Future` could be used to observe both successful and +unsuccessful outcomes. + +.. code-block:: scala + + consume(stream)(processor).respond { + case Return(()) => println("Consumed an entire stream successfully.") + case Throw(e) => println(s"Encountered an error while consuming a stream: $e") + } + +If an exception is thrown during read, the reader's onClose will be resolved with the same exception, so +alternatively, you could handle a stream error like below: + +.. code-block:: scala + + val reader = Reader.value(1) + reader.onClose.respond { + case Return(()) => println("Consumed an entire stream successfully.") + case Throw(e) => println(s"Encountered an error while consuming a stream: $e") + } + +.. NOTE:: + Once failed, a stream can not be restarted such that all future reads will resolve into a failure. + There is no need to discard an already failed stream. + +Resource Safety +^^^^^^^^^^^^^^^ + +One of the important implications of readers, and streams in general, is that they are prone to resource +leaks unless fully consumed，discarded, or failed. Specifically, readers backed with a resource, like a +network connection, **MUST** be discarded unless already consumed (EOF observed), or failed to prevent +connection leaks. From 7c9cf7f80d19dd78501031b3c86ea5296ff17ff9 Mon Sep 17 00:00:00 2001 From: Todd Segal Date: Wed, 20 Nov 2019 19:34:00 +0000 Subject: [PATCH 243/796] util-core: Add size to Memoize.Snappable interface Problem: The Memoize.Snapable interface has a 'snap' member that creates a new map. For users just wanting to get the size of the current table, creating a new map is unnecessary. Solution: Add a .size member to the interface. Store this in a simple variable and increment it whenever something new is added to the memo table. Differential Revision: https://phabricator.twitter.biz/D402887 --- .../src/main/scala/com/twitter/util/Memoize.scala | 8 ++++++++ .../src/test/scala/com/twitter/util/MemoizeTest.scala | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/util-core/src/main/scala/com/twitter/util/Memoize.scala b/util-core/src/main/scala/com/twitter/util/Memoize.scala index 66d7b49d48..cb85640dc4 100644 --- a/util-core/src/main/scala/com/twitter/util/Memoize.scala +++ b/util-core/src/main/scala/com/twitter/util/Memoize.scala @@ -36,6 +36,12 @@ object Memoize { * [[scala.collection.immutable.Map]] */ def snap: Map[A, B] + + /** + * Returns the most recent size of the currently memoized computations. + * Useful for stats reporting. + */ + def size: Long } /** @@ -79,6 +85,8 @@ object Memoize { new Snappable[A, B] { private[this] var memo = Map.empty[A, Either[GuardedCountDownLatch, B]] + override def size: Long = memo.size + def snap: Map[A, B] = synchronized(memo) collect { case (a, Right(b)) => (a, b) diff --git a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala index ff8d883389..c13ab58134 100644 --- a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala @@ -136,6 +136,7 @@ class MemoizeTest extends FunSuite { assert(2 == memoizer(1)) assert(3 == memoizer(2)) assertResult(Map(1 -> 2, 2 -> 3))(memoizer.snap) + assert(memoizer.size == 2) } test("Memoize.snappable: snap ignores in-process computations") { @@ -161,7 +162,10 @@ class MemoizeTest extends FunSuite { callReadyLatch.countDown() assert(3 == Await.result(result)) - assertResult(Map(1 -> 2, 2 -> 3))(memoizer.snap) + val snap = memoizer.snap + assertResult(Map(1 -> 2, 2 -> 3))(snap) + assert(memoizer.size == snap.size) + assert(snap.size == 2) } test("Memoize.snappable: snap ignores failed computations") { @@ -175,8 +179,12 @@ class MemoizeTest extends FunSuite { memoizer(2) } assert(memoizer.snap.isEmpty) + assert(memoizer.size == memoizer.snap.size) + assert(memoizer.snap.size == 0) assert(2 == memoizer(1)) assertResult(Map(1 -> 2))(memoizer.snap) + assert(memoizer.size == memoizer.snap.size) + assert(memoizer.snap.size == 1) } } From 0dc6641cba19c87a1d8c3e888f3d5187310650e2 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 21 Nov 2019 00:13:15 +0000 Subject: [PATCH 244/796] util-test: Deprecate TestLogging Problem / Solution util-test's `TestLogging` is a mixin for testing java.util.logging or util-logging-based functionality in a `WordSpec` test. `WordSpec` has not been Twitter's recommended testing approach for many years, and util-logging will eventually be replaced in util and finagle by slf4j. Let's deprecate and (hopefully) eventually remove `TestLogging`. Differential Revision: https://phabricator.twitter.biz/D403574 --- CHANGELOG.rst | 7 +++++++ .../src/main/scala/com/twitter/logging/TestLogging.scala | 1 + 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 165d321d65..c64edbb512 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,13 @@ Breaking API Changes did not work as intended. Users should provide Java friendly objects, classes, and methods instead. ``PHAB_ID=D399947`` +Deprecations +~~~~~~~~~~~~ + +* util-test: The `c.t.logging.TestLogging` mixin has been deprecated. Users are encouraged to + move to slf4j for logging and minimize dependencies on `com.twitter.logging` in general, as + it is intended to be replaced entirely by slf4j. ``PHAB_ID=D403574`` + 19.11.0 ------- diff --git a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala index c1f3a21f42..80a3affb58 100644 --- a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala +++ b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala @@ -23,6 +23,7 @@ import org.scalatest.{BeforeAndAfter, WordSpec} /** * Specify logging during unit tests via system property, defaulting to FATAL only. */ +@deprecated("Prefer using slf4j for logging", "2019-11-20") trait TestLogging extends BeforeAndAfter { self: WordSpec => val logLevel: Level = Logger.levelNames(Option[String](System.getenv("log")).getOrElse("FATAL").toUpperCase) From 43321f81c805d390e9c6f8c2350fb60807aa02e3 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 23 Nov 2019 00:51:22 +0000 Subject: [PATCH 245/796] Add ibennett to some OWNERS Problem/Solution ibennett/ibennett Differential Revision: https://phabricator.twitter.biz/D405285 --- util-app/OWNERS | 1 + util-intellij/OWNERS | 1 + util-logging/OWNERS | 1 + util-slf4j-api/OWNERS | 1 + util-slf4j-jul-bridge/OWNERS | 1 + 5 files changed, 5 insertions(+) diff --git a/util-app/OWNERS b/util-app/OWNERS index 05d43901dd..ddd4683efe 100644 --- a/util-app/OWNERS +++ b/util-app/OWNERS @@ -1,5 +1,6 @@ banderson ccoco +ibennett koliver mnakamura roanta diff --git a/util-intellij/OWNERS b/util-intellij/OWNERS index 05d43901dd..ddd4683efe 100644 --- a/util-intellij/OWNERS +++ b/util-intellij/OWNERS @@ -1,5 +1,6 @@ banderson ccoco +ibennett koliver mnakamura roanta diff --git a/util-logging/OWNERS b/util-logging/OWNERS index 05d43901dd..ddd4683efe 100644 --- a/util-logging/OWNERS +++ b/util-logging/OWNERS @@ -1,5 +1,6 @@ banderson ccoco +ibennett koliver mnakamura roanta diff --git a/util-slf4j-api/OWNERS b/util-slf4j-api/OWNERS index dc165413a2..995bde0b3c 100644 --- a/util-slf4j-api/OWNERS +++ b/util-slf4j-api/OWNERS @@ -1,5 +1,6 @@ banderson ccoco +ibennett koliver mnakamura roanta diff --git a/util-slf4j-jul-bridge/OWNERS b/util-slf4j-jul-bridge/OWNERS index dc165413a2..995bde0b3c 100644 --- a/util-slf4j-jul-bridge/OWNERS +++ b/util-slf4j-jul-bridge/OWNERS @@ -1,5 +1,6 @@ banderson ccoco +ibennett koliver mnakamura roanta From a42b56d8c70c7eca68f1ba06619d538720abc5b6 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 25 Nov 2019 20:42:56 +0000 Subject: [PATCH 246/796] Jackson: Update to 2.10.1 Problem/Solution Update Jackson to version `2.10.1`. The `ScalaObjectMapper` has been promoted out of `experimental` thus it's package has changed from `com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper` to `com.fasterxml.jackson.module.scala.ScalaObjectMapper`. The JsonNode API has also been updated and in some cases requires a specified type. More details in the Jackson release notes which helps to explain some of the changes herein: https://github.com/FasterXML/jackson/wiki/Jackson-Release-2.10 JIRA Issues: CSL-9115 Differential Revision: https://phabricator.twitter.biz/D402722 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c64edbb512..afafcaaf6f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,6 +27,11 @@ Deprecations move to slf4j for logging and minimize dependencies on `com.twitter.logging` in general, as it is intended to be replaced entirely by slf4j. ``PHAB_ID=D403574`` +Changes +~~~~~~~ + + * util: Upgrade Jackson dependency from version `2.9.9` to version `2.10.1` ``PHAB_ID=D402722`` + 19.11.0 ------- diff --git a/build.sbt b/build.sbt index e5647f6773..1b09099731 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll( ExclusionRule("javax.jms", "jms") ) val slf4jVersion = "1.7.21" -val jacksonVersion = "2.9.9" +val jacksonVersion = "2.10.1" val guavaLib = "com.google.guava" % "guava" % "19.0" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" From 3705237c655d9a152abd6c9de5ff0694bad84dc8 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Mon, 2 Dec 2019 16:11:24 +0000 Subject: [PATCH 247/796] util-stats: implement MetricSchema functions for each StatsReceiver Problem The default implementation chosen for Metric Schema functions is inadequate for most StatsReceivers, as it does not maintain the special behaviors for which those StatsReceivers were created (ex, NameTranslatingStatsReceiver was not translating names). Solution Make the MetricSchema functions abstract in the base StatsReceiver class, thereby requiring each StatsReceiver to have its own well thought out MetricSchema functions. Result StatsReceivers have reasonable implementations of the MetricSchema functions JIRA Issues: CSL-8955 Differential Revision: https://phabricator.twitter.biz/D385068 --- CHANGELOG.rst | 6 ++ .../stats/BroadcastStatsReceiver.scala | 30 +++++----- .../finagle/stats/CumulativeGauge.scala | 6 +- .../finagle/stats/DenylistStatsReceiver.scala | 12 ++-- .../finagle/stats/InMemoryStatsReceiver.scala | 45 +++++++-------- .../finagle/stats/LazyStatsReceiver.scala | 8 +-- .../twitter/finagle/stats/MetricBuilder.scala | 38 +++++++++++++ .../stats/NameTranslatingStatsReceiver.scala | 21 ++++--- .../finagle/stats/NullStatsReceiver.scala | 9 ++- .../finagle/stats/RollupStatsReceiver.scala | 17 +++--- .../twitter/finagle/stats/StatsReceiver.scala | 56 ++++++++----------- .../finagle/stats/StatsReceiverProxy.scala | 7 +-- .../VerbosityAdjustingStatsReceiver.scala | 17 +++--- 13 files changed, 158 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index afafcaaf6f..e0fbcc4566 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -59,6 +59,12 @@ Runtime Behavior Changes Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ +* util-stats: abstract methods of StatsReceiver now take Schemas. The old APIs + are now final and cannot be overriden. For custom implementations, define + schema based methods (eg, counter(verbosity, name) is now + counter(CounterSchema)). NB: users can continue to call the old interface; + only implementors must migrate.``PHAB_ID=D385068`` + * util-core: Removed `c.t.io.Pipe.copyMany` (was `Reader.copyMany`). Use `AsyncStream.foreachF` link to `Pipe.copy` for substitution. ``PHAB_ID=D396590`` diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala index 29aa0c8e9b..fd5ba7043e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala @@ -17,17 +17,19 @@ object BroadcastStatsReceiver { with DelegatingStatsReceiver { val repr: AnyRef = this - def counter(verbosity: Verbosity, names: String*): Counter = new BroadcastCounter.Two( - first.counter(verbosity, names: _*), - second.counter(verbosity, names: _*) + def counter(schema: CounterSchema) = new BroadcastCounter.Two( + first.counter(schema), + second.counter(schema) ) - def stat(verbosity: Verbosity, names: String*): Stat = - new BroadcastStat.Two(first.stat(verbosity, names: _*), second.stat(verbosity, names: _*)) + def stat(schema: HistogramSchema) = new BroadcastStat.Two( + first.stat(schema), + second.stat(schema) + ) - def addGauge(verbosity: Verbosity, names: String*)(f: => Float): Gauge = new Gauge { - val firstGauge = first.addGauge(verbosity, names: _*)(f) - val secondGauge = second.addGauge(verbosity, names: _*)(f) + def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { + val firstGauge = first.addGauge(schema)(f) + val secondGauge = second.addGauge(schema)(f) def remove(): Unit = { firstGauge.remove() secondGauge.remove() @@ -43,14 +45,14 @@ object BroadcastStatsReceiver { private class N(srs: Seq[StatsReceiver]) extends StatsReceiver with DelegatingStatsReceiver { val repr: AnyRef = this - def counter(verbosity: Verbosity, names: String*): Counter = - BroadcastCounter(srs.map { _.counter(verbosity, names: _*) }) + def counter(schema: CounterSchema) = + BroadcastCounter(srs.map { _.counter(schema) }) - def stat(verbosity: Verbosity, names: String*): Stat = - BroadcastStat(srs.map { _.stat(verbosity, names: _*) }) + def stat(schema: HistogramSchema) = + BroadcastStat(srs.map { _.stat(schema) }) - def addGauge(verbosity: Verbosity, names: String*)(f: => Float): Gauge = new Gauge { - val gauges = srs.map { _.addGauge(verbosity, names: _*)(f) } + def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { + val gauges = srs.map { _.addGauge(schema)(f) } def remove(): Unit = gauges.foreach { _.remove() } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index 5accc7db61..aa981fba76 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -173,10 +173,12 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => } } - def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = { + def addGauge(schema: GaugeSchema)(f: => Float): Gauge = { var gauge: Gauge = null while (gauge == null) { - val cumulativeGauge = gauges.computeIfAbsent(name, getWhenNotPresent(verbosity)) + val cumulativeGauge = gauges.computeIfAbsent( + schema.metricBuilder.name, + getWhenNotPresent(schema.metricBuilder.verbosity)) gauge = cumulativeGauge.addGauge(f) } gauge diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala index a158bcca4c..a3e23164a5 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala @@ -12,14 +12,14 @@ package com.twitter.finagle.stats class DenylistStatsReceiver(protected val self: StatsReceiver, denylisted: Seq[String] => Boolean) extends StatsReceiverProxy { - override def counter(verbosity: Verbosity, name: String*): Counter = - getStatsReceiver(name).counter(verbosity, name: _*) + override def counter(schema: CounterSchema) = + getStatsReceiver(schema.metricBuilder.name).counter(schema) - override def stat(verbosity: Verbosity, name: String*): Stat = - getStatsReceiver(name).stat(verbosity, name: _*) + override def stat(schema: HistogramSchema) = + getStatsReceiver(schema.metricBuilder.name).stat(schema) - override def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = - getStatsReceiver(name).addGauge(verbosity, name: _*)(f) + override def addGauge(schema: GaugeSchema)(f: => Float) = + getStatsReceiver(schema.metricBuilder.name).addGauge(schema)(f) private[this] def getStatsReceiver(name: Seq[String]): StatsReceiver = if (denylisted(name)) NullStatsReceiver else self diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 03494414b1..333831bb19 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -66,83 +66,84 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { new ConcurrentHashMap[Seq[String], () => Float]().asScala override def counter(name: String*): ReadableCounter = - counter(Verbosity.Default, name: _*) + counter(CounterSchema(new MetricBuilder(name = name, statsReceiver = this))) /** * Creates a [[ReadableCounter]] of the given `name`. */ - def counter(v: Verbosity, name: String*): ReadableCounter = + def counter(schema: CounterSchema): ReadableCounter = new ReadableCounter { - verbosity += name -> v + verbosity += schema.metricBuilder.name -> schema.metricBuilder.verbosity // eagerly initialize counters.synchronized { - if (!counters.contains(name)) { - counters(name) = 0 + if (!counters.contains(schema.metricBuilder.name)) { + counters(schema.metricBuilder.name) = 0 } } def incr(delta: Long): Unit = counters.synchronized { val oldValue = apply() - counters(name) = oldValue + delta + counters(schema.metricBuilder.name) = oldValue + delta } - def apply(): Long = counters.getOrElse(name, 0) + def apply(): Long = counters.getOrElse(schema.metricBuilder.name, 0) override def toString: String = - s"Counter(${name.mkString("/")}=${apply()})" + s"Counter(${schema.metricBuilder.name.mkString("/")}=${apply()})" } - override def stat(name: String*): ReadableStat = stat(Verbosity.Default, name: _*) + override def stat(name: String*): ReadableStat = + stat(HistogramSchema(new MetricBuilder(name = name, statsReceiver = this))) /** * Creates a [[ReadableStat]] of the given `name`. */ - def stat(v: Verbosity, name: String*): ReadableStat = + def stat(schema: HistogramSchema): ReadableStat = new ReadableStat { - verbosity += name -> v + verbosity += schema.metricBuilder.name -> schema.metricBuilder.verbosity // eagerly initialize stats.synchronized { - if (!stats.contains(name)) { - stats(name) = Nil + if (!stats.contains(schema.metricBuilder.name)) { + stats(schema.metricBuilder.name) = Nil } } def add(value: Float): Unit = stats.synchronized { val oldValue = apply() - stats(name) = oldValue :+ value + stats(schema.metricBuilder.name) = oldValue :+ value } - def apply(): Seq[Float] = stats.getOrElse(name, Seq.empty) + def apply(): Seq[Float] = stats.getOrElse(schema.metricBuilder.name, Seq.empty) override def toString: String = { val vals = apply() - s"Stat(${name.mkString("/")}=${statValuesToStr(vals)})" + s"Stat(${schema.metricBuilder.name.mkString("/")}=${statValuesToStr(vals)})" } } /** * Creates a [[Gauge]] of the given `name`. */ - def addGauge(v: Verbosity, name: String*)(f: => Float): Gauge = + def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { - gauges += name -> (() => f) - verbosity += name -> v + gauges += schema.metricBuilder.name -> (() => f) + verbosity += schema.metricBuilder.name -> schema.metricBuilder.verbosity def remove(): Unit = { - gauges -= name + gauges -= schema.metricBuilder.name } override def toString: String = { // avoid holding a reference to `f` - val current = gauges.get(name) match { + val current = gauges.get(schema.metricBuilder.name) match { case Some(fn) => fn() case None => -0.0f } - s"Gauge(${name.mkString("/")}=$current)" + s"Gauge(${schema.metricBuilder.name.mkString("/")}=$current)" } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala index f9732263c1..5f7e76c82c 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala @@ -19,13 +19,13 @@ package com.twitter.finagle.stats * modeling whether a gauge is "used" or not. */ final class LazyStatsReceiver(val self: StatsReceiver) extends StatsReceiverProxy { - override def counter(verbosity: Verbosity, names: String*): Counter = new Counter { - private[this] lazy val underlying = self.counter(verbosity, names: _*) + override def counter(schema: CounterSchema): Counter = new Counter { + private[this] lazy val underlying = self.counter(schema) def incr(delta: Long): Unit = underlying.incr(delta) } - override def stat(verbosity: Verbosity, names: String*): Stat = new Stat { - private[this] lazy val underlying = self.stat(verbosity, names: _*) + override def stat(schema: HistogramSchema): Stat = new Stat { + private[this] lazy val underlying = self.stat(schema) def add(value: Float): Unit = underlying.add(value) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 3f911f92ea..e42ddca5bf 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -92,6 +92,8 @@ class MetricBuilder( def withRole(role: SourceRole): MetricBuilder = this.copy(role = role) + def withName(name: Seq[String]): MetricBuilder = this.copy(name = name) + /** * Generates a CounterSchema which can be used to create a counter in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. @@ -139,4 +141,40 @@ class MetricBuilder( val schema = this.copy(name = name).histogramSchema this.statsReceiver.stat(schema) } + + def canEqual(other: Any): Boolean = other.isInstanceOf[MetricBuilder] + + override def equals(other: Any): Boolean = other match { + case that: MetricBuilder => + (that.canEqual(this)) && + keyIndicator == that.keyIndicator && + description == that.description && + units == that.units && + role == that.role && + verbosity == that.verbosity && + sourceClass == that.sourceClass && + sourceLibrary == that.sourceLibrary && + name == that.name && + identifier == that.identifier && + percentiles == that.percentiles + case _ => false + } + + override def hashCode(): Int = { + val state = Seq[AnyRef]( + description, + units, + role, + verbosity, + sourceClass, + sourceLibrary, + name, + identifier, + percentiles) + val hashCodes = keyIndicator.hashCode() +: state.map(_.hashCode()) + hashCodes.foldLeft(0)((a, b) => 31 * a + b) + } + + override def toString = + s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $sourceLibrary, $name, $identifier, $percentiles, $statsReceiver)" } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index 0cb029010e..7992ee9013 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -17,16 +17,21 @@ abstract class NameTranslatingStatsReceiver( protected def translate(name: Seq[String]): Seq[String] - override def counter(verbosity: Verbosity, name: String*): Counter = - self.counter(verbosity, translate(name): _*) + override def counter(counterSchema: CounterSchema): Counter = { + self.counter( + CounterSchema( + counterSchema.metricBuilder.withName(translate(counterSchema.metricBuilder.name)))) + } - override def stat(verbosity: Verbosity, name: String*): Stat = - self.stat(verbosity, translate(name): _*) + override def stat(histogramSchema: HistogramSchema): Stat = { + self.stat( + HistogramSchema( + histogramSchema.metricBuilder.withName(translate(histogramSchema.metricBuilder.name)))) + } - override def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = { - // scalafix:off StoreGaugesAsMemberVariables - self.addGauge(verbosity, translate(name): _*)(f) - // scalafix:on StoreGaugesAsMemberVariables + override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { + self.addGauge( + GaugeSchema(gaugeSchema.metricBuilder.withName(translate(gaugeSchema.metricBuilder.name))))(f) } override def toString: String = s"$self/$namespacePrefix" diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala index 3e5e3a34fb..a6dae83ae9 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala @@ -16,11 +16,9 @@ class NullStatsReceiver extends StatsReceiver { private[this] val NullStat = new Stat { def add(value: Float): Unit = () } private[this] val NullGauge = new Gauge { def remove(): Unit = () } - def counter(verbosity: Verbosity, name: String*): Counter = NullCounter - def stat(verbosity: Verbosity, name: String*): Stat = NullStat - def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = NullGauge - - override def provideGauge(name: String*)(f: => Float): Unit = () + def counter(schema: CounterSchema) = NullCounter + def stat(schema: HistogramSchema) = NullStat + def addGauge(schema: GaugeSchema)(f: => Float) = NullGauge override def scope(namespace: String): StatsReceiver = this @@ -29,4 +27,5 @@ class NullStatsReceiver extends StatsReceiver { override def isNull: Boolean = true override def toString: String = "NullStatsReceiver" + } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala index 2e030495f1..e9c91d5196 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala @@ -24,22 +24,25 @@ class RollupStatsReceiver(protected val self: StatsReceiver) extends StatsReceiv } } - override def counter(verbosity: Verbosity, names: String*): Counter = new Counter { + override def counter(schema: CounterSchema) = new Counter { private[this] val allCounters = BroadcastCounter( - tails(names).map(n => self.counter(verbosity, n: _*)) + tails(schema.metricBuilder.name).map(n => + self.counter(CounterSchema(schema.metricBuilder.withName(n)))) ) def incr(delta: Long): Unit = allCounters.incr(delta) } - - override def stat(verbosity: Verbosity, names: String*): Stat = new Stat { + override def stat(schema: HistogramSchema) = new Stat { private[this] val allStats = BroadcastStat( - tails(names).map(n => self.stat(verbosity, n: _*)) + tails(schema.metricBuilder.name).map(n => + self.stat(HistogramSchema(schema.metricBuilder.withName(n)))) ) def add(value: Float): Unit = allStats.add(value) } - override def addGauge(verbosity: Verbosity, names: String*)(f: => Float): Gauge = new Gauge { - private[this] val underlying = tails(names).map(n => self.addGauge(verbosity, n: _*)(f)) + override def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { + private[this] val underlying = + tails(schema.metricBuilder.name).map(n => + self.addGauge(GaugeSchema(schema.metricBuilder.withName(n)))(f)) def remove(): Unit = underlying.foreach(_.remove()) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 6e08289e8f..5d2e24f97e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -98,17 +98,14 @@ trait StatsReceiver { * Get a [[Counter counter]] with the given `name`. */ @varargs - def counter(verbosity: Verbosity, name: String*): Counter + def counter(verbosity: Verbosity, name: String*): Counter = + counter( + CounterSchema(new MetricBuilder(verbosity = verbosity, name = name, statsReceiver = this))) /** * Get a [[Counter counter]] with the given schema. - * - * This default implementation passes the relevant fields to the pre-schema/pre-metadata interface. - * StatsReceivers with support for schemas/metadata will override this to make better use of that - * information. */ - private[stats] def counter(schema: CounterSchema): Counter = - counter(schema.metricBuilder.verbosity, schema.metricBuilder.name: _*) + def counter(schema: CounterSchema): Counter /** * Get a [[Stat stat]] with the given name. @@ -120,17 +117,14 @@ trait StatsReceiver { * Get a [[Stat stat]] with the given name. */ @varargs - def stat(verbosity: Verbosity, name: String*): Stat + def stat(verbosity: Verbosity, name: String*): Stat = + stat( + HistogramSchema(new MetricBuilder(verbosity = verbosity, name = name, statsReceiver = this))) /** * Get a [[Stat stat]] with the given schema. - * - * This default implementation passes the relevant fields to the pre-schema/pre-metadata interface. - * StatsReceivers with support for schemas/metadata will override this to make better use of that - * information. */ - private[stats] def stat(schema: HistogramSchema): Stat = - stat(schema.metricBuilder.verbosity, schema.metricBuilder.name: _*) + def stat(schema: HistogramSchema): Stat /** * Register a function `f` as a [[Gauge gauge]] with the given name that has @@ -186,7 +180,9 @@ trait StatsReceiver { * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ - def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge + def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = + addGauge( + GaugeSchema(new MetricBuilder(name = name, verbosity = verbosity, statsReceiver = this)))(f) /** * Add the function `f` as a [[Gauge gauge]] with the given name. @@ -200,16 +196,11 @@ trait StatsReceiver { * * Measurements under the same name are added together. * - * This default implementation passes the relevant fields to the pre-schema/pre-metadata interface. - * StatsReceivers with support for schemas/metadata will override this to make better use of that - * information. - * * @see [[StatsReceiver.provideGauge]] when there is not a good location * to store the returned [[Gauge gauge]] that can give the desired lifecycle. * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ - private[stats] def addGauge(schema: GaugeSchema)(f: => Float): Gauge = - addGauge(schema.metricBuilder.verbosity, schema.metricBuilder.name: _*)(f) + def addGauge(schema: GaugeSchema)(f: => Float): Gauge /** * Prepend `namespace` to the names of the returned [[StatsReceiver]]. @@ -305,23 +296,20 @@ trait StatsReceiver { } abstract class AbstractStatsReceiver extends StatsReceiver { - @varargs - final override def counter(name: String*): Counter = counter(Verbosity.Default, name: _*) - @varargs - final def counter(verbosity: Verbosity, name: String*): Counter = - counterImpl(verbosity, name) + + final def counter(schema: CounterSchema): Counter = + counterImpl(schema.metricBuilder.verbosity, schema.metricBuilder.name) + protected def counterImpl(verbosity: Verbosity, name: scala.collection.Seq[String]): Counter - @varargs - final override def stat(name: String*): Stat = stat(Verbosity.Default, name: _*) - @varargs - final def stat(verbosity: Verbosity, name: String*): Stat = statImpl(verbosity, name) + final def stat(schema: HistogramSchema): Stat = + statImpl(schema.metricBuilder.verbosity, schema.metricBuilder.name) + protected def statImpl(verbosity: Verbosity, name: scala.collection.Seq[String]): Stat - final override def addGauge(name: String*)(f: => Float): Gauge = - addGauge(Verbosity.Default, name: _*)(f) - final def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = - addGaugeImpl(verbosity, name, f) + final def addGauge(schema: GaugeSchema)(f: => Float): Gauge = + addGaugeImpl(schema.metricBuilder.verbosity, schema.metricBuilder.name, f) + protected def addGaugeImpl( verbosity: Verbosity, name: scala.collection.Seq[String], diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala index 0eeaece890..d801ea09d2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala @@ -9,10 +9,9 @@ trait StatsReceiverProxy extends StatsReceiver with DelegatingStatsReceiver { def repr: AnyRef = self.repr - def counter(verbosity: Verbosity, names: String*): Counter = self.counter(verbosity, names: _*) - def stat(verbosity: Verbosity, names: String*): Stat = self.stat(verbosity, names: _*) - def addGauge(verbosity: Verbosity, names: String*)(f: => Float): Gauge = - self.addGauge(verbosity, names: _*)(f) // scalafix:ok StoreGaugesAsMemberVariables + def counter(schema: CounterSchema): Counter = self.counter(schema) + def stat(schema: HistogramSchema): Stat = self.stat(schema) + def addGauge(schema: GaugeSchema)(f: => Float): Gauge = self.addGauge(schema)(f) def underlying: Seq[StatsReceiver] = Seq(self) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala index 348472ab9a..3ddb35795b 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala @@ -9,15 +9,16 @@ class VerbosityAdjustingStatsReceiver( defaultVerbosity: Verbosity) extends StatsReceiverProxy { - override def counter(verbosity: Verbosity, names: String*): Counter = - self.counter(defaultVerbosity, names: _*) + override def stat(histogramSchema: HistogramSchema): Stat = { + self.stat(HistogramSchema(histogramSchema.metricBuilder.withVerbosity(defaultVerbosity))) + } - override def stat(verbosity: Verbosity, names: String*): Stat = - self.stat(defaultVerbosity, names: _*) + override def counter(counterSchema: CounterSchema): Counter = { + self.counter(CounterSchema(counterSchema.metricBuilder.withVerbosity(defaultVerbosity))) + } - override def addGauge(verbosity: Verbosity, names: String*)(f: => Float): Gauge = { - // scalafix:off StoreGaugesAsMemberVariables - self.addGauge(defaultVerbosity, names: _*)(f) - // scalafix:on StoreGaugesAsMemberVariables + override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { + self.addGauge(GaugeSchema(gaugeSchema.metricBuilder.withVerbosity(defaultVerbosity)))(f) } + } From e6f39b077cf0ad04c7add22ab6a39a90855c2339 Mon Sep 17 00:00:00 2001 From: David Rusek Date: Mon, 2 Dec 2019 19:19:59 +0000 Subject: [PATCH 248/796] finagle-core,util-core,util-benchmark: tag traces with microsecond timestamp resolution Problem Currently traces are stamped with millisecond resolution timestamps. This makes it hard to sequence within traces because events often occur less than a millisecond apart. In addition we cannot derive meaningful information about relative timing of events. Solution/Result Offer a new method on the Time object to allow us to gather timestamps at nanosecond resolution so that, when using a JDK that supports it, we will have more precise timing information in traces. JIRA Issues: CSL-8695 Differential Revision: https://phabricator.twitter.biz/D400661 --- CHANGELOG.rst | 3 ++ .../com/twitter/util/TimeBenchmark.scala | 6 ++++ .../main/scala/com/twitter/util/Time.scala | 28 +++++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0fbcc4566..010a497111 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,9 @@ New Features * util-stats: Introduces `c.t.f.stats.LazyStatsReceiver` which ensures that counters and histograms don't export metrics until after they have been `incr`ed or `add`ed at least once. ``PHAB_ID=D398898`` +* util-core: Introduce `Time#nowNanoPrecision` to produce nanosecond resolution timestamps in JDK9 + or later. ``PHAB_ID=D400661`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-benchmark/src/main/scala/com/twitter/util/TimeBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/TimeBenchmark.scala index c71cc06611..cc313f4eb6 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/TimeBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/TimeBenchmark.scala @@ -32,4 +32,10 @@ class TimeBenchmark extends StdBenchAnnotations { @Benchmark def timeDiffOverflow: Duration = t5.diff(t4) + + @Benchmark + def timeNow: Time = Time.now + + @Benchmark + def nowNanoPrecision: Time = Time.nowNanoPrecision } diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index b0a09b8096..8edc830fec 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -17,6 +17,7 @@ package com.twitter.util import java.io.Serializable +import java.time.{Clock, Instant} import java.util.concurrent.TimeUnit import java.util.{Date, Locale, TimeZone} @@ -295,6 +296,8 @@ object Time extends TimeLikeOps[Time] { override def fromMilliseconds(millis: Long): Time = super.fromMilliseconds(millis) override def fromMicroseconds(micros: Long): Time = super.fromMicroseconds(micros) + private[this] val SystemClock = Clock.systemUTC() + /** * Time `Top` is greater than any other definable time, and is * equal only to itself. It may be used as a sentinel value, @@ -393,6 +396,31 @@ object Time extends TimeLikeOps[Time] { } } + /** + * Returns the current [[Time]] with, at best, nanosecond precision. + * + * Note that nanosecond precision is only available in JDK versions + * greater than JDK8. In JDK8 this API has the same precision as + * [[Time#now]] and [[System#currentTimeMillis]]. In JDK9+ this will + * change and all timestamps taken from this API will have nanosecond + * resolution. + * + * Note that returned values are '''not''' monotonic. This + * means it is not suitable for measuring durations where + * the clock cannot drift backwards. If monotonicity is desired + * prefer a monotonic [[Stopwatch]]. + * + */ + def nowNanoPrecision: Time = { + localGetTime() match { + case None => + val rightNow = Instant.now(SystemClock) + fromNanoseconds((rightNow.getEpochSecond * Duration.NanosPerSecond) + rightNow.getNano) + case Some(f) => + f() + } + } + /** * The unix epoch. Times are measured relative to this. */ From 5f5f3389f0c3a25d40d52d2d4679fcf5610caa61 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 3 Dec 2019 20:19:58 +0000 Subject: [PATCH 249/796] Revert "Jackson: Update to 2.10.1" Problem / Solution This change reverts the Jackson update to 2.10.1 due to incompatibilities with other existing libraries. JIRA Issues: CSL-9165 Differential Revision: https://phabricator.twitter.biz/D408069 --- CHANGELOG.rst | 5 ----- build.sbt | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 010a497111..e403078d8d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -30,11 +30,6 @@ Deprecations move to slf4j for logging and minimize dependencies on `com.twitter.logging` in general, as it is intended to be replaced entirely by slf4j. ``PHAB_ID=D403574`` -Changes -~~~~~~~ - - * util: Upgrade Jackson dependency from version `2.9.9` to version `2.10.1` ``PHAB_ID=D402722`` - 19.11.0 ------- diff --git a/build.sbt b/build.sbt index 1b09099731..e5647f6773 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll( ExclusionRule("javax.jms", "jms") ) val slf4jVersion = "1.7.21" -val jacksonVersion = "2.10.1" +val jacksonVersion = "2.9.9" val guavaLib = "com.google.guava" % "guava" % "19.0" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" From 2642791962dc34812a3da75be5906e62f495982a Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 5 Dec 2019 23:49:26 +0000 Subject: [PATCH 250/796] util-core: Provide Future#toCompletableFuture Problem Java supplies a CompletableFuture out of the box since jdk8, which is used heavily when integrating with Java libraries. However, it's not always obvious how to derive a Java CompletableFuture from a Twitter Future. Solution Provide a Future#toCompletableFuture method, analogous to Future#toJavaFuture, which provides normal Twitter Future semantics, but via a CompletableFuture. That means that canceling the returned CompletableFuture raises on the underlying Future, and satisfying the underlying Future will satisfy the returned CompletableFuture. JIRA Issues: CSL-9098 Differential Revision: https://phabricator.twitter.biz/D408656 --- CHANGELOG.rst | 10 +++ .../main/scala/com/twitter/util/Future.scala | 61 ++++++++----- .../scala/com/twitter/util/FutureTest.scala | 85 ++++++++++--------- 3 files changed, 95 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e403078d8d..129b0beac5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,9 @@ New Features * util-core: Introduce `Time#nowNanoPrecision` to produce nanosecond resolution timestamps in JDK9 or later. ``PHAB_ID=D400661`` +* util-core: Introduce `Future#toCompletableFuture`, which derives a `CompletableFuture` from + a `com.twitter.util.Future` to make integrating with Java APIs simpler. ``PHAB_ID=D408656`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ @@ -30,6 +33,13 @@ Deprecations move to slf4j for logging and minimize dependencies on `com.twitter.logging` in general, as it is intended to be replaced entirely by slf4j. ``PHAB_ID=D403574`` +Bug Fixes +~~~~~~~~~ + + * util-core: `Future#toJavaFuture` incorrectly threw the exception responsible for failing it, + instead of a `j.u.c.ExecutionException` wrapping the exception responsible for failing it. + ``PHAB_ID=D408656`` + 19.11.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 929a8be933..24e6284b49 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1,8 +1,13 @@ package com.twitter.util import com.twitter.concurrent.{Offer, Tx} -import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger, AtomicReference} -import java.util.concurrent.{CancellationException, TimeUnit, Future => JavaFuture} +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} +import java.util.concurrent.{ + CancellationException, + CompletableFuture, + Future => JavaFuture +} +import java.util.function.BiConsumer import java.util.{List => JList} import scala.collection.compat.immutable.ArraySeq import scala.collection.{Seq => AnySeq} @@ -2153,35 +2158,45 @@ abstract class Future[+A] extends Awaitable[A] { self => /** * Convert a Twitter Future to a Java native Future. This should * match the semantics of a Java Future as closely as possible to - * avoid issues with the way another API might use them. See: + * avoid issues with the way another API might use them. At the same time, + * its semantics should be similar to those of a Twitter Future, so it + * propagates cancellation. See: * * https://download.oracle.com/javase/6/docs/api/java/util/concurrent/Future.html#cancel(boolean) */ - def toJavaFuture: JavaFuture[_ <: A] = { - new JavaFuture[A] { - private[this] val wasCancelled = new AtomicBoolean(false) + def toJavaFuture: JavaFuture[_ <: A] = toCompletableFuture - def cancel(mayInterruptIfRunning: Boolean): Boolean = { - if (wasCancelled.compareAndSet(false, true)) - self.raise(new CancellationException) - true - } + /** + * Convert a Twitter Future to a Java native CompletableFuture. This should + * match the semantics of a Java Future as closely as possible to + * avoid issues with the way another API might use them. At the same time, + * its semantics should be similar to those of a Twitter Future, so it + * propagates cancellation. See: + * + * https://download.oracle.com/javase/6/docs/api/java/util/concurrent/Future.html#cancel(boolean) + */ + def toCompletableFuture[B >: A]: CompletableFuture[B] = { + if (self.isDefined) { + return CompletableFuture.completedFuture(Await.result(self)) + } - def isCancelled: Boolean = wasCancelled.get - def isDone: Boolean = isCancelled || self.isDefined + val f = new CompletableFuture[B]() + this.respond { + case Return(result) => f.complete(result) + case Throw(t) => f.completeExceptionally(t) + } - def get(): A = { - if (isCancelled) - throw new CancellationException - Await.result(self) + f.whenComplete(new BiConsumer[B, Throwable] { + def accept(result: B, t: Throwable): Unit = { + if (t != null && t.isInstanceOf[CancellationException]) { + self.raise(t) + } } + }) - def get(time: Long, timeUnit: TimeUnit): A = { - if (isCancelled) - throw new CancellationException - Await.result(self, Duration.fromTimeUnit(time, timeUnit)) - } - } + // we have to return the original one because cancellation isn't propagated + // by CompletionStages + f } /** diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index be81e32c72..aa39358c46 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -1,7 +1,7 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.{ConcurrentLinkedQueue, ExecutionException, Future => JFuture} import java.util.concurrent.atomic.AtomicInteger import org.mockito.Matchers.any import org.mockito.Mockito.{never, times, verify, when} @@ -858,51 +858,60 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert testJoin("f join g", _ join _) testJoin("Future.join(f, g)", Future.join(_, _)) - "toJavaFuture" should { - "return the same thing as our Future when initialized" which { - val f = const.value(1) - val jf = f.toJavaFuture - assert(await(f) == jf.get()) - "must both be done" in { - assert(f.isDefined) - assert(jf.isDone) - assert(!jf.isCancelled) + def testJavaFuture(methodName: String, fn: Future[Int] => JFuture[_ <: Int]): Unit = { + methodName should { + "return the same thing as our Future when initialized" which { + val f = const.value(1) + val jf = fn(f) + assert(await(f) == jf.get()) + "must both be done" in { + assert(f.isDefined) + assert(jf.isDone) + assert(!jf.isCancelled) + } } - } - "return the same thing as our Future when set later" which { - val f = new Promise[Int] - val jf = f.toJavaFuture - f.setValue(1) - assert(await(f) == jf.get()) - "must both be done" in { - assert(f.isDefined) - assert(jf.isDone) - assert(!jf.isCancelled) + "return the same thing as our Future when set later" which { + val f = new Promise[Int] + val jf = fn(f) + f.setValue(1) + assert(await(f) == jf.get()) + "must both be done" in { + assert(f.isDefined) + assert(jf.isDone) + assert(!jf.isCancelled) + } } - } - "java future should throw an exception" in { - val f = new Promise[Int] - val jf = f.toJavaFuture - val e = new RuntimeException() - f.setException(e) - val actual = intercept[RuntimeException] { jf.get() } - assert(actual == e) - } + "java future should throw an exception" in { + val f = new Promise[Int] + val jf = fn(f) + val e = new RuntimeException() + f.setException(e) + val actual = intercept[ExecutionException] { jf.get() } + val cause = intercept[RuntimeException] { throw actual.getCause } + assert(cause == e) + } - "interrupt Future when cancelled" in { - val f = new HandledPromise[Int] - val jf = f.toJavaFuture - assert(f.handled.isEmpty) - jf.cancel(true) - assert(f.handled match { - case Some(_: java.util.concurrent.CancellationException) => true - case _ => false - }) + "interrupt Future when cancelled" in { + val f = new HandledPromise[Int] + val jf = fn(f) + assert(f.handled.isEmpty) + jf.cancel(true) + intercept[java.util.concurrent.CancellationException] { + throw f.handled.get + } + } } } + testJavaFuture("toJavaFuture", { f: Future[Int] => + f.toJavaFuture + }) + testJavaFuture("toCompletableFuture", { f: Future[Int] => + f.toCompletableFuture + }) + "monitored" should { trait MonitoredHelper { val inner = new HandledPromise[Int] From 7105de04f55f174e952177f833c294cbfdb19a4c Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Sat, 7 Dec 2019 00:22:33 +0000 Subject: [PATCH 251/796] util-jvm: Remove Use of TestLogging in JvmTest Problem / Solution JvmTest uses the now deprecated `TestLogging` mixin. `TestLogging` is dependent on java.util.logging, which we're trying to remove from util and finagle. In addition, `TestLogging` also only works with `WordSpec`, which has not been the recommended testing approach at Twitter for many years. Let's rewrite JvmTest to make it easier to replace jul logging/util-logging with slf4j. JIRA Issues: CSL-9135 Differential Revision: https://phabricator.twitter.biz/D405162 --- .../src/main/scala/com/twitter/jvm/Jvm.scala | 7 +- .../test/scala/com/twitter/jvm/JvmTest.scala | 87 +++++++++---------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala index 10fd812145..90f06d3c12 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala @@ -16,6 +16,9 @@ import scala.util.control.NonFatal trait Jvm { import Jvm.log + // This is intended to only be overridden by tests + protected def logger: Logger = log + trait Opts { def compileThresh: Option[Int] } @@ -87,8 +90,8 @@ trait Jvm { } else if (lastCount != count) { missedCollections += count - 1 - lastCount if (missedCollections > 0 && Time.now - lastLog > LogPeriod) { - if (log.isLoggable(Level.FINE)) { - log.fine( + if (logger.isLoggable(Level.FINE)) { + logger.fine( "Missed %d collections for %s due to sampling" .format(missedCollections, name) ) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala index d525c96f58..28e737b8de 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala @@ -1,42 +1,48 @@ package com.twitter.jvm import com.twitter.conversions.DurationOps._ -import com.twitter.logging.{Level, TestLogging} import com.twitter.util.Time +import java.util.logging.{Level, Logger} +import org.mockito.Matchers.contains +import org.mockito.Mockito.{times, verify, when} import org.scalatest.WordSpec +import org.scalatestplus.mockito.MockitoSugar import scala.collection.mutable -class JvmTest extends WordSpec with TestLogging { +class JvmTest extends WordSpec with MockitoSugar { "Jvm" should { - class JvmHelper { - object jvm extends Jvm { - @volatile private[this] var currentSnap: Snapshot = - Snapshot(Time.epoch, Heap(0, 0, Seq()), Seq()) + class JvmHelper(suppliedLogger: Option[Logger] = None) extends Jvm { + protected override def logger: Logger = suppliedLogger match { + case Some(l) => l + case None => super.logger + } - val opts = new Opts { - def compileThresh = None - } - def snapCounters = Map() - def setSnap(snap: Snapshot): Unit = { - currentSnap = snap - } + @volatile private[this] var currentSnap: Snapshot = + Snapshot(Time.epoch, Heap(0, 0, Seq()), Seq()) - override val executor = new MockScheduledExecutorService + val opts = new Opts { + def compileThresh = None + } + def snapCounters = Map() + def setSnap(snap: Snapshot): Unit = { + currentSnap = snap + } - def snap = currentSnap + override val executor = new MockScheduledExecutorService - def pushGc(gc: Gc): Unit = { - val gcs = snap.lastGcs filter (_.name != gc.name) - setSnap(snap.copy(lastGcs = gc +: gcs)) - } + def snap = currentSnap - def forceGc() = () - def edenPool = NilJvm.edenPool - def metaspaceUsage = NilJvm.metaspaceUsage - def safepoint = NilJvm.safepoint - def applicationTime = NilJvm.applicationTime - def tenuringThreshold = NilJvm.tenuringThreshold + def pushGc(gc: Gc): Unit = { + val gcs = snap.lastGcs filter (_.name != gc.name) + setSnap(snap.copy(lastGcs = gc +: gcs)) } + + def forceGc() = () + def edenPool = NilJvm.edenPool + def metaspaceUsage = NilJvm.metaspaceUsage + def safepoint = NilJvm.safepoint + def applicationTime = NilJvm.applicationTime + def tenuringThreshold = NilJvm.tenuringThreshold } "ProcessId" should { @@ -54,8 +60,7 @@ class JvmTest extends WordSpec with TestLogging { "foreachGc" should { "Capture interleaving GCs with different names" in { - val h = new JvmHelper - import h._ + val jvm = new JvmHelper() val b = mutable.Buffer[Gc]() assert(jvm.executor.schedules == List()) jvm foreachGc { b += _ } @@ -89,13 +94,11 @@ class JvmTest extends WordSpec with TestLogging { "Complain when sampling rate is too low, every 30 minutes" in Time.withCurrentTimeFrozen { tc => - val h = new JvmHelper - import h._ + val logger = mock[Logger] + when(logger.isLoggable(Level.FINE)).thenReturn(true) + val jvm = new JvmHelper(Some(logger)) - traceLogger(Level.DEBUG) - - jvm foreachGc { _ => /*ignore*/ - } + jvm.foreachGc(_ => () /*ignore*/ ) assert(jvm.executor.schedules.size == 1) val r = jvm.executor.schedules.head._1 val gc = Gc(0, "pcopy", Time.now, 1.millisecond) @@ -104,30 +107,24 @@ class JvmTest extends WordSpec with TestLogging { r.run() jvm.pushGc(gc.copy(count = 2)) r.run() - assert(logLines() == Seq("Missed 1 collections for pcopy due to sampling")) + verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) jvm.pushGc(gc.copy(count = 10)) - assert(logLines() == Seq("Missed 1 collections for pcopy due to sampling")) + verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) r.run() tc.advance(29.minutes) r.run() - assert(logLines() == Seq("Missed 1 collections for pcopy due to sampling")) + verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) tc.advance(2.minutes) jvm.pushGc(gc.copy(count = 12)) r.run() - assert( - logLines() == Seq( - "Missed 1 collections for pcopy due to sampling", - "Missed 8 collections for pcopy due to sampling" - ) - ) + verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) + verify(logger, times(1)).fine(contains("Missed 8 collections for pcopy due to sampling")) } } "monitorsGcs" should { "queries gcs in range, in reverse chronological order" in Time.withCurrentTimeFrozen { tc => - val h = new JvmHelper - import h._ - + val jvm = new JvmHelper() val query = jvm.monitorGcs(10.seconds) assert(jvm.executor.schedules.size == 1) val r = jvm.executor.schedules.head._1 From c6e3f31772f1b6a8fb9abc039e446cbf0678ce89 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Mon, 9 Dec 2019 23:55:31 +0000 Subject: [PATCH 252/796] jackson: Update to 2.9.10 (and jackson-databind to 2.9.10.1) Problem / Solution We want to upgrade to the latest version of jackson that we can, yet at this time upgrading to jackson 2.10 is problematic. Let's upgrade instead to the latest 2.9 version - 2.9.10.1. JIRA Issues: CSL-9183 Differential Revision: https://phabricator.twitter.biz/D410846 --- CHANGELOG.rst | 11 ++++++++--- build.sbt | 7 ++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 129b0beac5..7bac246f24 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,11 @@ New Features * util-core: Introduce `Future#toCompletableFuture`, which derives a `CompletableFuture` from a `com.twitter.util.Future` to make integrating with Java APIs simpler. ``PHAB_ID=D408656`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Upgrade to jackson 2.9.10 and jackson-databind 2.9.10.1 ``PHAB_ID=D410846`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ @@ -36,9 +41,9 @@ Deprecations Bug Fixes ~~~~~~~~~ - * util-core: `Future#toJavaFuture` incorrectly threw the exception responsible for failing it, - instead of a `j.u.c.ExecutionException` wrapping the exception responsible for failing it. - ``PHAB_ID=D408656`` +* util-core: `Future#toJavaFuture` incorrectly threw the exception responsible for failing it, + instead of a `j.u.c.ExecutionException` wrapping the exception responsible for failing it. + ``PHAB_ID=D408656`` 19.11.0 ------- diff --git a/build.sbt b/build.sbt index e5647f6773..12d7da00b7 100644 --- a/build.sbt +++ b/build.sbt @@ -12,7 +12,8 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll( ExclusionRule("javax.jms", "jms") ) val slf4jVersion = "1.7.21" -val jacksonVersion = "2.9.9" +val jacksonVersion = "2.9.10" +val jacksonDatabindVersion = "2.9.10.1" val guavaLib = "com.google.guava" % "guava" % "19.0" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" @@ -478,7 +479,7 @@ lazy val utilThrift = Project( "org.apache.thrift" % "libthrift" % "0.10.0", slf4jApi % "provided", "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion ) ).dependsOn(utilCodec) @@ -491,7 +492,7 @@ lazy val utilTunable = Project( name := "util-tunable", libraryDependencies ++= Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude("com.google.guava", "guava") ) ).dependsOn(utilApp, utilCore) From cdc69e03d85729ab6f271e694f3e9a75ae1b25b5 Mon Sep 17 00:00:00 2001 From: Pratik Tandel Date: Tue, 10 Dec 2019 23:03:13 +0000 Subject: [PATCH 253/796] Upgrade guava to 23.6.1-jre **Overview** This ugprades guava to version `23.6.1-jre`. This is not the latest version but upgrading to newer versions leads to more breakages and this will still give us incremental upgrade benefits. Notably this guava version fixes a DoS attack that's prevalent in guava since v11 Multiple backwards compatible fixes have already been committed to source: D379142 D372978 D372978 We had to revert a previous attempt of this due to hadoop jobs loading incompatible guava jars. This has been remediated with AITOOLS-5695 which has been landed. We will wait until next week to allow all jobs to pickup these options in case these new hadoop options introduce their own set of bugs. Differential Revision: https://phabricator.twitter.biz/D372886 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 12d7da00b7..b71c2692f2 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,7 @@ val slf4jVersion = "1.7.21" val jacksonVersion = "2.9.10" val jacksonDatabindVersion = "2.9.10.1" -val guavaLib = "com.google.guava" % "guava" % "19.0" +val guavaLib = "com.google.guava" % "guava" % "23.6.1-jre" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" From a5fb340b44fc3c1da138230f1040c1c73b306a07 Mon Sep 17 00:00:00 2001 From: Pratik Tandel Date: Thu, 12 Dec 2019 19:17:43 +0000 Subject: [PATCH 254/796] Revert "Upgrade guava to 23.6.1-jre" This reverts commit cc00358bc84234a8f50188d174021ffcca3126fd. JIRA Issues: AITOOLS-5798 Differential Revision: https://phabricator.twitter.biz/D412828 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index b71c2692f2..12d7da00b7 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,7 @@ val slf4jVersion = "1.7.21" val jacksonVersion = "2.9.10" val jacksonDatabindVersion = "2.9.10.1" -val guavaLib = "com.google.guava" % "guava" % "23.6.1-jre" +val guavaLib = "com.google.guava" % "guava" % "19.0" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" From b755028bdfded082ba0394f7baadd0489c425b63 Mon Sep 17 00:00:00 2001 From: David Rusek Date: Thu, 12 Dec 2019 23:45:53 +0000 Subject: [PATCH 255/796] Twitter-oss: Prepare OSS libraries for release 19.12.0 Problem We want to release the next versions of our Twitter OSS libraries 19.12.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D413458 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7bac246f24..b1cb1278c0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +19.12.0 +------- + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 58b901d6e8..526eee04b5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "19.11.0" +val utilCore = "com.twitter" %% "util-core" % "19.12.0" ``` # Units diff --git a/build.sbt b/build.sbt index 12d7da00b7..528a9115dc 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.12.0-SNAPSHOT" +val releaseVersion = "19.12.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From fdc4f8f4b6b5866e2a24c6eec7b0bfed7ad8bebb Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Tue, 17 Dec 2019 16:38:43 +0000 Subject: [PATCH 256/796] finagle-stats: store MetricSchemas for MetricsStatsReceiver in MetricStore Problem MetricMetadata (MetricSchemas) are not currently being stored, which makes returning them to the user rather difficult. Solution A map from metric name to MetricSchema (the metric's metadata) has been added to MetricsStatsReceiver. Result MetricSchemas are now stored and will be later surfaced via the metric_metadata endpoint. JIRA Issues: CSL-8920 Differential Revision: https://phabricator.twitter.biz/D381740 --- .../scala/com/twitter/finagle/stats/MetricBuilder.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index e42ddca5bf..8068c20b0a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -57,7 +57,8 @@ class MetricBuilder( sourceClass: String = this.sourceClass, sourceLibrary: Option[String] = this.sourceLibrary, name: Seq[String] = this.name, - identifier: String = this.identifier + identifier: String = this.identifier, + percentiles: IndexedSeq[Double] = this.percentiles ): MetricBuilder = { new MetricBuilder( keyIndicator = keyIndicator, @@ -69,7 +70,7 @@ class MetricBuilder( sourceLibrary = sourceLibrary, name = name, identifier = identifier, - percentiles = this.percentiles, + percentiles = percentiles, statsReceiver = this.statsReceiver ) } @@ -94,6 +95,8 @@ class MetricBuilder( def withName(name: Seq[String]): MetricBuilder = this.copy(name = name) + def withPercentiles(percentiles: IndexedSeq[Double]) = this.copy(percentiles = percentiles) + /** * Generates a CounterSchema which can be used to create a counter in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. From 0b04dbca1b2f1b0fa73b0c7181206ea40c7fb13a Mon Sep 17 00:00:00 2001 From: Zhiyuan Yang Date: Tue, 17 Dec 2019 19:25:29 +0000 Subject: [PATCH 257/796] Revert D381740 : finagle-stats: store MetricSchemas for MetricsStatsReceiver in MetricStore This reverts commit ffa5d416d979c4532520cf39ce2f56488b71229c. Problem D381740 make `stats` to stop publishing percentile metrics like `p90`, `p99`. Solution Revert Result See test section. NO-QUEUE=true Differential Revision: https://phabricator.twitter.biz/D415436 --- .../scala/com/twitter/finagle/stats/MetricBuilder.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 8068c20b0a..e42ddca5bf 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -57,8 +57,7 @@ class MetricBuilder( sourceClass: String = this.sourceClass, sourceLibrary: Option[String] = this.sourceLibrary, name: Seq[String] = this.name, - identifier: String = this.identifier, - percentiles: IndexedSeq[Double] = this.percentiles + identifier: String = this.identifier ): MetricBuilder = { new MetricBuilder( keyIndicator = keyIndicator, @@ -70,7 +69,7 @@ class MetricBuilder( sourceLibrary = sourceLibrary, name = name, identifier = identifier, - percentiles = percentiles, + percentiles = this.percentiles, statsReceiver = this.statsReceiver ) } @@ -95,8 +94,6 @@ class MetricBuilder( def withName(name: Seq[String]): MetricBuilder = this.copy(name = name) - def withPercentiles(percentiles: IndexedSeq[Double]) = this.copy(percentiles = percentiles) - /** * Generates a CounterSchema which can be used to create a counter in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. From 2517c45112c4e123869f0f067a46c3ac825632b8 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Tue, 17 Dec 2019 23:34:14 +0000 Subject: [PATCH 258/796] [logging] bump slf4j to 1.7.30 Problem slf4j is not at the latest version Solution Update slf4j to the latest version Result slf4j is at the latest version Differential Revision: https://phabricator.twitter.biz/D415019 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 528a9115dc..786af8820f 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll( ExclusionRule("com.sun.jmx", "jmxri"), ExclusionRule("javax.jms", "jms") ) -val slf4jVersion = "1.7.21" +val slf4jVersion = "1.7.30" val jacksonVersion = "2.9.10" val jacksonDatabindVersion = "2.9.10.1" From 0f2beaa4971dda5e601fe2b8894a53a67b6e4e96 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sun, 29 Dec 2019 03:10:11 +0000 Subject: [PATCH 259/796] Twitter-oss: update OSS libraries post-release to 20.1.0 We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9268 Differential Revision: https://phabricator.twitter.biz/D418605 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 526eee04b5..98fb1c3f47 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 786af8820f..04aabc48c0 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "19.12.0" +val releaseVersion = "20.1.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 1451e03b6263d16ce9989d0610d64e9735f3b1ca Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 29 Jan 2020 05:13:36 +0000 Subject: [PATCH 260/796] Twitter-oss: Prepare OSS libraries for release 20.1.0 Problem We want to release the next versions of our Twitter OSS libraries 20.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D426839 --- CHANGELOG.rst | 5 +++++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b1cb1278c0..dbf1cc3337 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.1.0 +------ + +No Changes + 19.12.0 ------- diff --git a/README.md b/README.md index 98fb1c3f47..ace64eee08 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "19.12.0" +val utilCore = "com.twitter" %% "util-core" % "20.1.0" ``` # Units diff --git a/build.sbt b/build.sbt index 04aabc48c0..02b35fafc5 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.1.0-SNAPSHOT" +val releaseVersion = "20.1.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 8929141fcc3eb9f4bedceaf875aec2eaf5e3855e Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 29 Jan 2020 23:15:56 +0000 Subject: [PATCH 261/796] Twitter-oss: update OSS libraries post-release to 20.2.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D427350 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ace64eee08..34160c56d0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 02b35fafc5..65b0a46b54 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.1.0" +val releaseVersion = "20.2.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 26beb347f7f24dbf42272c1344d5d59d9ebe896a Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 4 Feb 2020 20:23:31 +0000 Subject: [PATCH 262/796] util-core: Remove Future.unapply Problem `Future.unapply` promotes a very confusing and somewhat non-idiomatic pattern of polling a Future state via Scala's pattern-matching. There is a more explicit way of doing that via `Future.poll` API. Solution Remove `Future.unapply`. JIRA Issues: CSL-8816 Differential Revision: https://phabricator.twitter.biz/D427429 --- CHANGELOG.rst | 7 ++ .../main/scala/com/twitter/util/Future.scala | 2 - .../com/twitter/concurrent/OfferTest.scala | 34 +++--- .../com/twitter/concurrent/SpoolTest.scala | 103 +++++++++--------- 4 files changed, 73 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dbf1cc3337..ffaaf1bda1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: `Future.unapply` has been removed. Use `Future.poll` to retrieve Future's + state. ``PHAB_ID=D427429`` + + 20.1.0 ------ diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 24e6284b49..7c52ed1788 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -162,8 +162,6 @@ object Future { case nlrc: NonLocalReturnControl[_] => Future.exception(new FutureNonLocalReturnControl(nlrc)) } - def unapply[A](f: Future[A]): Option[Try[A]] = f.poll - // The thread-safety for `outer` is guaranteed by synchronizing on `this`. private[this] final class MonitoredPromise[A](private[this] var outer: Promise[A]) extends Monitor diff --git a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala index 00c4096c8e..69dac52d1f 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala @@ -20,6 +20,9 @@ class SimpleOffer[T](var futures: Stream[Future[Tx[T]]]) extends Offer[T] { } class OfferTest extends WordSpec with MockitoSugar { + + def await[A](f: Future[A]): A = Await.result(f, 2.seconds) + import Tx.{Commit, Abort} "Offer.map" should { @@ -39,9 +42,8 @@ class OfferTest extends WordSpec with MockitoSugar { val f = mapped.prepare() flatMap { tx => tx.ack() } - f match { - case Future(Return(Commit("23"))) => assert(true) - } + + assert(await(f) == Commit("23")) } } @@ -79,9 +81,7 @@ class OfferTest extends WordSpec with MockitoSugar { val h = new TxReadyHelper import h._ - offer.prepare() match { - case Future(Return(tx)) => assert(tx eq tx1) - } + assert(await(offer.prepare()) == tx1) verify(tx1, never()).ack() verify(tx1, never()).nack() } @@ -109,9 +109,8 @@ class OfferTest extends WordSpec with MockitoSugar { assert(tx.isDefined == false) val tx0 = mock[Tx[Int]] pendingTxs(0).setValue(tx0) - tx match { - case Future(Return(tx)) => assert(tx eq tx0) - } + + assert(await(tx) == tx0) } "nack losers" in { @@ -158,12 +157,8 @@ class OfferTest extends WordSpec with MockitoSugar { val h = new AllTxsReadyHelper import h._ - offer.prepare() match { - case Future(Return(tx)) => - for (loser <- txs if loser ne tx) - verify(loser).nack() - assert(true) - } + val tx = await(offer.prepare()) + for (loser <- txs if loser ne tx) verify(loser).nack() } } @@ -186,9 +181,8 @@ class OfferTest extends WordSpec with MockitoSugar { assert(tx.ack() == (result)) txp.setValue(tx) - offer.sync() match { - case Future(Return(123)) => assert(true) - } + + assert(await(offer.sync()) == 123) verify(tx, times(2)).ack() verify(tx, never()).nack() } @@ -231,9 +225,7 @@ class OfferTest extends WordSpec with MockitoSugar { assert(tx.ack() == (result)) val offer = spy(new SimpleOffer(tx)) - offer.sync() match { - case Future(Return(123)) => assert(true) - } + assert(await(offer.sync()) == 123) verify(tx, times(2)).ack() verify(tx, never()).nack() verify(offer).prepare() diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala index eff045d682..787b03253a 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala @@ -10,6 +10,9 @@ import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable.ArrayBuffer class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { + + def await[A](f: Future[A]): A = Await.result(f, 2.seconds) + "Empty Spool" should { val s = Spool.empty[Int] @@ -30,7 +33,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { "deconstruct" in { assert(s match { - case x *:: Future(rest) => false + case x *:: _ => false case _ => true }) } @@ -40,28 +43,28 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { assert((Spool.empty[Int] ++ s) == Spool.empty[Int]) val s2 = s ++ (3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) - assert(Await.result(s2.toSeq) == Seq(3, 4)) + assert(await(s2.toSeq) == Seq(3, 4)) } "append via ++ with Future rhs" in { - assert(Await.result(s ++ Future(Spool.empty[Int])) == Spool.empty[Int]) - assert(Await.result(Spool.empty[Int] ++ Future(s)) == Spool.empty[Int]) + assert(await(s ++ Future(Spool.empty[Int])) == Spool.empty[Int]) + assert(await(Spool.empty[Int] ++ Future(s)) == Spool.empty[Int]) val s2 = s ++ Future(3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) - assert(Await.result(s2 flatMap (_.toSeq)) == Seq(3, 4)) + assert(await(s2 flatMap (_.toSeq)) == Seq(3, 4)) } "flatMap" in { val f = (x: Int) => Future(x.toString *:: Future.value((x * 2).toString *:: Future.value(Spool.empty[String]))) - assert(Await.result(s flatMap f) == Spool.empty[Int]) + assert(await(s flatMap f) == Spool.empty[Int]) } "fold left" in { val fold = s.foldLeft(0) { (x, y) => x + y } - assert(Await.result(fold) == 0) + assert(await(fold) == 0) } "reduce left" in { @@ -69,18 +72,18 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { x + y } intercept[UnsupportedOperationException] { - Await.result(fold) + await(fold) } } "zip with empty" in { val result = s.zip(Spool.empty[Int]) - assert(Await.result(result.toSeq) == Nil) + assert(await(result.toSeq) == Nil) } "zip with non-empty" in { val result = s.zip(Seq(1, 2, 3).toSpool) - assert(Await.result(result.toSeq) == Nil) + assert(await(result.toSeq) == Nil) } "take" in { @@ -98,11 +101,11 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { } "buffer to a sequence" in { - assert(Await.result(s.toSeq) == Seq(1, 2)) + assert(await(s.toSeq) == Seq(1, 2)) } "map" in { - assert(Await.result(s.map { _ * 2 }.toSeq) == Seq(2, 4)) + assert(await(s.map { _ * 2 }.toSeq) == Seq(2, 4)) } "mapFuture" in { @@ -112,88 +115,88 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { "deconstruct" in { assert(s match { - case x *:: Future(Return(rest)) => + case x *:: rest => assert(x == 1) - rest match { - case y *:: Future(Return(rest)) if y == 2 && rest.isEmpty => true + await(rest) match { + case y *:: rest if y == 2 => await(rest).isEmpty } }) } "append via ++" in { - assert(Await.result((s ++ Spool.empty[Int]).toSeq) == Seq(1, 2)) - assert(Await.result((Spool.empty[Int] ++ s).toSeq) == Seq(1, 2)) + assert(await((s ++ Spool.empty[Int]).toSeq) == Seq(1, 2)) + assert(await((Spool.empty[Int] ++ s).toSeq) == Seq(1, 2)) val s2 = s ++ (3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) - assert(Await.result(s2.toSeq) == Seq(1, 2, 3, 4)) + assert(await(s2.toSeq) == Seq(1, 2, 3, 4)) } "append via ++ with Future rhs" in { - assert(Await.result(s ++ Future.value(Spool.empty[Int]) flatMap (_.toSeq)) == Seq(1, 2)) - assert(Await.result(Spool.empty[Int] ++ Future.value(s) flatMap (_.toSeq)) == Seq(1, 2)) + assert(await(s ++ Future.value(Spool.empty[Int]) flatMap (_.toSeq)) == Seq(1, 2)) + assert(await(Spool.empty[Int] ++ Future.value(s) flatMap (_.toSeq)) == Seq(1, 2)) val s2 = s ++ Future.value(3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) - assert(Await.result(s2 flatMap (_.toSeq)) == Seq(1, 2, 3, 4)) + assert(await(s2 flatMap (_.toSeq)) == Seq(1, 2, 3, 4)) } "flatMap" in { val f = (x: Int) => Future(Seq(x.toString, (x * 2).toString).toSpool) val s2 = s flatMap f - assert(Await.result(s2 flatMap (_.toSeq)) == Seq("1", "2", "2", "4")) + assert(await(s2 flatMap (_.toSeq)) == Seq("1", "2", "2", "4")) } "fold left" in { val fold = s.foldLeft(0) { (x, y) => x + y } - assert(Await.result(fold) == 3) + assert(await(fold) == 3) } "reduce left" in { val fold = s.reduceLeft { (x, y) => x + y } - assert(Await.result(fold) == 3) + assert(await(fold) == 3) } "zip with empty" in { val zip = s.zip(Spool.empty[Int]) - assert(Await.result(zip.toSeq) == Nil) + assert(await(zip.toSeq) == Nil) } "zip with same size spool" in { val zip = s.zip(Seq("a", "b").toSpool) - assert(Await.result(zip.toSeq) == Seq((1, "a"), (2, "b"))) + assert(await(zip.toSeq) == Seq((1, "a"), (2, "b"))) } "zip with larger spool" in { val zip = s.zip(Seq("a", "b", "c", "d").toSpool) - assert(Await.result(zip.toSeq) == Seq((1, "a"), (2, "b"))) + assert(await(zip.toSeq) == Seq((1, "a"), (2, "b"))) } "be roundtrippable through toSeq/toSpool" in { val seq = (0 to 10).toSeq - assert(Await.result(seq.toSpool.toSeq) == seq) + assert(await(seq.toSpool.toSeq) == seq) } "flatten via flatMap of toSpool" in { val spool = Seq(Seq(1, 2), Seq(3, 4)).toSpool - val seq = Await.result(spool.toSeq) + val seq = await(spool.toSeq) val flatSpool = spool.flatMap { inner => Future.value(inner.toSpool) } - assert(Await.result(flatSpool.flatMap(_.toSeq)) == seq.flatten) + assert(await(flatSpool.flatMap(_.toSeq)) == seq.flatten) } "take" in { val ls = (1 to 4).toSeq.toSpool - assert(Await.result(ls.take(2).toSeq) == Seq(1, 2)) - assert(Await.result(ls.take(1).toSeq) == Seq(1)) - assert(Await.result(ls.take(0).toSeq) == Seq.empty) - assert(Await.result(ls.take(-2).toSeq) == Seq.empty) + assert(await(ls.take(2).toSeq) == Seq(1, 2)) + assert(await(ls.take(1).toSeq) == Seq(1)) + assert(await(ls.take(0).toSeq) == Seq.empty) + assert(await(ls.take(-2).toSeq) == Seq.empty) } } @@ -216,7 +219,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { val xs = new ArrayBuffer[Option[Int]] s foreachElem { xs += _ } intercept[Exception] { - Await.result(s.toSeq) + await(s.toSeq) } } @@ -226,7 +229,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { throw new Exception("sad panda") } intercept[Exception] { - Await.result(f) + await(f) } } @@ -236,7 +239,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { throw new EOFException("sad panda") } intercept[EOFException] { - Await.result(f) + await(f) } } } @@ -282,7 +285,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { assert(xs.toSeq == Seq(Some(1))) p() = Throw(new Exception("sad panda")) intercept[Exception] { - Await.result(s.toSeq) + await(s.toSeq) } } @@ -296,7 +299,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { } p() = Return(2 *:: p1) intercept[Exception] { - Await.result(f) + await(f) } } @@ -310,7 +313,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { } p() = Return(2 *:: p1) intercept[EOFException] { - Await.result(f) + await(f) } } @@ -324,7 +327,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { assert(f.isDefined == false) p1() = Return(Spool.empty) assert(f.isDefined == true) - assert(Await.result(f) == Seq(1, 2)) + assert(await(f) == Seq(1, 2)) } "deconstruct" in { @@ -348,7 +351,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { assert(f.isDefined == false) // 1 != 2 mod 0 p() = Return(2 *:: p1) assert(f.isDefined == true) - val s1 = Await.result(f) + val s1 = await(f) assert(s1 match { case x *:: rest if x == 4 && !rest.isDefined => true case _ => false @@ -361,7 +364,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { p2() = Return(4 *:: Future.value(Spool.empty[Int])) val s1s = s1.toSeq assert(s1s.isDefined == true) - assert(Await.result(s1s) == Seq(4, 8)) + assert(await(s1s) == Seq(4, 8)) } "fold left" in { @@ -377,7 +380,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { assert(f.isDefined == false) p1() = Return(Spool.empty) assert(f.isDefined == true) - assert(Await.result(f) == 3) + assert(await(f) == 3) } "take while" in { @@ -394,7 +397,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { p1() = Return(3 *:: p2) // despite the Spool having an unfulfilled tail, the takeWhile is satisfied assert(f.isDefined == true) - assert(Await.result(f) == Seq(1, 2)) + assert(await(f) == Seq(1, 2)) } } @@ -508,7 +511,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { val c = a ++ f assert(!p.isDefined) - Await.result(c.flatMap(_.tail).select(b.tail)) + await(c.flatMap(_.tail).select(b.tail)) assert(p.isDefined) } @@ -516,7 +519,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { forAll(Arbitrary.arbitrary[List[List[Int]]]) { ss => val all = Spool.merge(ss.map(s => Future.value(s.toSpool))) val interleaved = ss.flatMap(_.zipWithIndex).sortBy(_._2).map(_._1) - assert(Await.result(all.flatMap(_.toSeq)) == interleaved) + assert(await(all.flatMap(_.toSeq)) == interleaved) } } @@ -528,7 +531,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { "foo" *:: Future.value("bar" *:: Future.value("baz" *:: Future.value(Spool.empty[String]))) ).map(Future.value) assert( - Await.result(Spool.merge(spools).flatMap(_.toSeq), 5.seconds) == + await(Spool.merge(spools).flatMap(_.toSeq)) == Seq("a", "1", "foo", "b", "2", "bar", "c", "3", "baz") ) } @@ -536,13 +539,13 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { "Spool.distinctBy should distinct" in { forAll { (s: List[Char]) => val d = s.toSpool.distinctBy(x => x).toSeq - assert(Await.result(d).toSet == s.toSet) + assert(await(d).toSet == s.toSet) } } "Spool.distinctBy should distinct by in order" in { val spool: Spool[String] = "ac" *:: Future.value("bbe" *:: Future.value("ab" *:: Future.value(Spool.empty[String]))) - assert(Await.result(spool.distinctBy(_.length).toSeq, 5.seconds) == Seq("ac", "bbe")) + assert(await(spool.distinctBy(_.length).toSeq) == Seq("ac", "bbe")) } } From d8cc723962da36f6fe844f0d4a6d8dd42e9efbac Mon Sep 17 00:00:00 2001 From: Martijn Hoekstra Date: Wed, 5 Feb 2020 00:36:23 +0000 Subject: [PATCH 263/796] util: replace deprecated JavaConverters Problem/Solution `scala.collection.JavaConverters` is deprecated. Use `scala.jdk.CollectionConverters` instead on 2.11.x and 2.12.x which is provided by the compat library. Signed-off-by: Bryce Anderson JIRA Issues: CSL-9358 Differential Revision: https://phabricator.twitter.biz/D430143 --- util-app/src/main/scala/com/twitter/app/App.scala | 2 +- util-app/src/main/scala/com/twitter/app/ClassPath.scala | 2 +- util-app/src/main/scala/com/twitter/app/Flaggable.scala | 2 +- util-app/src/main/scala/com/twitter/app/Flags.scala | 2 +- util-app/src/main/scala/com/twitter/app/LoadService.scala | 2 +- util-app/src/test/scala/com/twitter/app/AppTest.scala | 2 +- .../main/scala/com/twitter/concurrent/AsyncSemaphore.scala | 2 +- util-core/src/main/scala/com/twitter/util/Activity.scala | 2 +- util-core/src/main/scala/com/twitter/util/Awaitable.scala | 2 +- .../src/main/scala/com/twitter/util/Credentials.scala | 2 +- util-core/src/main/scala/com/twitter/util/Futures.scala | 7 +------ util-core/src/main/scala/com/twitter/util/Var.scala | 2 +- util-core/src/test/scala/com/twitter/util/FutureTest.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala | 2 +- util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala | 2 +- .../src/main/scala/com/twitter/logging/Logger.scala | 2 +- .../main/scala/com/twitter/logging/ThrottledHandler.scala | 2 +- .../com/twitter/util/security/X500PrincipalInfo.scala | 2 +- .../scala/com/twitter/finagle/stats/CumulativeGauge.scala | 2 +- .../com/twitter/finagle/stats/InMemoryStatsReceiver.scala | 2 +- .../scala/com/twitter/util/testing/ArgumentCapture.scala | 2 +- .../scala/com/twitter/util/tunable/JsonTunableMapper.scala | 2 +- .../main/scala/com/twitter/util/tunable/TunableMap.scala | 2 +- .../com/twitter/util/tunable/JsonTunableMapperTest.scala | 2 +- .../main/scala/com/twitter/zk/AsyncCallbackPromise.scala | 2 +- util-zk/src/main/scala/com/twitter/zk/ZNode.scala | 2 +- util-zk/src/main/scala/com/twitter/zk/ZkClient.scala | 2 +- util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala | 2 +- .../com/twitter/zk/coordination/ShardCoordinatorTest.scala | 2 +- .../com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala | 2 +- 32 files changed, 32 insertions(+), 37 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index bb4775447a..26afba54e9 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -6,8 +6,8 @@ import java.lang.reflect.InvocationTargetException import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicReference import java.util.logging.Logger -import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal /** diff --git a/util-app/src/main/scala/com/twitter/app/ClassPath.scala b/util-app/src/main/scala/com/twitter/app/ClassPath.scala index ad0d3b3c3f..bdbaa3ca85 100644 --- a/util-app/src/main/scala/com/twitter/app/ClassPath.scala +++ b/util-app/src/main/scala/com/twitter/app/ClassPath.scala @@ -8,8 +8,8 @@ import java.nio.file.Paths import java.util.jar.{JarEntry, JarFile} import scala.collection.mutable import scala.collection.mutable.Builder -import scala.collection.JavaConverters._ import scala.io.Source +import scala.jdk.CollectionConverters._ private[app] object ClassPath { diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index 6a1c2cb4fd..8a824a6747 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -11,7 +11,7 @@ import java.lang.{ } import java.net.InetSocketAddress import java.util.{List => JList, Map => JMap, Set => JSet} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ /** * A type class providing evidence for parsing type `T` as a flag value. diff --git a/util-app/src/main/scala/com/twitter/app/Flags.scala b/util-app/src/main/scala/com/twitter/app/Flags.scala index 3f8a6683d4..6e76ab9cd2 100644 --- a/util-app/src/main/scala/com/twitter/app/Flags.scala +++ b/util-app/src/main/scala/com/twitter/app/Flags.scala @@ -1,9 +1,9 @@ package com.twitter.app -import scala.collection.JavaConverters._ import scala.collection.immutable.TreeSet import scala.collection.mutable import scala.collection.mutable.ArrayBuffer +import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal /** diff --git a/util-app/src/main/scala/com/twitter/app/LoadService.scala b/util-app/src/main/scala/com/twitter/app/LoadService.scala index 4f0b556948..32c3eaadd2 100644 --- a/util-app/src/main/scala/com/twitter/app/LoadService.scala +++ b/util-app/src/main/scala/com/twitter/app/LoadService.scala @@ -6,9 +6,9 @@ import java.util.ServiceConfigurationError import java.util.concurrent.ConcurrentHashMap import java.util.function.{Function => JFunction} import java.util.logging.{Level, Logger} -import scala.collection.JavaConverters._ import scala.collection.mutable import scala.io.Source +import scala.jdk.CollectionConverters._ import scala.reflect.ClassTag import scala.util.control.NonFatal diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 817924c769..509bf34824 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -101,7 +101,7 @@ class AppTest extends FunSuite { } test("App: order of hooks") { - import scala.collection.JavaConverters._ + import scala.jdk.CollectionConverters._ val q = new ConcurrentLinkedQueue[Int] class Test1 extends App { diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala index 75e64b59f4..3b1f9bd05c 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala @@ -4,7 +4,7 @@ import com.twitter.util._ import java.util.ArrayDeque import java.util.concurrent.RejectedExecutionException import scala.annotation.tailrec -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal /** diff --git a/util-core/src/main/scala/com/twitter/util/Activity.scala b/util-core/src/main/scala/com/twitter/util/Activity.scala index 574cc4bbc3..0e3e475861 100644 --- a/util-core/src/main/scala/com/twitter/util/Activity.scala +++ b/util-core/src/main/scala/com/twitter/util/Activity.scala @@ -2,8 +2,8 @@ package com.twitter.util import java.util.{List => JList} -import scala.collection.JavaConverters._ import scala.collection.mutable.Buffer +import scala.jdk.CollectionConverters._ import scala.language.higherKinds import scala.reflect.ClassTag import scala.util.control.NonFatal diff --git a/util-core/src/main/scala/com/twitter/util/Awaitable.scala b/util-core/src/main/scala/com/twitter/util/Awaitable.scala index 70aa9027ad..ea0942c7f4 100644 --- a/util-core/src/main/scala/com/twitter/util/Awaitable.scala +++ b/util-core/src/main/scala/com/twitter/util/Awaitable.scala @@ -3,7 +3,7 @@ package com.twitter.util import com.twitter.concurrent.Scheduler import java.util.concurrent.atomic.AtomicBoolean import scala.annotation.varargs -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.control.{NonFatal => NF} /** diff --git a/util-core/src/main/scala/com/twitter/util/Credentials.scala b/util-core/src/main/scala/com/twitter/util/Credentials.scala index b780076f3c..fba84ee62f 100644 --- a/util-core/src/main/scala/com/twitter/util/Credentials.scala +++ b/util-core/src/main/scala/com/twitter/util/Credentials.scala @@ -18,8 +18,8 @@ package com.twitter.util import java.io.{File, IOException} -import scala.collection.JavaConverters._ import scala.io.Source +import scala.jdk.CollectionConverters._ import scala.util.parsing.combinator._ import scala.util.matching.Regex diff --git a/util-core/src/main/scala/com/twitter/util/Futures.scala b/util-core/src/main/scala/com/twitter/util/Futures.scala index b7719db384..21184652bb 100644 --- a/util-core/src/main/scala/com/twitter/util/Futures.scala +++ b/util-core/src/main/scala/com/twitter/util/Futures.scala @@ -1,12 +1,7 @@ package com.twitter.util import java.util.{List => JList, Map => JMap} -import scala.collection.JavaConverters.{ - asScalaBufferConverter, - mapAsJavaMapConverter, - mapAsScalaMapConverter, - seqAsJavaListConverter -} +import scala.jdk.CollectionConverters._ /** * Twitter Future utility methods for ease of use from java diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 062edd013e..a9b4e93fcc 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -4,9 +4,9 @@ import java.util.concurrent.atomic.{AtomicLong, AtomicReference, AtomicReference import java.util.{List => JList} import scala.annotation.tailrec import scala.collection.compat._ -import scala.collection.JavaConverters._ import scala.collection.immutable import scala.collection.mutable.Buffer +import scala.jdk.CollectionConverters._ import scala.language.higherKinds import scala.reflect.ClassTag import scala.Iterable diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index aa39358c46..ac9eafab59 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -11,7 +11,7 @@ import org.scalacheck.{Arbitrary, Gen} import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.runtime.NonLocalReturnControl import scala.util.Random import scala.util.control.ControlThrowable diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala b/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala index b33089dee1..11ab3876e9 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala @@ -12,8 +12,8 @@ import javax.management.{ NotificationListener, NotificationEmitter } -import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.jdk.CollectionConverters._ private[jvm] object Allocations { val Unknown: Long = -1L diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index 9d0189e9bc..13bee37de6 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -13,7 +13,7 @@ import javax.management.{ ObjectName, RuntimeMBeanException } -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.language.reflectiveCalls class Hotspot extends Jvm { diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala index 90f06d3c12..119a329f42 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala @@ -6,7 +6,7 @@ import com.twitter.util._ import java.lang.management.ManagementFactory import java.util.concurrent.{ConcurrentHashMap, Executors, ScheduledExecutorService, TimeUnit} import java.util.logging.{Level, Logger} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal /** diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 7b9715eecc..6890604cd6 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -3,8 +3,8 @@ package com.twitter.jvm import com.twitter.conversions.StringOps._ import com.twitter.finagle.stats.StatsReceiver import java.lang.management.{ManagementFactory, BufferPoolMXBean} -import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.jdk.CollectionConverters._ object JvmStats { // set used for keeping track of jvm gauges (otherwise only weakly referenced) diff --git a/util-logging/src/main/scala/com/twitter/logging/Logger.scala b/util-logging/src/main/scala/com/twitter/logging/Logger.scala index f8ff9ea224..0319f9ac56 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Logger.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Logger.scala @@ -19,8 +19,8 @@ package com.twitter.logging import java.util.concurrent.ConcurrentHashMap import java.util.{logging => javalog} import scala.annotation.varargs -import scala.collection.JavaConverters._ import scala.collection.Map +import scala.jdk.CollectionConverters._ class LoggingException(reason: String) extends Exception(reason) diff --git a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala index 3d72256cc1..9108d7970e 100644 --- a/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/ThrottledHandler.scala @@ -23,7 +23,7 @@ import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import java.util.{logging => javalog} import java.util.function.{Function => JFunction} import scala.annotation.tailrec -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ object ThrottledHandler { diff --git a/util-security/src/main/scala/com/twitter/util/security/X500PrincipalInfo.scala b/util-security/src/main/scala/com/twitter/util/security/X500PrincipalInfo.scala index 588bc6a79c..29859fcbea 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X500PrincipalInfo.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X500PrincipalInfo.scala @@ -3,7 +3,7 @@ package com.twitter.util.security import javax.naming.ldap.{LdapName, Rdn} import javax.security.auth.x500.X500Principal import com.twitter.util.security.X500PrincipalInfo._ -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ /** * Parses an [[javax.security.auth.x500.X500Principal X500Principal]] into diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index aa981fba76..0232c6b15f 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -7,7 +7,7 @@ import java.util.concurrent.{ConcurrentHashMap, Executor, ForkJoinPool} import java.util.concurrent.atomic.AtomicInteger import java.util.function.{Function => JFunction} import scala.annotation.tailrec -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ /** * `CumulativeGauge` provides a [[Gauge gauge]] that is composed of the (addition) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 333831bb19..41b32999c2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -2,8 +2,8 @@ package com.twitter.finagle.stats import java.io.PrintStream import java.util.concurrent.ConcurrentHashMap -import scala.collection.JavaConverters._ import scala.collection.{SortedMap, mutable} +import scala.jdk.CollectionConverters._ object InMemoryStatsReceiver { private[stats] implicit class RichMap[K, V](val self: mutable.Map[K, V]) { diff --git a/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala b/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala index 488f5b16dc..e179e0b2d2 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala +++ b/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala @@ -2,7 +2,7 @@ package com.twitter.util.testing import org.mockito.ArgumentCaptor import org.mockito.exceptions.Reporter -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import scala.reflect._ // This file was generated from codegen/util-test/ArgumentCapture.scala.mako diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala index f298a67f1e..99cf886ae2 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.{JsonDeserializer, ObjectMapper} import com.fasterxml.jackson.module.scala.DefaultScalaModule import com.twitter.util.{Return, Throw, Try} import java.net.URL -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ object JsonTunableMapper { diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala index c9b31c0be3..49d770a60c 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala @@ -3,8 +3,8 @@ package com.twitter.util.tunable import java.util.concurrent.ConcurrentHashMap import java.util.function.BiFunction import scala.annotation.varargs -import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.jdk.CollectionConverters._ /** * A Map that can be used to access [[Tunable]]s using [[TunableMap.Key]]s. diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala index 3fea91c811..37c3b0c41e 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala @@ -7,7 +7,7 @@ import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ import com.twitter.util.{Duration, Return, StorageUnit, Throw} import org.scalatest.FunSuite -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ // Used for veryifying custom deserialization case class Foo(number: Double) diff --git a/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala b/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala index 97fc18e561..e7a70debfc 100644 --- a/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala +++ b/util-zk/src/main/scala/com/twitter/zk/AsyncCallbackPromise.scala @@ -2,7 +2,7 @@ package com.twitter.zk import java.util.{List => JList} -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import org.apache.zookeeper.data.Stat import org.apache.zookeeper.{AsyncCallback, KeeperException} diff --git a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala index e04b67c7c6..600415779d 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala @@ -1,7 +1,7 @@ package com.twitter.zk -import scala.collection.JavaConverters._ import scala.collection.{Seq, Set} +import scala.jdk.CollectionConverters._ import org.apache.zookeeper.common.PathUtils import org.apache.zookeeper.data.{ACL, Stat} diff --git a/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala b/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala index 7020ab361a..ede3d426a9 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZkClient.scala @@ -1,6 +1,6 @@ package com.twitter.zk -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import org.apache.zookeeper.ZooDefs.Ids.CREATOR_ALL_ACL import org.apache.zookeeper.data.ACL diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index e1f19d9724..bc5fb0be41 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -1,7 +1,7 @@ package com.twitter.zk -import scala.collection.JavaConverters._ import scala.collection.Set +import scala.jdk.CollectionConverters._ import org.apache.zookeeper._ import org.apache.zookeeper.data.{ACL, Stat} diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala index 0633f320e9..648af965ec 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala @@ -1,6 +1,6 @@ package com.twitter.zk.coordination -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ import org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE import org.scalatest.WordSpec diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala index ffa523eafe..6fe820b1bc 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala @@ -12,7 +12,7 @@ import org.scalatest.concurrent.AsyncAssertions import org.scalatest.concurrent.Eventually._ import org.scalatest.time.{Millis, Seconds, Span} import org.scalatestplus.mockito.MockitoSugar -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ class ZkAsyncSemaphoreTest extends WordSpec with MockitoSugar with AsyncAssertions { From c875d359d6505a6979258f9fd596fa05ebd3c792 Mon Sep 17 00:00:00 2001 From: Matt Kipps Date: Wed, 5 Feb 2020 00:44:42 +0000 Subject: [PATCH 264/796] util-logging: add missing type ascription to Logger's apply method Add missing `_*` to fix potential `java.util.UnknownFormatConversionException` if the log message string contains printf-like things. Signed-off-by: Bryce Anderson JIRA Issues: CSL-9356 Differential Revision: https://phabricator.twitter.biz/D430122 --- CHANGELOG.rst | 5 +++++ util-logging/src/main/scala/com/twitter/logging/Logger.scala | 2 +- .../src/test/scala/com/twitter/logging/LoggerTest.scala | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ffaaf1bda1..b6cfde934c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,11 @@ Breaking API Changes * util-core: `Future.unapply` has been removed. Use `Future.poll` to retrieve Future's state. ``PHAB_ID=D427429`` +Bug Fixes +~~~~~~~~~ + +* util-logging: Add a missing `_*` that could result in exceptions when using the + `Logger.apply(Level, Throwable, String, Any*)` signature. ``PHAB_ID=D430122`` 20.1.0 ------ diff --git a/util-logging/src/main/scala/com/twitter/logging/Logger.scala b/util-logging/src/main/scala/com/twitter/logging/Logger.scala index 0319f9ac56..a703afb050 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Logger.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Logger.scala @@ -83,7 +83,7 @@ class Logger protected (val name: String, private val wrapped: javalog.Logger) { final def apply(level: Level, message: String, items: Any*): Unit = log(level, message, items: _*) final def apply(level: Level, thrown: Throwable, message: String, items: Any*): Unit = - log(level, thrown, message, items) + log(level, thrown, message, items: _*) // convenience methods: @varargs diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index 354b3c01f6..159d07f838 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -205,7 +205,9 @@ class LoggerTest extends WordSpec with TempFolder with BeforeAndAfter { "log & trace a message with percent signs" in { traceLogger(Level.INFO) Logger.get("")(Level.INFO, "%i") + Logger.get("")(Level.INFO, new Exception(), "%j") mustLog("%i") + mustLog("%j") } "log a message, with timestamp" in { From 276fadb20128395eb66a6c66ec9bae51d5399b4d Mon Sep 17 00:00:00 2001 From: Matt Kantor Date: Wed, 5 Feb 2020 01:03:22 +0000 Subject: [PATCH 265/796] util-core: Fix a typo in a Scaladoc comment Problem / Solution There is a typo in the `Monitor` documentation, so let's fix it. Signed-off-by: Bryce Anderson JIRA Issues: CSL-9359 Differential Revision: https://phabricator.twitter.biz/D430210 --- util-core/src/main/scala/com/twitter/util/Monitor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-core/src/main/scala/com/twitter/util/Monitor.scala b/util-core/src/main/scala/com/twitter/util/Monitor.scala index be01c45175..f708cf515f 100644 --- a/util-core/src/main/scala/com/twitter/util/Monitor.scala +++ b/util-core/src/main/scala/com/twitter/util/Monitor.scala @@ -17,7 +17,7 @@ case class MonitorException(handlingExc: Throwable, monitorExc: Throwable) /** * A Monitor is a composable exception handler. It is independent of * position, divorced from the notion of a call stack. Monitors do - * not recover values from a failed computations: It handles only true + * not recover values from failed computations: It handles only true * exceptions that may require cleanup. * * @see [[AbstractMonitor]] for an API friendly to creating instances from Java. From ccfb944bec034945a9736d8f9a2106155c62e567 Mon Sep 17 00:00:00 2001 From: Martijn Hoekstra Date: Thu, 6 Feb 2020 19:58:22 +0000 Subject: [PATCH 266/796] util-core: remove early initializer for TimeLike Problem / Solution Early initializers are deprecated and not needed in TimeLike so just remove it. Signed-off-by: Bryce Anderson JIRA Issues: CSL-9360 Differential Revision: https://phabricator.twitter.biz/D430263 --- .../scala/com/twitter/util/Duration.scala | 10 ++--- .../main/scala/com/twitter/util/Time.scala | 37 ++++++++++--------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index c6c0617623..55525c15ef 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -4,7 +4,6 @@ import java.io.Serializable import java.util.concurrent.TimeUnit object Duration extends TimeLikeOps[Duration] { - def fromNanoseconds(nanoseconds: Long): Duration = if (nanoseconds == 0L) Zero else new Duration(nanoseconds) @@ -304,10 +303,11 @@ private[util] object DurationBox { * their arithmetic follows. This is useful for representing durations * that are truly infinite; for example the absence of a timeout. */ -sealed class Duration private[util] (protected val nanos: Long) extends { - protected val ops: Duration.type = Duration -} with TimeLike[Duration] with Serializable { - import ops._ +sealed class Duration private[util] (protected val nanos: Long) + extends TimeLike[Duration] + with Serializable { + override def ops: TimeLikeOps[Duration] = Duration + import Duration._ def inNanoseconds: Long = nanos diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index 8edc830fec..87cc7b8ffc 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -139,8 +139,7 @@ trait TimeLikeOps[This <: TimeLike[This]] { * Overflows are also handled like doubles. */ trait TimeLike[This <: TimeLike[This]] extends Ordered[This] { self: This => - protected val ops: TimeLikeOps[This] - import ops._ + protected def ops: TimeLikeOps[This] /** The `TimeLike`'s value in nanoseconds. */ def inNanoseconds: Long @@ -182,14 +181,14 @@ trait TimeLike[This <: TimeLike[This]] extends Ordered[This] { self: This => def addNanos(a: Long, b: Long): This = { val c = a + b if (((a ^ c) & (b ^ c)) < 0) - if (b < 0) Bottom else Top - else fromNanoseconds(c) + if (b < 0) ops.Bottom else ops.Top + else ops.fromNanoseconds(c) } delta match { - case Duration.Top => Top - case Duration.Bottom => Bottom - case Duration.Undefined => Undefined + case Duration.Top => ops.Top + case Duration.Bottom => ops.Bottom + case Duration.Undefined => ops.Undefined case ns => addNanos(inNanoseconds, ns.inNanoseconds) } } @@ -223,12 +222,13 @@ trait TimeLike[This <: TimeLike[This]] extends Ordered[This] { self: This => * results because of timezones. */ def floor(increment: Duration): This = (this, increment) match { - case (num, ns) if num.isZero && ns.isZero => Undefined - case (num, ns) if num.isFinite && ns.isZero => if (num.inNanoseconds < 0) Bottom else Top + case (num, ns) if num.isZero && ns.isZero => ops.Undefined + case (num, ns) if num.isFinite && ns.isZero => + if (num.inNanoseconds < 0) ops.Bottom else ops.Top case (num, denom) if num.isFinite && denom.isFinite => - fromNanoseconds((num.inNanoseconds / denom.inNanoseconds) * denom.inNanoseconds) + ops.fromNanoseconds((num.inNanoseconds / denom.inNanoseconds) * denom.inNanoseconds) case (self, n) if n.isFinite => self - case (_, _) => Undefined + case (_, _) => ops.Undefined } def max(that: This): This = @@ -238,15 +238,15 @@ trait TimeLike[This <: TimeLike[This]] extends Ordered[This] { self: This => if ((this compare that) < 0) this else that def compare(that: This): Int = - if ((that eq Top) || (that eq Undefined)) -1 - else if (that eq Bottom) 1 + if ((that eq ops.Top) || (that eq ops.Undefined)) -1 + else if (that eq ops.Bottom) 1 else if (inNanoseconds < that.inNanoseconds) -1 else if (inNanoseconds > that.inNanoseconds) 1 else 0 /** Equality within `maxDelta` */ def moreOrLessEquals(other: This, maxDelta: Duration): Boolean = - (other ne Undefined) && ((this == other) || (this diff other).abs <= maxDelta) + (other ne ops.Undefined) && ((this == other) || (this diff other).abs <= maxDelta) } /** @@ -585,10 +585,11 @@ class TimeFormat( * @see [[Stopwatch]] for measuring elapsed time. * @see [[TimeFormat]] for converting to and from `String` representations. */ -sealed class Time private[util] (protected val nanos: Long) extends { - protected val ops: Time.type = Time -} with TimeLike[Time] with Serializable { - import ops._ +sealed class Time private[util] (protected val nanos: Long) + extends TimeLike[Time] + with Serializable { + protected def ops: Time.type = Time + import Time._ def inNanoseconds: Long = nanos From d523c0c50de551814d4eb8485d68db52e8c14c51 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 13 Feb 2020 00:54:59 +0000 Subject: [PATCH 267/796] util-app,twitter-server,inject-app,inject-server: Better Java support Problem We are missing an easy way for Java users to specify `closeOnExitLast` functions in addition to having any Java App/TwitterServer lifecycle ergonomics in the Finatra `c.t.inject.app.AbstractApp` or the `c.t.inject.server.AbstractTwitterServer`. Right now Java users cannot easily specify a last exit function and Finatra Java users do not have access to the same basic lifecycle functions as are exposed in the `c.t.server.AbstractTwitterServer`. Solution Introduce `c.t.util.app.App#onExitLast` and carry it through to `c.t.server.TwitterServer`. Expose Java hooks in `c.t.server.AbstractTwitterServer` as well as `c.t.inject.app.AbstractApp` and `c.t.inject.server.AbstractTwitterServer` for Finatra users. Result Better support Java users around the base App and TwitterServer lifecycle. JIRA Issues: CSL-9402 Differential Revision: https://phabricator.twitter.biz/D433874 --- CHANGELOG.rst | 6 ++++++ util-app/src/main/scala/com/twitter/app/App.scala | 15 +++++++++++++-- .../src/test/scala/com/twitter/app/AppTest.scala | 3 ++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b6cfde934c..26864762f4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-app: Add `c.t.util.app.App#onExitLast` to be able to provide better Java + ergonomics for designating a final exit function. ``PHAB_ID=D433874`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index 26afba54e9..14f517798c 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -5,7 +5,6 @@ import com.twitter.util._ import java.lang.reflect.InvocationTargetException import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicReference -import java.util.logging.Logger import scala.collection.mutable import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal @@ -272,6 +271,19 @@ trait App extends Closable with CloseAwaitably { } } + /** + * Invoke `f` when shutdown is requested. Exit hooks run in parallel and are + * executed after all closeOnExit functions complete. The thread resumes when all exit + * hooks complete or `closeDeadline` expires. + */ + protected final def onExitLast(f: => Unit): Unit = { + closeOnExitLast { + Closable.make { deadline => // close() ensures that this deadline is sane + FuturePool.unboundedPool(f).by(shutdownTimer, deadline) + } + } + } + /** * Invoke `f` after the user's main has exited. */ @@ -393,7 +405,6 @@ trait App extends Closable with CloseAwaitably { } object App { - private[this] val log = Logger.getLogger(getClass.getName) private[this] val ref = new AtomicReference[Option[App]](None) /** diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 509bf34824..c2665e50fe 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -105,6 +105,7 @@ class AppTest extends FunSuite { val q = new ConcurrentLinkedQueue[Int] class Test1 extends App { + onExitLast(q.add(5)) onExit(q.add(4)) postmain(q.add(3)) def main(): Unit = q.add(2) @@ -112,7 +113,7 @@ class AppTest extends FunSuite { init(q.add(0)) } new Test1().main(Array.empty) - assert(q.asScala.toSeq == Seq(0, 1, 2, 3, 4)) + assert(q.asScala.toSeq == Seq(0, 1, 2, 3, 4, 5)) } test("loadServiceBinds") { From 9976376728569582585a8d46132321c9acc86ca2 Mon Sep 17 00:00:00 2001 From: pliu Date: Wed, 19 Feb 2020 21:27:08 +0000 Subject: [PATCH 268/796] Update BUILD files - deprecate [rz]globs `pants 1.25.x` will be deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K BUILD files - replaces [rz]globs. JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D436777 --- util-core/src/test/java/com/twitter/concurrent/BUILD | 2 +- util-security/src/test/java/com/twitter/util/security/BUILD | 2 +- util-security/src/test/scala/com/twitter/util/security/BUILD | 2 +- util-slf4j-api/src/test/resources/BUILD | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util-core/src/test/java/com/twitter/concurrent/BUILD b/util-core/src/test/java/com/twitter/concurrent/BUILD index 809d672c40..d03eadbe88 100644 --- a/util-core/src/test/java/com/twitter/concurrent/BUILD +++ b/util-core/src/test/java/com/twitter/concurrent/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD index e46848252f..37a0ef71d1 100644 --- a/util-security/src/test/java/com/twitter/util/security/BUILD +++ b/util-security/src/test/java/com/twitter/util/security/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index bd109b27ba..e50f35069b 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-slf4j-api/src/test/resources/BUILD b/util-slf4j-api/src/test/resources/BUILD index c0cac6941d..ba87e2b54d 100644 --- a/util-slf4j-api/src/test/resources/BUILD +++ b/util-slf4j-api/src/test/resources/BUILD @@ -1,3 +1,3 @@ resources( - sources = globs("*"), + sources = ["*"], ) From 69f0a5c5c231770abca4680ad37401b56426a3a0 Mon Sep 17 00:00:00 2001 From: Alex Levenson Date: Thu, 20 Feb 2020 00:39:27 +0000 Subject: [PATCH 269/796] util-core: Add Buf.slowFromHexString to reverse Buf.slowHexString Problem: Buf.slowHexString is a useful helper for logging + debugging binary blobs for inspection. However, often the first thing a user will want to do with one of these hex strings is load it as a Buf in some test environment / REPL. Currently there's no built in helper to parse one of these hex Strings back into a Buf. Solution: Introduce Buf.slowFromHexString, which parses a hex string into a Buf. Result: Users may use this helper when debugging a logged hex string. Differential Revision: https://phabricator.twitter.biz/D436739 --- .../src/main/scala/com/twitter/io/Buf.scala | 23 ++++++++++++++++++ .../test/scala/com/twitter/io/BufTest.scala | 24 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 092119b788..8a2f3b0af6 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -1038,6 +1038,29 @@ object Buf { digits.toString } + /** + * Convert a hex string (eg one from slowHexString above) + * to a Buf of the string contents interpreted as a string of hexadecimal numbers. + * + * If an invalid string is passed to this method, the return value is not defined. It may throw + * or it may return a bogus Buf value. + */ + def slowFromHexString(hex: String): Buf = { + if (hex.length % 2 != 0) { + throw new IllegalArgumentException(s"Hex string is invalid: '$hex'") + } + val bytes = new Array[Byte](hex.length / 2) + var bytesIndex = 0 + while (bytesIndex < bytes.length) { + val hexIndex = bytesIndex * 2 + val b = Integer.parseInt(hex.substring(hexIndex, hexIndex + 2), 16) + bytes(bytesIndex) = b.toByte + bytesIndex = bytesIndex + 1 + } + + Buf.ByteArray.Owned(bytes) + } + /** * Create and deconstruct Utf-8 encoded buffers. * diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index 151daddd59..370592857b 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -1010,4 +1010,28 @@ class BufTest assert(constructor == concatRight) } } + + test("slowFromHexString handles empty string") { + val b = Buf.slowFromHexString("") + assert(b == Buf.ByteArray.Owned(new Array[Byte](0))) + } + + test("slowFromHexString throws for (some) invalid strings") { + // we don't guarantee all invalid strings throw but we do cover some of the basic ones + intercept[IllegalArgumentException] { + Buf.slowFromHexString("123") + } + + intercept[NumberFormatException] { + Buf.slowFromHexString("10ZZ") + } + } + + test("slowHexString round trips") { + forAll(arbBuf.arbitrary) { buf => + val asHex = Buf.slowHexString(buf) + val backToBuf = Buf.slowFromHexString(asHex) + assert(backToBuf == buf) + } + } } From 11fcbed00b725dbbbe8643c30d8772d2e85dbc16 Mon Sep 17 00:00:00 2001 From: pliu Date: Thu, 20 Feb 2020 07:47:00 +0000 Subject: [PATCH 270/796] Update Build files - deprecate [rz]globs (batch#2) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K BUILD files - replaces [rz]globs. (batch#2) JIRA Issues: DPB-12073 NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D437281 --- util-intellij/src/main/resources/BUILD | 2 +- util-jvm/src/test/scala/com/twitter/jvm/BUILD | 2 +- util-reflect/src/test/scala/com/twitter/util/reflect/BUILD | 2 +- util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD | 2 +- util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD | 5 ++++- util-tunable/src/test/java/com/twitter/util/tunable/BUILD | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/util-intellij/src/main/resources/BUILD b/util-intellij/src/main/resources/BUILD index cc44b5fc88..a001371a96 100644 --- a/util-intellij/src/main/resources/BUILD +++ b/util-intellij/src/main/resources/BUILD @@ -1,3 +1,3 @@ resources( - sources = rglobs("*"), + sources = ["**/*"], ) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/BUILD b/util-jvm/src/test/scala/com/twitter/jvm/BUILD index 396867d4eb..6150287f3e 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/test/scala/com/twitter/jvm/BUILD @@ -20,7 +20,7 @@ junit_tests( scala_library( name = "estimator_app", - sources = globs("EstimatorApp.scala"), + sources = ["EstimatorApp.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "util/util-jvm/src/main/scala/com/twitter/jvm", diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD index e8ffe7c0a1..0dda289399 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD index a261c4cc21..772f8de291 100644 --- a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index 15efdb04be..c18a516ee0 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -1,5 +1,8 @@ junit_tests( - sources = globs("*.scala", "*.java"), + sources = [ + "*.java", + "*.scala", + ], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD index b34dd76ac5..fea37d9a99 100644 --- a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", From b99fe186391d47876738173bfdb8124866b1b418 Mon Sep 17 00:00:00 2001 From: krisgun Date: Thu, 20 Feb 2020 21:29:37 +0000 Subject: [PATCH 271/796] util-core: improve test coverage for method parse in Duration Problem The way it is currently implemented, the test will pass after the first NumberFormatException for the string "". The rest of the strings are therefore not tested. Solution Moving the assertion to within the foreach will ensure that all strings in the Seq are tested. Result Improves code coverage in parse. Two additional branches in the code are now checked. Signed-off-by: Yufan Gong Differential Revision: https://phabricator.twitter.biz/D437860 --- .../scala/com/twitter/util/DurationTest.scala | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 6c2c701a28..56d29e77f6 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -223,7 +223,7 @@ class DurationTest extends { val ops: Duration.type = Duration } with TimeLikeSp Duration.Bottom, Duration.Top, Duration.Undefined - ) foreach { d => + ).foreach { d => assert(Duration.parse(d.toString) == d) } } @@ -251,26 +251,26 @@ class DurationTest extends { val ops: Duration.type = Duration } with TimeLikeSp "Duration.Top" -> Duration.Top, "Duration.Undefined" -> Duration.Undefined, "duration.TOP" -> Duration.Top - ) foreach { + ).foreach { case (s, d) => assert(Duration.parse(s) == d) } } "reject obvious human impostors" in { - intercept[NumberFormatException] { - Seq( - "", - "++1.second", - "1. second", - "1.milli", - "1.s", - "10.stardates", - "2.minutes 1.second", - "98 milliseconds", - "98 millisecons", - "99.minutes +" - ) foreach { s => + Seq( + "", + "++1.second", + "1. second", + "1.milli", + "1.s", + "10.stardates", + "2.minutes 1.second", + "98 milliseconds", + "98 millisecons", + "99.minutes +" + ).foreach { s => + assertThrows[NumberFormatException] { Duration.parse(s) } } From f73bb16f7117a9feae06d213e1b726f663e96a38 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Fri, 21 Feb 2020 15:38:24 +0000 Subject: [PATCH 272/796] finagle-stats: store MetricSchemas for MetricsStatsReceiver in Metrics Problem MetricMetadata (MetricSchemas) are not currently being stored, which makes returning them to the user rather difficult. Solution A map from metric name to MetricSchema (the metric's metadata) has been added to Metrics (the MetricStore implemenation used by MetricStatsReceiver). What's New https://phabricator.twitter.biz/D381740 solved the Problem, but contained a bug which caused all histograms to appear without buckets. This commit populates and propagates histogram buckets/percentiles properly. If a HistogramSchema reaches Metrics without any percentiles in place, Metrics set the percentiles to BucketedHistogram.DefaultQuantiles, this is an equivalent location to where they were set in the pre-schema-interfaces world. Result MetricSchemas are now stored and will be later surfaced via the metric_metadata endpoint. JIRA Issues: CSL-8920 Differential Revision: https://phabricator.twitter.biz/D429165 --- .../twitter/finagle/stats/InMemoryStatsReceiver.scala | 4 ++-- .../scala/com/twitter/finagle/stats/MetricBuilder.scala | 7 +++++-- .../scala/com/twitter/finagle/stats/StatsReceiver.scala | 9 +++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 41b32999c2..e6549897e7 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -66,7 +66,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { new ConcurrentHashMap[Seq[String], () => Float]().asScala override def counter(name: String*): ReadableCounter = - counter(CounterSchema(new MetricBuilder(name = name, statsReceiver = this))) + counter(CounterSchema(this.metricBuilder().withName(name))) /** * Creates a [[ReadableCounter]] of the given `name`. @@ -95,7 +95,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } override def stat(name: String*): ReadableStat = - stat(HistogramSchema(new MetricBuilder(name = name, statsReceiver = this))) + stat(HistogramSchema(this.metricBuilder().withName(name))) /** * Creates a [[ReadableStat]] of the given `name`. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index e42ddca5bf..8068c20b0a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -57,7 +57,8 @@ class MetricBuilder( sourceClass: String = this.sourceClass, sourceLibrary: Option[String] = this.sourceLibrary, name: Seq[String] = this.name, - identifier: String = this.identifier + identifier: String = this.identifier, + percentiles: IndexedSeq[Double] = this.percentiles ): MetricBuilder = { new MetricBuilder( keyIndicator = keyIndicator, @@ -69,7 +70,7 @@ class MetricBuilder( sourceLibrary = sourceLibrary, name = name, identifier = identifier, - percentiles = this.percentiles, + percentiles = percentiles, statsReceiver = this.statsReceiver ) } @@ -94,6 +95,8 @@ class MetricBuilder( def withName(name: Seq[String]): MetricBuilder = this.copy(name = name) + def withPercentiles(percentiles: IndexedSeq[Double]) = this.copy(percentiles = percentiles) + /** * Generates a CounterSchema which can be used to create a counter in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 5d2e24f97e..9c3459fcc3 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -99,8 +99,7 @@ trait StatsReceiver { */ @varargs def counter(verbosity: Verbosity, name: String*): Counter = - counter( - CounterSchema(new MetricBuilder(verbosity = verbosity, name = name, statsReceiver = this))) + counter(CounterSchema(this.metricBuilder().withVerbosity(verbosity).withName(name))) /** * Get a [[Counter counter]] with the given schema. @@ -118,8 +117,7 @@ trait StatsReceiver { */ @varargs def stat(verbosity: Verbosity, name: String*): Stat = - stat( - HistogramSchema(new MetricBuilder(verbosity = verbosity, name = name, statsReceiver = this))) + stat(HistogramSchema(this.metricBuilder().withVerbosity(verbosity).withName(name))) /** * Get a [[Stat stat]] with the given schema. @@ -181,8 +179,7 @@ trait StatsReceiver { * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = - addGauge( - GaugeSchema(new MetricBuilder(name = name, verbosity = verbosity, statsReceiver = this)))(f) + addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name)))(f) /** * Add the function `f` as a [[Gauge gauge]] with the given name. From 794331ee4487bd181d25c0b233afef395f3f1cf8 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 21 Feb 2020 16:54:27 +0000 Subject: [PATCH 273/796] Twitter-oss: Prepare OSS libraries for release 20.2.0 Problem We want to release the next versions of our Twitter OSS libraries 20.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9397 Differential Revision: https://phabricator.twitter.biz/D438131 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 26864762f4..26e64bd244 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.2.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 34160c56d0..83741c3d16 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.1.0" +val utilCore = "com.twitter" %% "util-core" % "20.2.0" ``` # Units diff --git a/build.sbt b/build.sbt index 65b0a46b54..9f2b4ecdc4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.2.0-SNAPSHOT" +val releaseVersion = "20.2.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 68b28ac1835e920f5f0a0a17cdb33953032679ee Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 21 Feb 2020 20:24:33 +0000 Subject: [PATCH 274/796] finatra/http: Add toBufReader into StreamingResponse Problem `StreamingResponse` only carries Json streams, valid Json arrays. However, sometimes there are cases we want to add prefixes or suffixes of the Json array. Solution Add a `toBufReader` to get the serialized `Reader[Buf]`, it can be used to be modified to any form as long as they are Bufs. Add a `Reader.concat` to merge a collection of Readers to a single Reader for convenience. JIRA Issues: CSL-9252 Differential Revision: https://phabricator.twitter.biz/D434448 --- CHANGELOG.rst | 3 ++ .../src/main/java/com/twitter/io/Readers.java | 14 +++++-- .../main/scala/com/twitter/io/Reader.scala | 9 +++++ .../scala/com/twitter/io/ReaderTest.scala | 37 ++++++++++++++++--- 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 26e64bd244..26ed9e1400 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,9 @@ New Features * util-app: Add `c.t.util.app.App#onExitLast` to be able to provide better Java ergonomics for designating a final exit function. ``PHAB_ID=D433874`` +* util-core: Add `c.t.io.Reader.concat` to conveniently concatenate a collection + of Reader to a single Reader. ``PHAB_ID=D434448`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index 5b29bb3416..fa2bd2e958 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -43,6 +43,14 @@ public static Reader concat(AsyncStream> readers) { return Reader$.MODULE$.concat(readers); } + /** + * See [[com.twitter.io.Reader.concat(scala.collection.Seq)]] + */ + public static Reader concat(List> list) { + return Reader$.MODULE$.concat(JavaConverters.asScalaIteratorConverter(list.iterator()) + .asScala().toSeq()); + } + /** * See {@code com.twitter.io.Pipe.copy}. */ @@ -65,7 +73,7 @@ public static Reader newInputStreamReader(InputStream in, int chunkSize) { } /** - * See [[com.twitter.io.Reader$#fromSeq(scala.collection.Seq)]] + * See [[com.twitter.io.Reader.fromSeq(scala.collection.Seq)]] */ public static Reader fromSeq(List list) { return Reader$.MODULE$.fromSeq(JavaConverters.asScalaIteratorConverter(list.iterator()) @@ -73,7 +81,7 @@ public static Reader fromSeq(List list) { } /** - * See [[com.twitter.io.Reader$#fromSeq(scala.collection.Seq)]] + * See [[com.twitter.io.Reader.fromSeq(scala.collection.Seq)]] */ public static Reader fromSeq(Stream stream) { return Reader$.MODULE$.fromSeq(JavaConverters.asScalaIteratorConverter(stream.iterator()) @@ -81,7 +89,7 @@ public static Reader fromSeq(Stream stream) { } /** - * See [[com.twitter.io.Reader#toAsyncStream]] + * See [[com.twitter.io.Reader.toAsyncStream]] */ public static AsyncStream toAsyncStream(Reader reader) { return Reader$.MODULE$.toAsyncStream(reader); diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 0dc4ef524d..979973cf1d 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -333,6 +333,15 @@ object Reader { target } + /** + * Convenient abstraction to read from a collection of Readers as if + * it were a single Reader. + * @param readers A collection of Reader[A] + */ + def concat[A](readers: Seq[Reader[A]]): Reader[A] = { + Reader.fromSeq(readers).flatten + } + /** * Convenient abstraction to read from a stream (Reader) of Readers as if it were a single Reader. * @param readers A Reader holds a stream of Reader[A] diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 3568b68926..a34f8b6045 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -60,7 +60,25 @@ class ReaderTest case _ => to.close() } - test("Reader.concat") { + test("Reader.concat collection of readers") { + forAll { ss: List[String] => + val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) + val concatedReaders = Reader.concat(readers) + val values = Reader.readAllItems(concatedReaders) + assert(await(values) == ss) + } + } + + test("Reader.concat from a collection won't force the stream") { + forAll { ss: List[String] => + val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) + val head = Reader.value("hmm") + Reader.concat(head +: readers) + assert(await(head.read()) == Some("hmm")) + } + } + + test("Reader.concat from a stream of readers") { forAll { ss: List[String] => val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) val concatedReaders = Reader.concat(AsyncStream.fromSeq(readers)) @@ -69,7 +87,16 @@ class ReaderTest } } - test("Reader.concat - discard") { + test("Reader.concat from a stream of readers forces the stream") { + forAll { ss: List[String] => + val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) + val head = Reader.value("hmm") + Reader.concat(AsyncStream.fromSeq(head +: readers)) + assert(await(head.read()) == None) + } + } + + test("Reader.concat from a stream of readers - discard") { val p = new Promise[Option[Buf]] val head = new Reader[Buf] { def read(): Future[Option[Buf]] = p @@ -81,7 +108,7 @@ class ReaderTest assert(p.isDefined) } - test("Reader.concat - read while reading") { + test("Reader.concat from a stream of readers - read while reading") { val p = new Promise[Option[Buf]] val head = new Reader[Buf] { def read(): Future[Option[Buf]] = p @@ -92,7 +119,7 @@ class ReaderTest assertReadWhileReading(reader) } - test("Reader.concat - failed") { + test("Reader.concat from a stream of readers - failed") { val p = new Promise[Option[Buf]] val head = new Reader[Buf] { def read(): Future[Option[Buf]] = p @@ -103,7 +130,7 @@ class ReaderTest assertFailed(reader, p) } - test("Reader.concat - lazy tail") { + test("Reader.concat from a stream of readers - lazy tail") { val head = new Reader[Buf] { def read(): Future[Option[Buf]] = Future.exception(new Exception) From 453795fdf3e3cc2dc45a4f777ba653eb4b144f37 Mon Sep 17 00:00:00 2001 From: pliu Date: Fri, 21 Feb 2020 22:07:13 +0000 Subject: [PATCH 275/796] Update Build files - deprecate [rz]globs (batch#3) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K+ BUILD files - replaces [rz]globs. (batch#3) JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D437945 --- util-core/src/test/scala/com/twitter/conversions/BUILD | 2 +- util-core/src/test/scala/com/twitter/io/exp/BUILD | 2 +- util-registry/src/test/java/com/twitter/util/registry/BUILD | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD index 6ba685bd61..c931f97e67 100644 --- a/util-core/src/test/scala/com/twitter/conversions/BUILD +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], # fatal_warnings runs into an issue with scalatest/Future.isDone: # # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` diff --git a/util-core/src/test/scala/com/twitter/io/exp/BUILD b/util-core/src/test/scala/com/twitter/io/exp/BUILD index 2e5443914c..b874b8d55c 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/BUILD +++ b/util-core/src/test/scala/com/twitter/io/exp/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], # fatal_warnings runs into an issue with scalatest/Future.isDone: # # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` diff --git a/util-registry/src/test/java/com/twitter/util/registry/BUILD b/util-registry/src/test/java/com/twitter/util/registry/BUILD index 20a7de6cba..d71095d706 100644 --- a/util-registry/src/test/java/com/twitter/util/registry/BUILD +++ b/util-registry/src/test/java/com/twitter/util/registry/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", From ee04e253bb570ad3869a95dbf40721b20e4d2f84 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Sat, 22 Feb 2020 00:58:59 +0000 Subject: [PATCH 276/796] Twitter-oss: update OSS libraries post-release to 20.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.3.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D438992 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 83741c3d16..39eab041d5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 9f2b4ecdc4..e98d7287b4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.2.0" +val releaseVersion = "20.3.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From f768feb7657cd4708ffb79bc9ad985242b42bc16 Mon Sep 17 00:00:00 2001 From: pliu Date: Sat, 22 Feb 2020 05:02:49 +0000 Subject: [PATCH 277/796] Update Build files - deprecate [rz]globs (batch#4) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K+ BUILD files - replaces [rz]globs. (batch#4) JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D438926 --- util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD | 2 +- util-core/src/test/scala/com/twitter/concurrent/BUILD | 2 +- util-core/src/test/scala/com/twitter/io/BUILD | 2 +- util-hashing/src/test/scala/com/twitter/hashing/BUILD | 2 +- util-lint/src/test/scala/com/twitter/util/lint/BUILD | 2 +- util-logging/src/main/scala/com/twitter/logging/BUILD | 2 +- util-tunable/src/test/scala/com/twitter/util/tunable/BUILD | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index eac4bf2f69..f85bfc8509 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index 6808830495..b24a60bffe 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], # fatal_warnings runs into an issue with scalatest/Future.isDone: # # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 88d823c101..18cfcc7c35 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], # fatal_warnings runs into an issue with scalatest/Future.isDone: # # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` diff --git a/util-hashing/src/test/scala/com/twitter/hashing/BUILD b/util-hashing/src/test/scala/com/twitter/hashing/BUILD index 9fd036514e..7c2793112a 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/test/scala/com/twitter/hashing/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index 76d6e0f76c..74348f4430 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-logging/src/main/scala/com/twitter/logging/BUILD b/util-logging/src/main/scala/com/twitter/logging/BUILD index c9214067a1..b2a4b92dfa 100644 --- a/util-logging/src/main/scala/com/twitter/logging/BUILD +++ b/util-logging/src/main/scala/com/twitter/logging/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD index b6f84bf142..813a604796 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", From 59c490a30bf19918863f7844d19b4a43107e2af2 Mon Sep 17 00:00:00 2001 From: muachilin <32566798+muachilin@users.noreply.github.com> Date: Mon, 24 Feb 2020 23:49:04 +0000 Subject: [PATCH 278/796] util-core: Improve test coverage in StorageUnit Problem Currently the test coverage of StorageUnit still has space for improvements. Solution Add more assertions to test StorageUnit. Result Improve the test coverage of StorageUnit. Signed-off-by: Ian Bennett Differential Revision: https://phabricator.twitter.biz/D439637 --- .../test/scala/com/twitter/util/StorageUnitTest.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala index cbfa0e6b0c..8b77193035 100644 --- a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala @@ -11,6 +11,15 @@ class StorageUnitTest extends FunSuite { assert(1.megabyte.inMegabytes == 1.0) assert(1.gigabyte.inMegabytes == 1024.0) assert(1.gigabyte.inKilobytes == 1024.0 * 1024.0) + assert(1.terabytes.inTerabytes == 1) + assert(1.terabytes.inGigabytes == 1024.0) + assert(1.terabytes.inMegabytes == 1024.0 * 1024.0) + assert(1.petabytes.inPetabytes == 1) + assert(1.petabytes.inTerabytes == 1024.0) + assert(1.petabytes.inGigabytes == 1024.0 * 1024.0) + assert((1.petabytes * 1024.0).inExabytes == 1) + assert((1.petabytes * 1024.0).inPetabytes == 1024.0) + assert((1.petabytes * 1024.0).inTerabytes == 1024.0 * 1024.0) } test("StorageUnit: should confer an essential humanity") { @@ -33,6 +42,7 @@ class StorageUnitTest extends FunSuite { assert(StorageUnit.parse("3.terabytes") == 3.terabytes) assert(StorageUnit.parse("9.petabytes") == 9.petabytes) assert(StorageUnit.parse("-3.megabytes") == -3.megabytes) + assert(StorageUnit.parse("3.exabytes") == 3.petabytes * 1024.0) } test("StorageUnit: should reject soulless robots") { From 10c0bc9811126fee707a0daa33d5b3b532afb099 Mon Sep 17 00:00:00 2001 From: pliu Date: Tue, 25 Feb 2020 02:33:53 +0000 Subject: [PATCH 279/796] Update Build files - deprecate [rz]globs (batch#5) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K+ BUILD files - replaces [rz]globs. (batch#5) JIRA Issues: DPB-12073 NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D439128 --- util-core/src/main/java/com/twitter/service/BUILD | 2 +- util-core/src/test/java/com/twitter/service/BUILD | 2 +- util-core/src/test/java/com/twitter/util/BUILD | 2 +- util-jvm/src/test/java/com/twitter/jvm/BUILD | 2 +- util-registry/src/test/scala/com/twitter/util/registry/BUILD | 2 +- util-tunable/src/test/resources/BUILD | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/util-core/src/main/java/com/twitter/service/BUILD b/util-core/src/main/java/com/twitter/service/BUILD index 45cc835b0c..10e7dfa119 100644 --- a/util-core/src/main/java/com/twitter/service/BUILD +++ b/util-core/src/main/java/com/twitter/service/BUILD @@ -1,5 +1,5 @@ java_library( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, provides = artifact( org = "com.twitter", diff --git a/util-core/src/test/java/com/twitter/service/BUILD b/util-core/src/test/java/com/twitter/service/BUILD index bb47e03bf5..c6780f9939 100644 --- a/util-core/src/test/java/com/twitter/service/BUILD +++ b/util-core/src/test/java/com/twitter/service/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index bee651849b..6c38ab94b9 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -1,6 +1,6 @@ java_library( name = "object_size_calculator", - sources = globs("ObjectSizeCalculator.java"), + sources = ["ObjectSizeCalculator.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", diff --git a/util-jvm/src/test/java/com/twitter/jvm/BUILD b/util-jvm/src/test/java/com/twitter/jvm/BUILD index 767a31f66c..a282525ce0 100644 --- a/util-jvm/src/test/java/com/twitter/jvm/BUILD +++ b/util-jvm/src/test/java/com/twitter/jvm/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-registry/src/test/scala/com/twitter/util/registry/BUILD b/util-registry/src/test/scala/com/twitter/util/registry/BUILD index 3867a21cf1..2643c0082c 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/BUILD +++ b/util-registry/src/test/scala/com/twitter/util/registry/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-tunable/src/test/resources/BUILD b/util-tunable/src/test/resources/BUILD index cc44b5fc88..a001371a96 100644 --- a/util-tunable/src/test/resources/BUILD +++ b/util-tunable/src/test/resources/BUILD @@ -1,3 +1,3 @@ resources( - sources = rglobs("*"), + sources = ["**/*"], ) From a98a8f6a91b33c2eff3e60c1aad45863e444da12 Mon Sep 17 00:00:00 2001 From: pliu Date: Wed, 26 Feb 2020 01:36:45 +0000 Subject: [PATCH 280/796] Update Build files - deprecate [rz]globs (batch#8) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K+ BUILD files - replaces [rz]globs. (batch#8) JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D440569 --- util-app/src/main/scala/BUILD | 2 +- util-app/src/test/java/com/twitter/app/BUILD | 2 +- util-app/src/test/scala/com/twitter/app/BUILD | 2 +- util-benchmark/src/main/scala/BUILD | 2 +- util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD | 2 +- util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD | 2 +- util-cache/src/test/java/com/twitter/cache/BUILD | 2 +- util-codec/src/test/scala/com/twitter/util/BUILD | 2 +- util-core/src/test/java/com/twitter/io/BUILD | 2 +- util-core/src/test/java/com/twitter/util/javainterop/BUILD | 2 +- util-core/src/test/scala/com/twitter/util/BUILD | 2 +- .../src/test/scala/com/twitter/util/capturepoints/BUILD | 2 +- util-logging/src/test/java/com/twitter/logging/BUILD | 2 +- util-logging/src/test/scala/com/twitter/logging/BUILD | 2 +- util-stats/src/test/java/com/twitter/finagle/stats/BUILD | 2 +- util-stats/src/test/scala/com/twitter/finagle/stats/BUILD | 2 +- util-test/src/test/scala/com/twitter/util/testing/BUILD | 2 +- util-thrift/src/test/scala/com/twitter/util/BUILD | 2 +- util-zk-test/src/test/java/com/twitter/zk/BUILD | 2 +- util-zk-test/src/test/scala/com/twitter/zk/BUILD | 2 +- util-zk/src/test/scala/com/twitter/zk/BUILD | 2 +- util-zk/src/test/scala/com/twitter/zk/coordination/BUILD | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index 73997192be..03f4556539 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", diff --git a/util-app/src/test/java/com/twitter/app/BUILD b/util-app/src/test/java/com/twitter/app/BUILD index cbd35024d7..5dbf5bdfe1 100644 --- a/util-app/src/test/java/com/twitter/app/BUILD +++ b/util-app/src/test/java/com/twitter/app/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-app/src/test/scala/com/twitter/app/BUILD b/util-app/src/test/scala/com/twitter/app/BUILD index b26f04ffde..088de4a208 100644 --- a/util-app/src/test/scala/com/twitter/app/BUILD +++ b/util-app/src/test/scala/com/twitter/app/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index 8843c1f265..d325c96201 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD index 73382d8c71..5b90d260c3 100644 --- a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/google/guava", diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD index 6c589a5dc6..a8f0c5da2d 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = rglobs("*.scala"), + sources = ["**/*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/com/google/code/findbugs:jsr305", diff --git a/util-cache/src/test/java/com/twitter/cache/BUILD b/util-cache/src/test/java/com/twitter/cache/BUILD index a67715fc2e..c48082bc18 100644 --- a/util-cache/src/test/java/com/twitter/cache/BUILD +++ b/util-cache/src/test/java/com/twitter/cache/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-codec/src/test/scala/com/twitter/util/BUILD b/util-codec/src/test/scala/com/twitter/util/BUILD index cf725c0052..b8b5baa9da 100644 --- a/util-codec/src/test/scala/com/twitter/util/BUILD +++ b/util-codec/src/test/scala/com/twitter/util/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD index 1e8340fa9d..2ff51af03b 100644 --- a/util-core/src/test/java/com/twitter/io/BUILD +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-core/src/test/java/com/twitter/util/javainterop/BUILD b/util-core/src/test/java/com/twitter/util/javainterop/BUILD index c66493ec25..04f9eeac81 100644 --- a/util-core/src/test/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/test/java/com/twitter/util/javainterop/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index 3d2ebe9e00..a27f7556d4 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], # fatal_warnings runs into an issue with scalatest/Future.isDone: # # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD index fbc6215bc0..4af58246ec 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, strict_deps = True, dependencies = [ diff --git a/util-logging/src/test/java/com/twitter/logging/BUILD b/util-logging/src/test/java/com/twitter/logging/BUILD index aa1d2539b9..6b1c43505c 100644 --- a/util-logging/src/test/java/com/twitter/logging/BUILD +++ b/util-logging/src/test/java/com/twitter/logging/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-logging/src/test/scala/com/twitter/logging/BUILD b/util-logging/src/test/scala/com/twitter/logging/BUILD index 2d6e110096..b41fbadbc2 100644 --- a/util-logging/src/test/scala/com/twitter/logging/BUILD +++ b/util-logging/src/test/scala/com/twitter/logging/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/BUILD b/util-stats/src/test/java/com/twitter/finagle/stats/BUILD index 5a636b8ad8..bee09e61dd 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/test/java/com/twitter/finagle/stats/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD index 7d9e9c982c..8efd337c48 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-test/src/test/scala/com/twitter/util/testing/BUILD b/util-test/src/test/scala/com/twitter/util/testing/BUILD index 14f42b3025..f90a796615 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/test/scala/com/twitter/util/testing/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-thrift/src/test/scala/com/twitter/util/BUILD b/util-thrift/src/test/scala/com/twitter/util/BUILD index 255b83cdb0..bc48263b0b 100644 --- a/util-thrift/src/test/scala/com/twitter/util/BUILD +++ b/util-thrift/src/test/scala/com/twitter/util/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-zk-test/src/test/java/com/twitter/zk/BUILD b/util-zk-test/src/test/java/com/twitter/zk/BUILD index 0fe79b870e..20568f96f6 100644 --- a/util-zk-test/src/test/java/com/twitter/zk/BUILD +++ b/util-zk-test/src/test/java/com/twitter/zk/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.java"), + sources = ["*.java"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-zk-test/src/test/scala/com/twitter/zk/BUILD b/util-zk-test/src/test/scala/com/twitter/zk/BUILD index be60034a67..8c832f4ed0 100644 --- a/util-zk-test/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk-test/src/test/scala/com/twitter/zk/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index c235ad4abc..f6effdd571 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index d190c175cc..1dad482043 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", From b1f2b59d71c494abc7efe550b0f2d8506dacb340 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 26 Feb 2020 17:08:08 +0000 Subject: [PATCH 281/796] Twitter-oss: Prepare OSS libraries for release 20.2.1 Problem We want to release the next versions of our Twitter OSS libraries 20.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9397 Differential Revision: https://phabricator.twitter.biz/D440817 --- CHANGELOG.rst | 2 +- README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 26ed9e1400..1e42571d23 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,7 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -20.2.0 +20.2.1 ------ New Features diff --git a/README.md b/README.md index 39eab041d5..32807c9f27 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.2.0" +val utilCore = "com.twitter" %% "util-core" % "20.2.1" ``` # Units diff --git a/build.sbt b/build.sbt index e98d7287b4..4a91fa8009 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.3.0-SNAPSHOT" +val releaseVersion = "20.2.1" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 23a44a9c0e96477c0ce175f801227774de58250d Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 27 Feb 2020 00:39:58 +0000 Subject: [PATCH 282/796] Twitter-oss: update OSS libraries post-release to 20.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.3.0-SNAPSHOT util scrooge finagle twitter-server finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9397 Differential Revision: https://phabricator.twitter.biz/D441416 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 32807c9f27..6b5afce72d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 4a91fa8009..e98d7287b4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.2.1" +val releaseVersion = "20.3.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 26c51cd6fae79313b4fbf4d1fdbdd04634ed69b0 Mon Sep 17 00:00:00 2001 From: pliu Date: Thu, 27 Feb 2020 01:15:30 +0000 Subject: [PATCH 283/796] Update Build files - deprecate [rz]globs (batch#10) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K+ BUILD files - replaces [rz]globs. (batch#10) JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D441173 --- util-core/BUILD | 8 ++++---- util-core/src/main/scala/com/twitter/io/BUILD | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/util-core/BUILD b/util-core/BUILD index 5fcc7c3bd3..4d74fc6d90 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -18,11 +18,11 @@ target( # com.twitter.concurrent and com.twitter.util scala_library( name = "util-core-util", - sources = globs( - "src/main/scala/com/twitter/util/*.scala", - "src/main/scala-2.12-/com/twitter/util/*.scala", + sources = [ "concurrent-extra/src/main/scala/com/twitter/concurrent/*.scala", - ), + "src/main/scala-2.12-/com/twitter/util/*.scala", + "src/main/scala/com/twitter/util/*.scala", + ], compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", diff --git a/util-core/src/main/scala/com/twitter/io/BUILD b/util-core/src/main/scala/com/twitter/io/BUILD index 5bdb5e3611..92de774949 100644 --- a/util-core/src/main/scala/com/twitter/io/BUILD +++ b/util-core/src/main/scala/com/twitter/io/BUILD @@ -1,7 +1,5 @@ scala_library( - sources = globs( - "*.scala", - ), + sources = ["*.scala"], compiler_option_sets = {"fatal_warnings"}, provides = scala_artifact( org = "com.twitter", From e6c63cb6ffea3e885a733a3adbd31518da4a5ef3 Mon Sep 17 00:00:00 2001 From: pliu Date: Fri, 28 Feb 2020 01:00:40 +0000 Subject: [PATCH 284/796] Update Build files - deprecate [rz]globs (batch#11) `pants 1.25.x` is deprecating `globs()`, `rglobs()`, and `zglobs()` in BUILD files This commit updates about 3K+ BUILD files - replaces [rz]globs. (batch#11) JIRA Issues: DPB-12073 Differential Revision: https://phabricator.twitter.biz/D441822 --- util-app/src/test/resources/BUILD | 9 +++++---- util-cache/src/test/scala/com/twitter/cache/BUILD | 12 ++++-------- util-core/src/test/java/com/twitter/util/BUILD | 6 +++--- util-hashing/src/test/resources/BUILD | 9 +++++---- util-jvm/src/test/scala/com/twitter/jvm/BUILD | 8 +++----- util-security/src/test/resources/BUILD | 9 +++++---- 6 files changed, 25 insertions(+), 28 deletions(-) diff --git a/util-app/src/test/resources/BUILD b/util-app/src/test/resources/BUILD index e0afa86adb..45a90181cc 100644 --- a/util-app/src/test/resources/BUILD +++ b/util-app/src/test/resources/BUILD @@ -1,6 +1,7 @@ resources( - sources = rglobs( - "*", - exclude = [rglobs("BUILD*", "*.pyc")], - ), + sources = [ + "!**/*.pyc", + "!BUILD*", + "**/*", + ], ) diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index f6ac12a6b4..48c3388627 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -1,13 +1,9 @@ junit_tests( - sources = globs( + sources = [ + "!AbstractFutureCacheTest.scala", + "!AbstractLoadingFutureCacheTest.scala", "*.scala", - exclude = [ - globs( - "AbstractFutureCacheTest.scala", - "AbstractLoadingFutureCacheTest.scala", - ), - ], - ), + ], compiler_option_sets = {"fatal_warnings"}, dependencies = [ ":abstract_tests", diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index 6c38ab94b9..dad78b3a3c 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -8,10 +8,10 @@ java_library( ) junit_tests( - sources = globs( + sources = [ + "!ObjectSizeCalculator.java", "*.java", - exclude = [globs("ObjectSizeCalculator.java")], - ), + ], compiler_option_sets = {"fatal_warnings"}, dependencies = [ ":object_size_calculator", diff --git a/util-hashing/src/test/resources/BUILD b/util-hashing/src/test/resources/BUILD index e0afa86adb..45a90181cc 100644 --- a/util-hashing/src/test/resources/BUILD +++ b/util-hashing/src/test/resources/BUILD @@ -1,6 +1,7 @@ resources( - sources = rglobs( - "*", - exclude = [rglobs("BUILD*", "*.pyc")], - ), + sources = [ + "!**/*.pyc", + "!BUILD*", + "**/*", + ], ) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/BUILD b/util-jvm/src/test/scala/com/twitter/jvm/BUILD index 6150287f3e..b6029ae29e 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/test/scala/com/twitter/jvm/BUILD @@ -1,10 +1,8 @@ junit_tests( - sources = globs( + sources = [ + "!EstimatorApp.scala", "*.scala", - exclude = [ - globs("EstimatorApp.scala"), - ], - ), + ], compiler_option_sets = {"fatal_warnings"}, dependencies = [ "3rdparty/jvm/junit", diff --git a/util-security/src/test/resources/BUILD b/util-security/src/test/resources/BUILD index e0afa86adb..45a90181cc 100644 --- a/util-security/src/test/resources/BUILD +++ b/util-security/src/test/resources/BUILD @@ -1,6 +1,7 @@ resources( - sources = rglobs( - "*", - exclude = [rglobs("BUILD*", "*.pyc")], - ), + sources = [ + "!**/*.pyc", + "!BUILD*", + "**/*", + ], ) From 3db2808ebbdbd8b2261d8a8d103c74cc3311d40c Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 28 Feb 2020 22:40:33 +0000 Subject: [PATCH 285/796] csl: Update build.sbt configurations to use overWriting Problem When we try to resume a failed release, if it fails midway through publishing a library, it's unable to continue because it gets stuck in a retry loop trying to republish JARs. Solution We should enable overwriting so that we're able to resume publishing successfully. JIRA Issues: CSL-9476 Differential Revision: https://phabricator.twitter.biz/D442363 --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index e98d7287b4..e8b7745f38 100644 --- a/build.sbt +++ b/build.sbt @@ -143,6 +143,7 @@ val baseSettings = Seq( publishArtifact in Test := false, pomIncludeRepository := { _ => false }, publishMavenStyle := true, + publishConfiguration := publishConfiguration.value.withOverwrite(true), autoAPIMappings := true, apiURL := Some(url("https://twitter.github.io/util/docs/")), pomExtra := From da6389421142b0d0f756abbf4508edf02812b9d9 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 2 Mar 2020 20:08:50 +0000 Subject: [PATCH 286/796] util-core: Remove configurability of Local state on Promise interrupts Problem We have been using the Local state present when an interrupt handler was set when a Promise has been raised on for some time now without issue. Solution Remove the associated code. JIRA Issues: CSL-9421, PCM-110275 Differential Revision: https://phabricator.twitter.biz/D442444 --- CHANGELOG.rst | 13 ++++++++ .../main/scala/com/twitter/util/Promise.scala | 24 +++----------- .../scala/com/twitter/util/PromiseTest.scala | 31 ------------------- 3 files changed, 18 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1e42571d23..481a7c01f0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,19 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: The system property `com.twitter.util.UseLocalInInterruptible` no longer + can be used to modulate which Local state is present when a Promise is interrupted. + ``PHAB_ID=D442444`` + +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-core: Promises now exclusively use the state local to setting the interrupt + handler when raising on a Promise. ``PHAB_ID=D442444`` + 20.2.1 ------ diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index bd193dbf5c..0a43527d92 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -7,16 +7,6 @@ import scala.util.control.NonFatal object Promise { - // An experimental property that enables the interrupt handler - // to use the state at the time of callback creation. - @volatile private var UseLocalInInterruptible: Boolean = - System.getProperty("com.twitter.util.UseLocalInInterruptible", "false").toBoolean - - // Switches to enabling the interrupt handler to use - // the state at the time of callback creation. - private[twitter] def useLocalInInterruptible(useLocal: Boolean): Unit = - UseLocalInInterruptible = useLocal - /** * Embeds an "interrupt handler" into a [[Promise]]. * @@ -579,15 +569,11 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ case s: Interruptible[A] => if (!cas(s, new Interrupted(s.waitq, intr))) raise(intr) else { - if (!UseLocalInInterruptible) - s.handler.applyOrElse(intr, Promise.AlwaysUnit) - else { - val current = Local.save() - if (current ne s.saved) - Local.restore(s.saved) - try s.handler.applyOrElse(intr, Promise.AlwaysUnit) - finally Local.restore(current) - } + val current = Local.save() + if (current ne s.saved) + Local.restore(s.saved) + try s.handler.applyOrElse(intr, Promise.AlwaysUnit) + finally Local.restore(current) } case s: Transforming[A] => diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index 79dc59ea41..46c302a8e5 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -220,8 +220,6 @@ class PromiseTest extends FunSuite { } test("interrupts use local state at the time of callback creation") { - Promise.useLocalInInterruptible(true) - val local = new Local[String] implicit val timer: Timer = new JavaTimer(true) @@ -246,33 +244,6 @@ class PromiseTest extends FunSuite { assert("main thread" == Await.result(f, 3.seconds)) } - test("interrupts use raisers state") { - Promise.useLocalInInterruptible(false) - - val local = new Local[String] - implicit val timer: Timer = new JavaTimer(true) - - local.set(Some("main thread")) - val p = new Promise[String] - - p.setInterruptHandler({ - case _ => - val localVal = local().getOrElse("") - p.updateIfEmpty(Return(localVal)) - }) - val f = Future.sleep(1.second).before(p) - - val thread = new Thread(new Runnable { - def run(): Unit = { - local.set(Some("worker thread")) - p.raise(new Exception) - } - }) - thread.start() - - assert("worker thread" == Await.result(f, 3.seconds)) - } - test("interrupt then map uses local state at the time of callback creation") { val local = new Local[String] implicit val timer: Timer = new JavaTimer(true) @@ -302,8 +273,6 @@ class PromiseTest extends FunSuite { } test("locals in Interruptible after detatch") { - Promise.useLocalInInterruptible(true) - val local = new Local[String] implicit val timer: Timer = new JavaTimer(true) From 18b91636784702fa0e4c73deef0c90e14e276629 Mon Sep 17 00:00:00 2001 From: Pratik Tandel Date: Mon, 2 Mar 2020 23:12:39 +0000 Subject: [PATCH 287/796] Upgrade guava to 23.6.1-jre **Overview** This ugprades guava to version `23.6.1-jre`. This is not the latest version but upgrading to newer versions leads to more breakages and this will still give us incremental upgrade benefits. Notably this guava version fixes a DoS attack that's prevalent in guava since v11 Multiple backwards compatible fixes have already been committed to source: D379142 D372978 D372978 We had to revert a previous attempt of this due to hadoop jobs loading incompatible guava jars. This has been remediated with AITOOLS-5695 which has been landed. We will wait until next week to allow all jobs to pickup these options in case these new hadoop options introduce their own set of bugs. Differential Revision: https://phabricator.twitter.biz/D372886 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index e8b7745f38..568bef93a7 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,7 @@ val slf4jVersion = "1.7.30" val jacksonVersion = "2.9.10" val jacksonDatabindVersion = "2.9.10.1" -val guavaLib = "com.google.guava" % "guava" % "19.0" +val guavaLib = "com.google.guava" % "guava" % "23.6.1-jre" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" From 533a40fbe624ef460cf8f5980d6049ec1496b400 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 3 Mar 2020 23:44:18 +0000 Subject: [PATCH 288/796] util|scrooge|finagle: Update to Scala 2.13.1 Problem/Solution Update util, scrooge, and finagle to all cross build with Scala 2.13.1. We need to also update the `scoverage` plugin here to deal with an issue where plugins need to be specified to be fully cross versioned (that is published with their full scala version, e.g., `2.13.0` instead of just `2.13`) because Scala is not binary compatible between minor releases and something changed between 2.13.0 and 2.13.1. The `scoverage` plugin version `1.6.1` appears to work properly. Differential Revision: https://phabricator.twitter.biz/D444211 --- .travis.yml | 2 +- build.sbt | 2 +- project/plugins.sbt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8acca293f8..f932960712 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_cache: scala: - 2.11.12 - 2.12.8 - - 2.13.0 + - 2.13.1 jdk: - openjdk8 diff --git a/build.sbt b/build.sbt index 568bef93a7..8c4e2059d5 100644 --- a/build.sbt +++ b/build.sbt @@ -77,7 +77,7 @@ def jdk11GcJavaOptions: Seq[String] = { val defaultProjectSettings = Seq( scalaVersion := "2.12.8", - crossScalaVersions := Seq("2.11.12", "2.12.8", "2.13.0") + crossScalaVersions := Seq("2.11.12", "2.12.8", "2.13.1") ) val baseSettings = Seq( diff --git a/project/plugins.sbt b/project/plugins.sbt index d83a672ad0..f7e8c4cd8b 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,5 +3,5 @@ resolvers += Resolver.sonatypeRepo("snapshots") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.0") addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC13") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.0") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.4") From 2b0b3dc040d3ca26c1bfdedf977e78dddd9fccb7 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 6 Mar 2020 17:34:25 +0000 Subject: [PATCH 289/796] Twitter-oss: Prepare OSS libraries for release 20.3.0 Problem We want to release the next versions of our Twitter OSS libraries 20.3.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D446094 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 481a7c01f0..82332c11b3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.3.0 +------ + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 6b5afce72d..4d14dbabcb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.2.1" +val utilCore = "com.twitter" %% "util-core" % "20.3.0" ``` # Units diff --git a/build.sbt b/build.sbt index 8c4e2059d5..be6d082f6c 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.3.0-SNAPSHOT" +val releaseVersion = "20.3.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 36c56e80e1e628d79a153f3cc47658ea99dfca16 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 7 Mar 2020 18:47:54 +0000 Subject: [PATCH 290/796] Twitter-oss: update OSS libraries post-release to 20.4.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.4.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D446383 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4d14dbabcb..26c94519fb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index be6d082f6c..2c9869c108 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.3.0" +val releaseVersion = "20.4.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 7d5e9ff53d9552a5385e7082401af73064cc3fb2 Mon Sep 17 00:00:00 2001 From: Pratik Tandel Date: Mon, 9 Mar 2020 21:50:55 +0000 Subject: [PATCH 291/796] Upgrade guava to 25.1-jre - Now that master is on guava 23.6-1-jre. - This incrementally upgrades to 25.1-jre (lowest set of changes needed to pass master) Differential Revision: https://phabricator.twitter.biz/D443757 --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 2c9869c108..ff593008fd 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,7 @@ val slf4jVersion = "1.7.30" val jacksonVersion = "2.9.10" val jacksonDatabindVersion = "2.9.10.1" -val guavaLib = "com.google.guava" % "guava" % "23.6.1-jre" +val guavaLib = "com.google.guava" % "guava" % "25.1-jre" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" @@ -480,7 +480,7 @@ lazy val utilThrift = Project( "org.apache.thrift" % "libthrift" % "0.10.0", slf4jApi % "provided", "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion ) ).dependsOn(utilCodec) From 41227fa84087375ae8d1def4c04eadaba64f30e5 Mon Sep 17 00:00:00 2001 From: muachilin <32566798+muachilin@users.noreply.github.com> Date: Mon, 9 Mar 2020 23:58:20 +0000 Subject: [PATCH 292/796] util: Add more tests for Cancellable Signed-off-by: Ryan O'Neill JIRA Issues: CSL-9502 Differential Revision: https://phabricator.twitter.biz/D447374 --- .../com/twitter/util/CancellableTest.scala | 77 ++++++++++++++++--- 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala index d64ec4512a..8e139d6fa3 100644 --- a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala @@ -1,17 +1,70 @@ package com.twitter.util -import org.scalatest.WordSpec - -class CancellableTest extends WordSpec { - "CancellableSink" should { - "cancel once" in { - var count = 0 - def increment: Unit = count += 1 - val s = new CancellableSink(increment) - s.cancel() - assert(count == 1) - s.cancel() - assert(count == 1) +import org.scalatest.FunSuite + +class CancellableTest extends FunSuite { + + test("Cancellable.nil is always cancelled") { + assert(!Cancellable.nil.isCancelled) + } + + test("Cancellable.nil cancel method") { + Cancellable.nil.cancel() + assert(!Cancellable.nil.isCancelled) + } + + test("Cancellable.nil linkTo method") { + Cancellable.nil.linkTo(Cancellable.nil) + assert(!Cancellable.nil.isCancelled) + } + + test("CancellableSink cancel normal case") { + var count = 2 + def square: Unit = count *= count + val s = new CancellableSink(square) + assert(count == 2) + s.cancel() + assert(count == 4) + } + + test("CancellableSink cancel once") { + var count = 0 + def increment: Unit = count += 1 + val s = new CancellableSink(increment) + s.cancel() + assert(count == 1) + s.cancel() + assert(count == 1) + } + + test("CancellableSink has not cancelled") { + var count = 0 + def increment: Unit = count += 1 + val s = new CancellableSink(increment) + assert(!s.isCancelled) + assert(count == 0) + } + + test("CancellableSink confirm cancelled") { + var count = 0 + def increment: Unit = count += 1 + val s = new CancellableSink(increment) + s.cancel() + assert(s.isCancelled) + assert(count == 1) + } + + test("CancellableSink not support linking") { + var count = 1 + def multiply: Unit = count *= 2 + val s1 = new CancellableSink(multiply) + val s2 = new CancellableSink(multiply) + assertThrows[Exception] { + s1.linkTo(s2) + } + assertThrows[Exception] { + s2.linkTo(s1) } } + } From 83145aae07485ff2b9dfbebd231f5cf90ea7fd82 Mon Sep 17 00:00:00 2001 From: muachilin <32566798+muachilin@users.noreply.github.com> Date: Wed, 11 Mar 2020 21:09:02 +0000 Subject: [PATCH 293/796] util: Move MonitorTest from WordSpec to FunSuite Signed-off-by: Ryan O'Neill JIRA Issues: CSL-9512 Differential Revision: https://phabricator.twitter.biz/D448511 --- .../scala/com/twitter/util/MonitorTest.scala | 249 +++++++++--------- 1 file changed, 121 insertions(+), 128 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala index 1a5a5acba6..15ed8db4bd 100644 --- a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala @@ -2,7 +2,7 @@ package com.twitter.util import org.mockito.Matchers._ import org.mockito.Mockito._ -import org.scalatest.WordSpec +import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar object MonitorTest { @@ -11,128 +11,124 @@ object MonitorTest { } } -class MonitorTest extends WordSpec with MockitoSugar { +class MonitorTest extends FunSuite with MockitoSugar { import MonitorTest._ - "Monitor#orElse" should { - class MonitorOrElseHelper { - val m0, m1, m2 = spy(new MockMonitor()) - Seq(m0, m1, m2).foreach { m => - when(m.handle(any[Throwable])).thenReturn(true) - } - val exc = new Exception - val m = m0.orElse(m1).orElse(m2) + class MonitorOrElseHelper { + val m0, m1, m2 = spy(new MockMonitor()) + Seq(m0, m1, m2).foreach { m => + when(m.handle(any[Throwable])).thenReturn(true) } + val exc = new Exception + val m = m0.orElse(m1).orElse(m2) + } - "stop at first successful handle" in { - val h = new MonitorOrElseHelper - import h._ + test("Monitor#orElse should stop at first successful handle") { + val h = new MonitorOrElseHelper + import h._ - assert(m.handle(exc)) + assert(m.handle(exc)) - verify(m0).handle(exc) - verify(m1, never()).handle(exc) - verify(m2, never()).handle(exc) + verify(m0).handle(exc) + verify(m1, never()).handle(exc) + verify(m2, never()).handle(exc) - when(m0.handle(any[Throwable])).thenReturn(false) + when(m0.handle(any[Throwable])).thenReturn(false) - assert(m.handle(exc)) - verify(m0, times(2)).handle(exc) - verify(m1).handle(exc) - verify(m2, never()).handle(exc) - } + assert(m.handle(exc)) + verify(m0, times(2)).handle(exc) + verify(m1).handle(exc) + verify(m2, never()).handle(exc) + } - "fail when no nothing got handled" in { - val h = new MonitorOrElseHelper - import h._ + test("Monitor#orElse should fail when no nothing got handled") { + val h = new MonitorOrElseHelper + import h._ - Seq(m0, m1, m2) foreach { m => - when(m.handle(any[Throwable])).thenReturn(false) - } - assert(!m.handle(exc)) - Seq(m0, m1, m2) foreach { m => - verify(m).handle(exc) - } + Seq(m0, m1, m2) foreach { m => + when(m.handle(any[Throwable])).thenReturn(false) } + assert(!m.handle(exc)) + Seq(m0, m1, m2) foreach { m => + verify(m).handle(exc) + } + } - "wrap Monitor exceptions and pass them on" in { - val h = new MonitorOrElseHelper - import h._ + test("Monitor#orElse should wrap Monitor exceptions and pass them on") { + val h = new MonitorOrElseHelper + import h._ - val rte = new RuntimeException("really bad news") - when(m0.handle(any[Throwable])).thenThrow(rte) + val rte = new RuntimeException("really bad news") + when(m0.handle(any[Throwable])).thenThrow(rte) - assert(m.handle(exc)) - verify(m0).handle(exc) - verify(m1).handle(MonitorException(exc, rte)) - } + assert(m.handle(exc)) + verify(m0).handle(exc) + verify(m1).handle(MonitorException(exc, rte)) } - "Monitor#andThen" should { - class MonitorAndThenHelper { - val m0, m1 = spy(new MockMonitor) - when(m0.handle(any[Throwable])).thenReturn(true) - when(m1.handle(any[Throwable])).thenReturn(true) - val m = m0.andThen(m1) - val exc = new Exception - } + class MonitorAndThenHelper { + val m0, m1 = spy(new MockMonitor) + when(m0.handle(any[Throwable])).thenReturn(true) + when(m1.handle(any[Throwable])).thenReturn(true) + val m = m0.andThen(m1) + val exc = new Exception + } - "run all monitors" in { - val h = new MonitorAndThenHelper - import h._ + test("Monitor#andThen should run all monitors") { + val h = new MonitorAndThenHelper + import h._ - when(m0.handle(any[Throwable])).thenReturn(true) - when(m1.handle(any[Throwable])).thenReturn(true) + when(m0.handle(any[Throwable])).thenReturn(true) + when(m1.handle(any[Throwable])).thenReturn(true) - assert(m.handle(exc)) - verify(m0).handle(exc) - verify(m1).handle(exc) - } + assert(m.handle(exc)) + verify(m0).handle(exc) + verify(m1).handle(exc) + } - "be succcessful when any underlying monitor is" in { - val h = new MonitorAndThenHelper - import h._ + test("Monitor#andThen should be succcessful when any underlying monitor is") { + val h = new MonitorAndThenHelper + import h._ - when(m0.handle(any[Throwable])).thenReturn(false) - assert(m.handle(exc)) - when(m1.handle(any[Throwable])).thenReturn(false) - assert(!m.handle(exc)) - } + when(m0.handle(any[Throwable])).thenReturn(false) + assert(m.handle(exc)) + when(m1.handle(any[Throwable])).thenReturn(false) + assert(!m.handle(exc)) + } - "wrap Monitor exceptions and pass them on" in { - val h = new MonitorAndThenHelper - import h._ + test("Monitor#andThen should wrap Monitor exceptions and pass them on") { + val h = new MonitorAndThenHelper + import h._ - val rte = new RuntimeException("really bad news") - when(m0.handle(any[Throwable])).thenThrow(rte) + val rte = new RuntimeException("really bad news") + when(m0.handle(any[Throwable])).thenThrow(rte) - assert(m.handle(exc)) - verify(m0).handle(exc) - verify(m1).handle(MonitorException(exc, rte)) - } + assert(m.handle(exc)) + verify(m0).handle(exc) + verify(m1).handle(MonitorException(exc, rte)) + } - "fail if both monitors throw" in { - val h = new MonitorAndThenHelper - import h._ + test("Monitor#andThen should fail if both monitors throw") { + val h = new MonitorAndThenHelper + import h._ - val rte = new RuntimeException("really bad news") - when(m0.handle(any[Throwable])).thenThrow(rte) - when(m1.handle(any[Throwable])).thenThrow(rte) + val rte = new RuntimeException("really bad news") + when(m0.handle(any[Throwable])).thenThrow(rte) + when(m1.handle(any[Throwable])).thenThrow(rte) - assert(!m.handle(exc)) - } + assert(!m.handle(exc)) } - "Monitor.get, Monitor.set()" should { + test("Monitor.get, Monitor.set() should maintain current monitor") { val m = new MockMonitor - "maintain current monitor" in Monitor.restoring { + Monitor.restoring { Monitor.set(m) assert(Monitor.get == m) } } - "Monitor.getOption, Monitor.setOption" should { - "maintain current monitor" in Monitor.restoring { + test("Monitor.getOption, Monitor.setOption should maintain current monitor") { + Monitor.restoring { val mon = new Monitor { def handle(cause: Throwable): Boolean = true } @@ -140,8 +136,10 @@ class MonitorTest extends WordSpec with MockitoSugar { assert(Monitor.getOption.contains(mon)) assert(mon == Monitor.get) } + } - "setOption of None restores the default" in Monitor.restoring { + test("Monitor.getOption, Monitor.setOption should setOption of None restores the default") { + Monitor.restoring { val mon = new Monitor { def handle(cause: Throwable): Boolean = true } @@ -154,10 +152,9 @@ class MonitorTest extends WordSpec with MockitoSugar { } } - "Monitor.handle" should { + test("Monitor.handle should dispatch to current monitor") { val m = spy(new MockMonitor) - - "dispatch to current monitor" in Monitor.restoring { + Monitor.restoring { when(m.handle(any[Throwable])).thenReturn(true) val exc = new Exception Monitor.set(m) @@ -166,52 +163,48 @@ class MonitorTest extends WordSpec with MockitoSugar { } } - "Monitor.restore" should { - "restore current configuration" in { - val orig = Monitor.get - Monitor.restoring { - Monitor.set(mock[Monitor]) - } - assert(Monitor.get == orig) + test("Monitor.restore should restore current configuration") { + val orig = Monitor.get + Monitor.restoring { + Monitor.set(mock[Monitor]) } + assert(Monitor.get == orig) } - "Monitor.mk" should { - class MonitorMkHelper { - class E1 extends Exception - class E2 extends E1 - class F1 extends Exception + class MonitorMkHelper { + class E1 extends Exception + class E2 extends E1 + class F1 extends Exception - var ran = false - val m = Monitor.mk { - case _: E1 => - ran = true - true - } + var ran = false + val m = Monitor.mk { + case _: E1 => + ran = true + true } + } - "handle E1" in { - val h = new MonitorMkHelper - import h._ + test("Monitor.mk should handle E1") { + val h = new MonitorMkHelper + import h._ - assert(m.handle(new E1)) - assert(ran) - } + assert(m.handle(new E1)) + assert(ran) + } - "handle E2" in { - val h = new MonitorMkHelper - import h._ + test("Monitor.mk should handle E2") { + val h = new MonitorMkHelper + import h._ - assert(m.handle(new E2)) - assert(ran) - } + assert(m.handle(new E2)) + assert(ran) + } - "not handle F1" in { - val h = new MonitorMkHelper - import h._ + test("Monitor.mk should handle F1") { + val h = new MonitorMkHelper + import h._ - assert(!m.handle(new F1)) - assert(!ran) - } + assert(!m.handle(new F1)) + assert(!ran) } } From f595ed971acacd2ccac84a49053e6b737cff860d Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 17 Mar 2020 02:16:43 +0000 Subject: [PATCH 294/796] finagle-partitioning: Rename Ketama to Hash Problem This was brought up in p-client design review, the hash node and hash key in the consistent hashing are named Ketama. However, the hashing algorithm can be configured by providing different keyHasher (for example keyHasher.MURMUR3). Solution Rename some Ketama related hash variable to Hash. `KetamaClientKey` => `HashNodeKey` `KetamaNode` => `HashNode` `KetamaFailureAccuralFactory` => `HashRingFailureAccuralFactory` `KetamaDistributor` => `HashRingDistributor` JIRA Issues: CSL-9136 Differential Revision: https://phabricator.twitter.biz/D449929 --- CHANGELOG.rst | 7 +++++ .../hashing/KetamaDistributorBench.scala | 7 ++--- ...ala => ConsistentHashingDistributor.scala} | 22 ++++++++-------- ...=> ConsistentHashingDistributorTest.scala} | 26 +++++++++---------- .../com/twitter/hashing/DiagnosticsTest.scala | 2 +- 5 files changed, 36 insertions(+), 28 deletions(-) rename util-hashing/src/main/scala/com/twitter/hashing/{KetamaDistributor.scala => ConsistentHashingDistributor.scala} (87%) rename util-hashing/src/test/scala/com/twitter/hashing/{KetamaDistributorTest.scala => ConsistentHashingDistributorTest.scala} (84%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 82332c11b3..ebadd1d3ce 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ +* util-hashing: Rename + `c.t.hashing.KetamaNode` => `HashNode`, + `c.t.hashing.KetamaDistributor` => `ConsistentHashingDistributor`. + ``PHAB_ID=D449929`` + 20.3.0 ------ diff --git a/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala b/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala index 5c7ef5b75b..de524d6f89 100644 --- a/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala +++ b/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala @@ -19,10 +19,11 @@ class KetamaDistributorBench extends StdBenchAnnotations { val hosts: scala.collection.immutable.IndexedSeq[String] = 0.until(numHosts).map { i => "abcd-zzz-" + i + "-ab2.qwer.twitter.com" } - val nodes: scala.collection.immutable.IndexedSeq[KetamaNode[String]] = hosts map { h => - KetamaNode[String](h + ":12021", 1, h) + val nodes: scala.collection.immutable.IndexedSeq[HashNode[String]] = hosts map { h => + HashNode[String](h + ":12021", 1, h) } @Benchmark - def ketama(): KetamaDistributor[String] = new KetamaDistributor(nodes, numReps, false) + def ketama(): ConsistentHashingDistributor[String] = + new ConsistentHashingDistributor(nodes, numReps, false) } diff --git a/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala b/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala similarity index 87% rename from util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala rename to util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala index f59dc5dc9c..a7ab923bdf 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/KetamaDistributor.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala @@ -2,7 +2,7 @@ package com.twitter.hashing import java.security.MessageDigest -case class KetamaNode[A](identifier: String, weight: Int, handle: A) +case class HashNode[A](identifier: String, weight: Int, handle: A) /** * @note Certain versions of libmemcached return subtly different results for points on @@ -10,8 +10,8 @@ case class KetamaNode[A](identifier: String, weight: Int, handle: A) * clients who depend on those versions of libmemcached, we have to reproduce their result. * If the `oldLibMemcachedVersionComplianceMode` is `true` the behavior will be reproduced. */ -class KetamaDistributor[A]( - ketamaNodes: Seq[KetamaNode[A]], +class ConsistentHashingDistributor[A]( + hashNodes: Seq[HashNode[A]], numReps: Int, oldLibMemcachedVersionComplianceMode: Boolean = false) extends Distributor[A] { @@ -53,15 +53,15 @@ class KetamaDistributor[A]( md.update(('0' + j).toByte) } - private[this] val continuum: java.util.TreeMap[Long, KetamaNode[A]] = { + private[this] val continuum: java.util.TreeMap[Long, HashNode[A]] = { val md5 = MessageDigest.getInstance("MD5") - val underlying = new java.util.TreeMap[Long, KetamaNode[A]]() + val underlying = new java.util.TreeMap[Long, HashNode[A]]() val dash: Byte = '-'.toByte - val nodeCount = ketamaNodes.size - val totalWeight = ketamaNodes.foldLeft(0) { _ + _.weight } + val nodeCount = hashNodes.size + val totalWeight = hashNodes.foldLeft(0) { _ + _.weight } - val it = ketamaNodes.iterator + val it = hashNodes.iterator while (it.hasNext) { val node = it.next() @@ -101,14 +101,14 @@ class KetamaDistributor[A]( underlying } - def nodes: Seq[A] = ketamaNodes.map(_.handle) - def nodeCount: Int = ketamaNodes.size + def nodes: Seq[A] = hashNodes.map(_.handle) + def nodeCount: Int = hashNodes.size // hashes are 32-bit because they are 32-bit on the libmemcached and // we need to maintain compatibility with libmemcached private[this] def truncateHash(hash: Long): Long = hash & 0xffffffffL - private def mapEntryForHash(hash: Long): java.util.Map.Entry[Long, KetamaNode[A]] = { + private def mapEntryForHash(hash: Long): java.util.Map.Entry[Long, HashNode[A]] = { val truncatedHash = truncateHash(hash) val entry = continuum.ceilingEntry(truncatedHash) if (entry == null) continuum.firstEntry else entry diff --git a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala similarity index 84% rename from util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala rename to util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala index bac185e0fe..20b74c0feb 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/KetamaDistributorTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala @@ -8,23 +8,23 @@ import java.io.{BufferedReader, InputStreamReader} import java.nio.{ByteBuffer, ByteOrder} import java.security.MessageDigest -class KetamaDistributorTest extends WordSpec with ScalaCheckDrivenPropertyChecks { +class ConsistentHashingDistributorTest extends WordSpec with ScalaCheckDrivenPropertyChecks { "KetamaDistributor" should { val nodes = Seq( - KetamaNode("10.0.1.1", 600, 1), - KetamaNode("10.0.1.2", 300, 2), - KetamaNode("10.0.1.3", 200, 3), - KetamaNode("10.0.1.4", 350, 4), - KetamaNode("10.0.1.5", 1000, 5), - KetamaNode("10.0.1.6", 800, 6), - KetamaNode("10.0.1.7", 950, 7), - KetamaNode("10.0.1.8", 100, 8) + HashNode("10.0.1.1", 600, 1), + HashNode("10.0.1.2", 300, 2), + HashNode("10.0.1.3", 200, 3), + HashNode("10.0.1.4", 350, 4), + HashNode("10.0.1.5", 1000, 5), + HashNode("10.0.1.6", 800, 6), + HashNode("10.0.1.7", 950, 7), + HashNode("10.0.1.8", 100, 8) ) // 160 is the hard coded value for libmemcached, which was this input data is from - val ketamaDistributor = new KetamaDistributor(nodes, 160) + val ketamaDistributor = new ConsistentHashingDistributor(nodes, 160) val ketamaDistributorInoldLibMemcachedVersionComplianceMode = - new KetamaDistributor(nodes, 160, true) + new ConsistentHashingDistributor(nodes, 160, true) "pick the correct node with ketama hash function" in { // Test from Smile's KetamaNodeLocatorSpec.scala @@ -77,7 +77,7 @@ class KetamaDistributorTest extends WordSpec with ScalaCheckDrivenPropertyChecks } "hashInt" in { - val ketama = new KetamaDistributor[Unit](Seq.empty, 0, false) + val ketama = new ConsistentHashingDistributor[Unit](Seq.empty, 0, false) forAll(Gen.chooseNum(0, Int.MaxValue)) { i => val md = MessageDigest.getInstance("MD5") ketama.hashInt(i, md) @@ -87,7 +87,7 @@ class KetamaDistributorTest extends WordSpec with ScalaCheckDrivenPropertyChecks } "byteArrayToLE" in { - val ketama = new KetamaDistributor[Unit](Seq.empty, 0, false) + val ketama = new ConsistentHashingDistributor[Unit](Seq.empty, 0, false) forAll { s: String => val ba = MessageDigest.getInstance("MD5").digest(s.getBytes("UTF-8")) val bb = ByteBuffer.wrap(ba).order(ByteOrder.LITTLE_ENDIAN) diff --git a/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala index f5777a7c0d..7f806b4913 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala @@ -10,7 +10,7 @@ class DiagnosticsTest extends WordSpec { val nodes = hosts.map { s => val Array(host, port, weight) = s.split(":") val identifier = host + ":" + port - KetamaNode(identifier, weight.toInt, identifier) + HashNode(identifier, weight.toInt, identifier) } val hashFunctions = List( From 176162c3d382279b70e385a2cfd57fb472bfecb7 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Tue, 17 Mar 2020 19:14:35 +0000 Subject: [PATCH 295/796] twitter-service: add metric_metadata endpoint to admin server Problem Finagle stats are currently storing but not exposing Metric Schemas containing useful metadata about metrics. Solution Create an endpoint (/admin/metric_metadata.json) which exposes this metadata in a JSON format, and which allows for metrics to be filtered by name with a query param "name" ala "m" in /admin/metrics.json. Result Users and tools can now query the new /admin/metric_metadata.json endpoint to obtain the stored metadata about a metric. JIRA Issues: CSL-8921 Differential Revision: https://phabricator.twitter.biz/D449149 --- .../scala/com/twitter/finagle/stats/BUILD | 5 ++ .../twitter/finagle/stats/MetricBuilder.scala | 46 +++++++------------ .../twitter/finagle/stats/MetricSchema.scala | 10 ++-- .../twitter/finagle/stats/MetricUnit.scala | 4 +- .../finagle/stats/SchemaRegistry.scala | 13 ++++++ 5 files changed, 43 insertions(+), 35 deletions(-) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD index 5a230c555e..61e9b8ad0d 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD @@ -6,6 +6,11 @@ scala_library( repo = artifactory, ), dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson:jackson-module-scala", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/datatype:jackson-datatype-joda", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/com/google/code/findbugs:jsr305", "util/util-core/src/main/scala/com/twitter/util", diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 8068c20b0a..395721d812 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -22,23 +22,21 @@ case object Server extends SourceRole * @param units the unit associated with the metrics value (milliseconds, megabytes, requests, etc) * @param role whether the service is playing the part of client or server regarding this metric * @param verbosity see StatsReceiver for details - * @param sourceClass the class pass of the module which generated this metric (ie, com.twitter...) - * @param sourceLibrary the name of the library which generated this metric (ie, finagle) + * @param sourceClass the name of the class which generated this metric (ie, com.twitter.finagle.StatsFilter) * @param name the relative metric name which will be appended to the scope of the StatsReceiver prior to long term storage - * @param identifier the service name used by obs stack for this service + * @param processPath a universal coordinate for the resource * @param percentiles used to indicate buckets for histograms, to be set by the StatsReceiver * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder */ class MetricBuilder( val keyIndicator: Boolean = false, - val description: String = "", + val description: String = "No description provided", val units: MetricUnit = Unspecified, val role: SourceRole = NoRoleSpecified, val verbosity: Verbosity = Verbosity.Default, - val sourceClass: String = "", - val sourceLibrary: Option[String] = None, + val sourceClass: Option[String] = None, val name: Seq[String] = Seq(), - val identifier: String = "", + val processPath: Option[String] = None, // Only persisted and relevant when building histograms. val percentiles: IndexedSeq[Double] = IndexedSeq(), val statsReceiver: StatsReceiver) { @@ -54,10 +52,9 @@ class MetricBuilder( units: MetricUnit = this.units, role: SourceRole = this.role, verbosity: Verbosity = this.verbosity, - sourceClass: String = this.sourceClass, - sourceLibrary: Option[String] = this.sourceLibrary, + sourceClass: Option[String] = this.sourceClass, name: Seq[String] = this.name, - identifier: String = this.identifier, + processPath: Option[String] = this.processPath, percentiles: IndexedSeq[Double] = this.percentiles ): MetricBuilder = { new MetricBuilder( @@ -67,9 +64,8 @@ class MetricBuilder( role = role, verbosity = verbosity, sourceClass = sourceClass, - sourceLibrary = sourceLibrary, name = name, - identifier = identifier, + processPath = processPath, percentiles = percentiles, statsReceiver = this.statsReceiver ) @@ -82,12 +78,11 @@ class MetricBuilder( def withVerbosity(verbosity: Verbosity): MetricBuilder = this.copy(verbosity = verbosity) - def withSourceClass(sourceClass: String): MetricBuilder = this.copy(sourceClass = sourceClass) + def withSourceClass(sourceClass: Option[String]): MetricBuilder = + this.copy(sourceClass = sourceClass) - def withSourceLibrary(sourceLibrary: Option[String]): MetricBuilder = - this.copy(sourceLibrary = sourceLibrary) - - def withIdentifier(identifier: String): MetricBuilder = this.copy(identifier = identifier) + def withIdentifier(processPath: Option[String]): MetricBuilder = + this.copy(processPath = processPath) def withUnits(units: MetricUnit): MetricBuilder = this.copy(units = units) @@ -156,28 +151,19 @@ class MetricBuilder( role == that.role && verbosity == that.verbosity && sourceClass == that.sourceClass && - sourceLibrary == that.sourceLibrary && name == that.name && - identifier == that.identifier && + processPath == that.processPath && percentiles == that.percentiles case _ => false } override def hashCode(): Int = { - val state = Seq[AnyRef]( - description, - units, - role, - verbosity, - sourceClass, - sourceLibrary, - name, - identifier, - percentiles) + val state = + Seq[AnyRef](description, units, role, verbosity, sourceClass, name, processPath, percentiles) val hashCodes = keyIndicator.hashCode() +: state.map(_.hashCode()) hashCodes.foldLeft(0)((a, b) => 31 * a + b) } override def toString = - s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $sourceLibrary, $name, $identifier, $percentiles, $statsReceiver)" + s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $name, $processPath, $percentiles, $statsReceiver)" } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala index 89e8b7baf2..f5d49f81ac 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala @@ -3,20 +3,22 @@ package com.twitter.finagle.stats /** * This trait represents a metric with the full set of its metadata. */ -sealed abstract class MetricSchema(metricBuilder: MetricBuilder) +sealed abstract class MetricSchema(val metricBuilder: MetricBuilder) /** * Represents a counter with its assorted metadata. */ -case class CounterSchema private (metricBuilder: MetricBuilder) extends MetricSchema(metricBuilder) +case class CounterSchema(override val metricBuilder: MetricBuilder) + extends MetricSchema(metricBuilder) /** * Represents a gauge with its assorted metadata. */ -case class GaugeSchema private (metricBuilder: MetricBuilder) extends MetricSchema(metricBuilder) +case class GaugeSchema private (override val metricBuilder: MetricBuilder) + extends MetricSchema(metricBuilder) /** * Represents a histogram with its assorted metadata. */ -case class HistogramSchema private (metricBuilder: MetricBuilder) +case class HistogramSchema private (override val metricBuilder: MetricBuilder) extends MetricSchema(metricBuilder) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala index de72ba0a56..2cf1557fc9 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala @@ -19,4 +19,6 @@ case object Milliseconds extends MetricUnit case object Microseconds extends MetricUnit case object Requests extends MetricUnit case object Percentage extends MetricUnit -case class CustomUnit(name: String) extends MetricUnit +case class CustomUnit(name: String) extends MetricUnit { + override val toString: String = s"CustomUnit($name)" +} diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala new file mode 100644 index 0000000000..cb0344cde7 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala @@ -0,0 +1,13 @@ +package com.twitter.finagle.stats + +/** + * Interface used via the LoadService mechanism to obtain an + * efficient mechanism to sample stats. + */ +private[twitter] trait SchemaRegistry { + + /** Whether or not the counters are latched. */ + def hasLatchedCounters: Boolean + + def schemas(): Map[String, MetricSchema] +} From 097095138cdd1b99dc4a20a9785c7bd71a26a7b6 Mon Sep 17 00:00:00 2001 From: Peter Vasterd Date: Mon, 23 Mar 2020 19:59:29 +0000 Subject: [PATCH 296/796] util-jvm: Support for HotSpot Jvm that do not implement counters Problem When creating an com.twitter.jmv.Hotspot object on HotSpot JVMs that do not implement JVM metrics, such as the HP NONSTOP JVM, the creation fails with a javax.naming.OperationNotSupportedException. Solution As these metrics/counters are not mandatory, creation of the com.twitter.jmv.Hotspot should just continue with a WARN message that these counters are not available. Result Support for more Hotspot JVMs, such as HP NONSTOP JVM. Signed-off-by: Jing Yan JIRA Issues: CSL-9555 Differential Revision: https://phabricator.twitter.biz/D454252 --- .../main/scala/com/twitter/jvm/Hotspot.scala | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index 13bee37de6..8a0779fdf0 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -13,6 +13,7 @@ import javax.management.{ ObjectName, RuntimeMBeanException } +import javax.naming.OperationNotSupportedException import scala.jdk.CollectionConverters._ import scala.language.reflectiveCalls @@ -56,10 +57,20 @@ class Hotspot extends Jvm { private[this] def long(c: Counter) = c.getValue().asInstanceOf[Long] private[this] def counters(pat: String) = { - val cs = jvm.getInternalCounters(pat).asScala - cs.map { c => - c.getName() -> c - }.toMap + try { + val cs = jvm.getInternalCounters(pat).asScala + cs.map { c => + c.getName() -> c + }.toMap + } catch { + case e: OperationNotSupportedException => + log.log( + Level.WARNING, + s"failed to get internal JVM counters as these are not available on your JVM.") + Map.empty[String, Counter] + case e: Throwable => + throw e + } } private[this] def counter(name: String): Option[Counter] = From 43671464bd0bcb925d970370d45eaa9e84400ccb Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Thu, 26 Mar 2020 21:25:58 +0000 Subject: [PATCH 297/796] util-core: Introduce a `CloseOnce` mixin and expose `isClosed` property on `ClosableOnce` Problem The `ClosableOnce` implementation does not expose its closed state. Due to this, there are multiple implementations of `ClosableOnce` that don't extend `ClosableOnce` in the Twitter stack. Because `ClosableOnce` is itself an extension of `Closable` and a `c.t.finagle.Service` also extends `Closable`, extending directly from `ClosableOnce` can result in compilation errors, without modifying code which would require making the `closeOnce` method no longer final. Solution Let's expose the closed state via a property, so that we can later change those implementations to extend from `ClosableOnce` or `CloseOnce` instead. We introduce the `CloseOnce` trait that can be mixed into an existing `Closable` (i.e. `c.t.finagle.Service`) without running into compilation issues. `ClosableOnce` now extends from `CloseOnce` and maintains the same behavior, with the added benefit of exposing the `isClosed` state. JIRA Issues: CSL-9560 Differential Revision: https://phabricator.twitter.biz/D455819 --- CHANGELOG.rst | 8 +++ .../scala/com/twitter/util/ClosableOnce.scala | 41 ++---------- .../scala/com/twitter/util/CloseOnce.scala | 63 +++++++++++++++++++ .../com/twitter/util/ClosableOnceTest.scala | 5 ++ 4 files changed, 80 insertions(+), 37 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/util/CloseOnce.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ebadd1d3ce..dde3c86dc8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,14 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ +* util-core: When looking to add idempotent close behavior, users should mix in `CloseOnce` to + classes which already extend (or implement) `Closable`, as mixing in `ClosableOnce` leads to + compiler errors. `ClosableOnce.of` can still be used to create a `ClosableOnce` proxy of an + already instantiated `Closable`. Classes which do not extend `Closable` can still + mix in `ClosableOnce`. ``PHAB_ID=D455819`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ * util-hashing: Rename diff --git a/util-core/src/main/scala/com/twitter/util/ClosableOnce.scala b/util-core/src/main/scala/com/twitter/util/ClosableOnce.scala index 8e5b9f6927..c40490dfa6 100644 --- a/util-core/src/main/scala/com/twitter/util/ClosableOnce.scala +++ b/util-core/src/main/scala/com/twitter/util/ClosableOnce.scala @@ -1,49 +1,16 @@ package com.twitter.util -import scala.util.control.NonFatal - /** * A mixin trait to describe resources that are an idempotent [[Closable]]. * * The first call to `close(Time)` triggers closing the resource with the * provided deadline while subsequent calls will yield the same `Future` * as the first invocation irrespective of the deadline provided. + * + * @see [[CloseOnce]] if you are mixing in or extending an existing [[Closable]] + * to prevent any "inherits from conflicting members" compile-time issues. */ -trait ClosableOnce extends Closable { - - // Our intrinsic lock for mutating the `closed` field - private[this] val closePromise: Promise[Unit] = Promise[Unit]() - private[this] var closed: Boolean = false - - private[this] def once(): Boolean = closePromise.synchronized { - if (closed) { - false - } else { - closed = true - true - } - } - - /** - * Close the resource with the given deadline exactly once. This deadline is - * advisory, giving the callee some leeway, for example to drain clients or - * finish up other tasks. - * - * @note if this method throws a synchronous exception, that exception will - * be wrapped in a failed future. - */ - protected def closeOnce(deadline: Time): Future[Unit] - - final def close(deadline: Time): Future[Unit] = { - if (once()) { - val closeF = - try closeOnce(deadline) - catch { case NonFatal(ex) => Future.exception(ex) } - closePromise.become(closeF) - } - closePromise - } -} +trait ClosableOnce extends Closable with CloseOnce object ClosableOnce { diff --git a/util-core/src/main/scala/com/twitter/util/CloseOnce.scala b/util-core/src/main/scala/com/twitter/util/CloseOnce.scala new file mode 100644 index 0000000000..2813480a97 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/CloseOnce.scala @@ -0,0 +1,63 @@ +package com.twitter.util + +import scala.util.control.NonFatal + +/** + * A mixin trait to describe resources that are an idempotent [[Closable]]. + * + * The first call to `close(Time)` triggers closing the resource with the + * provided deadline while subsequent calls will yield the same `Future` + * as the first invocation irrespective of the deadline provided. + * + * @see [[ClosableOnce]] if you are not mixing in or extending an existing [[Closable]] + * @see [[ClosableOnce.of(closable: Closable)]] for creating a proxy to a [[Closable]] + * that has already been instantiated. + */ +trait CloseOnce { self: Closable => + // Our intrinsic lock for mutating the `closed` field + private[this] val closePromise: Promise[Unit] = Promise[Unit]() + @volatile private[this] var closed: Boolean = false + + // optimized to limit synchronized locking scope + private[this] def firstCloseInvoked(): Boolean = closePromise.synchronized { + if (closed) { + false + } else { + closed = true + true + } + } + + /** + * Close the resource with the given deadline exactly once. This deadline is + * advisory, giving the callee some leeway, for example to drain clients or + * finish up other tasks. + * + * @note if this method throws a synchronous exception, that exception will + * be wrapped in a failed future. + */ + protected def closeOnce(deadline: Time): Future[Unit] + + /** + * Signals whether or not this [[Closable]] has been closed. + * + * @return return true if [[close]] has been initiated, false otherwise + */ + final def isClosed: Boolean = closed + + override final def close(deadline: Time): Future[Unit] = { + // only call `closeOnce()` and assign `closePromise` if this is the first `close()` invocation + if (firstCloseInvoked()) { + val closeF = + try { + closeOnce(deadline) + } catch { + case NonFatal(ex) => Future.exception(ex) + } + closePromise.become(closeF) + } + + closePromise + } + +} diff --git a/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala index 5ebc4a2462..2a9ad140dc 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala @@ -18,7 +18,9 @@ class ClosableOnceTest extends FunSuite { } val closableOnce = ClosableOnce.of(underlying) + assert(closableOnce.isClosed == false) closableOnce.close() + assert(closableOnce.isClosed == true) closableOnce.close() assert(closedCalls == 1) } @@ -33,8 +35,11 @@ class ClosableOnceTest extends FunSuite { } } + assert(closableOnce.isClosed == false) + assert(ready(closableOnce.close()).poll.get.throwable == ex) assert(closedCalls == 1) + assert(closableOnce.isClosed == true) assert(ready(closableOnce.close()).poll.get.throwable == ex) assert(closedCalls == 1) From 976ed332f06cbd0e57f4b760e87fb96c7b54d290 Mon Sep 17 00:00:00 2001 From: Shane Delmore Date: Fri, 27 Mar 2020 00:20:05 +0000 Subject: [PATCH 298/796] util: Reformat with scalafmt 2.4.2 Differential Revision: https://phabricator.twitter.biz/D455464 --- build.sbt | 423 +++++++++--------- project/build.sbt | 3 +- project/unidoc.sbt | 1 - .../src/main/scala/com/twitter/app/App.scala | 12 +- .../scala/com/twitter/app/ClassPath.scala | 9 +- .../scala/com/twitter/app/Flaggable.scala | 16 +- .../test/scala/com/twitter/app/AppTest.scala | 64 +-- .../com/twitter/app/ColonizeTestApp.scala | 2 +- .../scala/com/twitter/app/FlagsTest.scala | 4 +- .../concurrent/ConduitSpscBenchmark.scala | 14 +- .../stats/CumulativeGaugeBenchmark.scala | 4 +- .../hashing/KetamaDistributorBench.scala | 5 +- .../scala/com/twitter/jvm/CpuProfile.scala | 18 +- .../com/twitter/util/FutureBenchmark.scala | 28 +- .../com/twitter/cache/guava/GuavaCache.scala | 4 +- .../com/twitter/cache/EvictingCache.scala | 4 +- .../scala/com/twitter/cache/FutureCache.scala | 4 +- .../cache/caffeine/CaffeineCache.scala | 4 +- .../twitter/cache/KeyEncodingCacheTest.scala | 4 +- .../scala/com/twitter/concurrent/Offer.scala | 8 +- .../com/twitter/util/HealthyQueue.scala | 4 +- .../com/twitter/util/HealthyQueue.scala | 15 +- .../com/twitter/concurrent/AsyncMeter.scala | 6 +- .../twitter/concurrent/AsyncSemaphore.scala | 17 +- .../com/twitter/concurrent/AsyncStream.scala | 32 +- .../scala/com/twitter/concurrent/Spool.scala | 12 +- .../com/twitter/concurrent/SpoolSource.scala | 3 +- .../src/main/scala/com/twitter/io/Buf.scala | 2 +- .../scala/com/twitter/io/ByteReader.scala | 4 +- .../src/main/scala/com/twitter/io/Files.scala | 4 +- .../com/twitter/io/exp/ActivitySource.scala | 8 +- .../scala/com/twitter/util/Activity.scala | 18 +- .../scala/com/twitter/util/Base64Long.scala | 4 +- .../com/twitter/util/BatchExecutor.scala | 3 +- .../scala/com/twitter/util/Closable.scala | 11 +- .../scala/com/twitter/util/ConstFuture.scala | 17 +- .../scala/com/twitter/util/Credentials.scala | 4 +- .../main/scala/com/twitter/util/Diff.scala | 5 +- .../scala/com/twitter/util/Disposable.scala | 19 +- .../main/scala/com/twitter/util/Event.scala | 23 +- .../main/scala/com/twitter/util/Future.scala | 41 +- .../scala/com/twitter/util/FuturePool.scala | 15 +- .../main/scala/com/twitter/util/Futures.scala | 9 +- .../main/scala/com/twitter/util/Local.scala | 13 +- .../main/scala/com/twitter/util/NetUtil.scala | 4 +- .../main/scala/com/twitter/util/Signal.scala | 4 +- .../twitter/util/SlowProbeProxyTimer.scala | 3 +- .../scala/com/twitter/util/Stopwatch.scala | 12 +- .../main/scala/com/twitter/util/Timer.scala | 4 +- .../src/main/scala/com/twitter/util/Try.scala | 11 +- .../src/main/scala/com/twitter/util/Var.scala | 4 +- .../twitter/concurrent/AsyncStreamTest.scala | 52 +-- .../com/twitter/concurrent/OfferTest.scala | 36 +- .../com/twitter/concurrent/SpoolTest.scala | 62 +-- .../twitter/conversions/PercentOpsTest.scala | 6 +- .../com/twitter/conversions/U64OpsTest.scala | 4 +- .../scala/com/twitter/io/BufReaderTest.scala | 4 +- .../test/scala/com/twitter/io/BufTest.scala | 12 +- .../scala/com/twitter/io/ByteReaderTest.scala | 4 +- .../test/scala/com/twitter/io/FilesTest.scala | 4 +- .../test/scala/com/twitter/io/PipeTest.scala | 4 +- .../com/twitter/util/Base64LongTest.scala | 8 +- .../scala/com/twitter/util/ClosableTest.scala | 4 +- .../com/twitter/util/CloseAwaitablyTest.scala | 8 +- .../scala/com/twitter/util/DurationTest.scala | 12 +- .../scala/com/twitter/util/EventTest.scala | 16 +- .../scala/com/twitter/util/FutureTest.scala | 69 +-- .../scala/com/twitter/util/MemoizeTest.scala | 4 +- .../scala/com/twitter/util/MonitorTest.scala | 12 +- .../scala/com/twitter/util/PromiseTest.scala | 8 +- .../scala/com/twitter/util/TimeTest.scala | 4 +- .../test/scala/com/twitter/util/VarTest.scala | 90 +--- .../com/twitter/util/WaitQueueTest.scala | 4 +- .../ConsistentHashingDistributor.scala | 10 +- .../scala/com/twitter/hashing/Hashable.scala | 18 +- .../ConsistentHashingDistributorTest.scala | 4 +- .../twitter/hashing/DistributionTester.scala | 8 +- .../com/twitter/hashing/HashableTest.scala | 4 +- .../com/twitter/util/capturepoints/Demo.scala | 8 +- .../scala/com/twitter/jvm/Allocations.scala | 4 +- .../com/twitter/jvm/ContentionSnapshot.scala | 4 +- .../scala/com/twitter/jvm/Estimator.scala | 12 +- .../main/scala/com/twitter/jvm/Hotspot.scala | 4 +- .../src/main/scala/com/twitter/jvm/Jvm.scala | 3 +- .../main/scala/com/twitter/jvm/JvmStats.scala | 8 +- .../scala/com/twitter/jvm/EstimatorTest.scala | 2 +- .../com/twitter/logging/FileHandler.scala | 4 +- .../scala/com/twitter/logging/Formatter.scala | 4 +- .../scala/com/twitter/logging/Handler.scala | 4 +- .../scala/com/twitter/logging/Logger.scala | 8 +- .../com/twitter/logging/LoggerFactory.scala | 8 +- .../com/twitter/logging/QueueingHandler.scala | 4 +- .../com/twitter/logging/FileHandlerTest.scala | 18 +- .../com/twitter/logging/FormatterTest.scala | 34 +- .../com/twitter/logging/LogRecordTest.scala | 9 +- .../com/twitter/util/reflect/ProxyTest.scala | 16 +- .../com/twitter/util/registry/Registry.scala | 4 +- .../finagle/stats/ExceptionStatsHandler.scala | 10 +- .../finagle/stats/RollupStatsReceiver.scala | 4 +- .../com/twitter/finagle/stats/Stat.scala | 4 +- .../finagle/stats/WithHistogramDetails.scala | 3 +- .../finagle/stats/CumulativeGaugeTest.scala | 4 +- .../finagle/stats/StatsReceiverTest.scala | 10 +- .../util/testing/ArgumentCapture.scala | 96 ++-- .../util/tunable/JsonTunableMapper.scala | 4 +- .../com/twitter/util/tunable/TunableMap.scala | 4 +- .../twitter/util/tunable/TunableMapTest.scala | 12 +- .../com/twitter/zk/NativeConnector.scala | 8 +- .../src/main/scala/com/twitter/zk/ZNode.scala | 20 +- .../src/main/scala/com/twitter/zk/ZOp.scala | 6 +- .../zk/coordination/ShardCoordinator.scala | 8 +- .../zk/coordination/ZkAsyncSemaphore.scala | 22 +- .../scala/com/twitter/zk/ConnectorTest.scala | 28 +- .../test/scala/com/twitter/zk/ZNodeTest.scala | 4 +- .../scala/com/twitter/zk/ZkClientTest.scala | 69 +-- .../coordination/ZkAsyncSemaphoreTest.scala | 4 +- 116 files changed, 699 insertions(+), 1212 deletions(-) diff --git a/build.sbt b/build.sbt index ff593008fd..d7161281c2 100644 --- a/build.sbt +++ b/build.sbt @@ -6,7 +6,7 @@ val releaseVersion = "20.4.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" val zkGroupVersion = "0.0.92" -val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll( +val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll ( ExclusionRule("com.sun.jdmk", "jmxtools"), ExclusionRule("com.sun.jmx", "jmxri"), ExclusionRule("javax.jms", "jms") @@ -102,43 +102,35 @@ val baseSettings = Seq( case _ => sourceDir / "scala-2.12-" } }, - ScoverageKeys.coverageHighlighting := true, resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", - scalacOptions := Seq( "-target:jvm-1.8", "-unchecked", "-deprecation", "-feature", - "-encoding", "utf8", + "-encoding", + "utf8", // Needs -missing-interpolator due to https://issues.scala-lang.org/browse/SI-8761 "-Xlint:-missing-interpolator", "-Yrangepos" ), - // Note: Use -Xlint rather than -Xlint:unchecked when TestThriftStructure // warnings are resolved javacOptions ++= Seq("-Xlint:unchecked", "-source", "1.8", "-target", "1.8"), javacOptions in doc := Seq("-source", "1.8"), - javaOptions ++= Seq( "-Djava.net.preferIPv4Stack=true", "-XX:+AggressiveOpts", "-server" ), - javaOptions ++= gcJavaOptions, - javaOptions in Test ++= travisTestJavaOptions, - // -a: print stack traces for failing asserts testOptions += Tests.Argument(TestFrameworks.JUnit, "-a"), - // This is bad news for things like com.twitter.util.Time parallelExecution in Test := false, - // Sonatype publishing publishArtifact in Test := false, pomIncludeRepository := { _ => false }, @@ -170,7 +162,7 @@ val baseSettings = Seq( if (version.value.trim.endsWith("SNAPSHOT")) Some("snapshots" at nexus + "content/repositories/snapshots") else - Some("releases" at nexus + "service/local/staging/deploy/maven2") + Some("releases" at nexus + "service/local/staging/deploy/maven2") } ) @@ -188,332 +180,331 @@ lazy val util = Project( id = "util", base = file(".") ).enablePlugins( - ScalaUnidocPlugin -).settings( - sharedSettings ++ - noPublishSettings ++ - Seq( - unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(utilBenchmark) - ) -).aggregate( - utilApp, - utilBenchmark, - utilCache, - utilCacheGuava, - utilClassPreloader, - utilCodec, - utilCore, - utilDoc, - utilFunction, - utilHashing, - utilJvm, - utilLint, - utilLogging, - utilReflect, - utilRegistry, - utilSecurity, - utilSlf4jApi, - utilSlf4jJulBridge, - utilStats, - utilTest, - utilThrift, - utilTunable, - utilZk, - utilZkTest -) + ScalaUnidocPlugin + ).settings( + sharedSettings ++ + noPublishSettings ++ + Seq( + unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(utilBenchmark) + ) + ).aggregate( + utilApp, + utilBenchmark, + utilCache, + utilCacheGuava, + utilClassPreloader, + utilCodec, + utilCore, + utilDoc, + utilFunction, + utilHashing, + utilJvm, + utilLint, + utilLogging, + utilReflect, + utilRegistry, + utilSecurity, + utilSlf4jApi, + utilSlf4jJulBridge, + utilStats, + utilTest, + utilThrift, + utilTunable, + utilZk, + utilZkTest + ) lazy val utilApp = Project( id = "util-app", base = file("util-app") ).settings( - sharedSettings -).settings( - name := "util-app" -).dependsOn(utilCore, utilRegistry) + sharedSettings + ).settings( + name := "util-app" + ).dependsOn(utilCore, utilRegistry) lazy val utilBenchmark = Project( id = "util-benchmark", base = file("util-benchmark") ).settings( - sharedSettings -).enablePlugins( - JmhPlugin -).settings( - name := "util-benchmark" -).dependsOn(utilCore, utilHashing, utilJvm, utilStats) + sharedSettings + ).enablePlugins( + JmhPlugin + ).settings( + name := "util-benchmark" + ).dependsOn(utilCore, utilHashing, utilJvm, utilStats) lazy val utilCache = Project( id = "util-cache", base = file("util-cache") ).settings( - sharedSettings -).settings( - name := "util-cache", - libraryDependencies ++= Seq(caffeineLib, jsr305Lib) -).dependsOn(utilCore) + sharedSettings + ).settings( + name := "util-cache", + libraryDependencies ++= Seq(caffeineLib, jsr305Lib) + ).dependsOn(utilCore) lazy val utilCacheGuava = Project( id = "util-cache-guava", base = file("util-cache-guava") ).settings( - sharedSettings -).settings( - name := "util-cache-guava", - libraryDependencies ++= Seq(guavaLib, jsr305Lib) -).dependsOn(utilCache % "compile->compile;test->test", utilCore) + sharedSettings + ).settings( + name := "util-cache-guava", + libraryDependencies ++= Seq(guavaLib, jsr305Lib) + ).dependsOn(utilCache % "compile->compile;test->test", utilCore) lazy val utilClassPreloader = Project( id = "util-class-preloader", base = file("util-class-preloader") ).settings( - sharedSettings -).settings( - name := "util-class-preloader" -).dependsOn(utilCore) + sharedSettings + ).settings( + name := "util-class-preloader" + ).dependsOn(utilCore) lazy val utilCodec = Project( id = "util-codec", base = file("util-codec") ).settings( - sharedSettings -).settings( - name := "util-codec" -).dependsOn(utilCore) + sharedSettings + ).settings( + name := "util-codec" + ).dependsOn(utilCore) lazy val utilCore = Project( id = "util-core", base = file("util-core") ).settings( - sharedSettings -).settings( - name := "util-core", - // Moved some code to 'concurrent-extra' to conform to Pants' 1:1:1 principle (https://www.pantsbuild.org/build_files.html#target-granularity) - // so that util-core would work better for Pants projects in IntelliJ. - unmanagedSourceDirectories in Compile += baseDirectory.value / "concurrent-extra", - libraryDependencies ++= Seq( - caffeineLib % "test", - scalacheckLib, - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2" - ), - resourceGenerators in Compile += Def.task { - val projectName = name.value - val file = resourceManaged.value / "com" / "twitter" / projectName / "build.properties" - val buildRev = scala.sys.process.Process("git" :: "rev-parse" :: "HEAD" :: Nil).!!.trim - val buildName = new java.text.SimpleDateFormat("yyyyMMdd-HHmmss").format(new java.util.Date) - val contents = s"name=$projectName\nversion=${version.value}\nbuild_revision=$buildRev\nbuild_name=$buildName" - IO.write(file, contents) - Seq(file) - }.taskValue, - sources in (Compile, doc) := { - // Monitors.java causes "not found: type Monitor$" (CSL-5034) - // so exclude it from the sources used for scaladoc - val previous = (sources in (Compile, doc)).value - previous.filterNot(file => file.getName() == "Monitors.java") - } -).dependsOn(utilFunction) + sharedSettings + ).settings( + name := "util-core", + // Moved some code to 'concurrent-extra' to conform to Pants' 1:1:1 principle (https://www.pantsbuild.org/build_files.html#target-granularity) + // so that util-core would work better for Pants projects in IntelliJ. + unmanagedSourceDirectories in Compile += baseDirectory.value / "concurrent-extra", + libraryDependencies ++= Seq( + caffeineLib % "test", + scalacheckLib, + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2" + ), + resourceGenerators in Compile += Def.task { + val projectName = name.value + val file = resourceManaged.value / "com" / "twitter" / projectName / "build.properties" + val buildRev = scala.sys.process.Process("git" :: "rev-parse" :: "HEAD" :: Nil).!!.trim + val buildName = new java.text.SimpleDateFormat("yyyyMMdd-HHmmss").format(new java.util.Date) + val contents = + s"name=$projectName\nversion=${version.value}\nbuild_revision=$buildRev\nbuild_name=$buildName" + IO.write(file, contents) + Seq(file) + }.taskValue, + sources in (Compile, doc) := { + // Monitors.java causes "not found: type Monitor$" (CSL-5034) + // so exclude it from the sources used for scaladoc + val previous = (sources in (Compile, doc)).value + previous.filterNot(file => file.getName() == "Monitors.java") + } + ).dependsOn(utilFunction) lazy val utilDoc = Project( id = "util-doc", base = file("doc") ).enablePlugins( - SphinxPlugin -).settings( - sharedSettings ++ - noPublishSettings ++ Seq( - scalacOptions in doc ++= Seq("-doc-title", "Util", "-doc-version", version.value), - includeFilter in Sphinx := ("*.html" | "*.png" | "*.svg" | "*.js" | "*.css" | "*.gif" | "*.txt") - )) + SphinxPlugin + ).settings( + sharedSettings ++ + noPublishSettings ++ Seq( + scalacOptions in doc ++= Seq("-doc-title", "Util", "-doc-version", version.value), + includeFilter in Sphinx := ("*.html" | "*.png" | "*.svg" | "*.js" | "*.css" | "*.gif" | "*.txt") + )) lazy val utilFunction = Project( id = "util-function", base = file("util-function") ).settings( - sharedSettings -).settings( - name := "util-function" -) + sharedSettings + ).settings( + name := "util-function" + ) lazy val utilReflect = Project( id = "util-reflect", base = file("util-reflect") ).settings( - sharedSettings -).settings( - name := "util-reflect", - libraryDependencies ++= Seq( - "asm" % "asm" % "3.3.1", - "asm" % "asm-util" % "3.3.1", - "asm" % "asm-commons" % "3.3.1", - "cglib" % "cglib" % "2.2.2" - ) -).dependsOn(utilCore) + sharedSettings + ).settings( + name := "util-reflect", + libraryDependencies ++= Seq( + "asm" % "asm" % "3.3.1", + "asm" % "asm-util" % "3.3.1", + "asm" % "asm-commons" % "3.3.1", + "cglib" % "cglib" % "2.2.2" + ) + ).dependsOn(utilCore) lazy val utilHashing = Project( id = "util-hashing", base = file("util-hashing") ).settings( - sharedSettings -).settings( - name := "util-hashing", - libraryDependencies += scalacheckLib -).dependsOn(utilCore % "test") + sharedSettings + ).settings( + name := "util-hashing", + libraryDependencies += scalacheckLib + ).dependsOn(utilCore % "test") lazy val utilIntellij = Project( id = "util-intellij", base = file("util-intellij") ).settings( - baseSettings -).settings( - name := "util-intellij", - scalaVersion := "2.11.12" -).dependsOn(utilCore % "test") + baseSettings + ).settings( + name := "util-intellij", + scalaVersion := "2.11.12" + ).dependsOn(utilCore % "test") lazy val utilJvm = Project( id = "util-jvm", base = file("util-jvm") ).settings( - sharedSettings -).settings( - name := "util-jvm" -).dependsOn(utilApp, utilCore, utilStats, utilTest % "test") + sharedSettings + ).settings( + name := "util-jvm" + ).dependsOn(utilApp, utilCore, utilStats, utilTest % "test") lazy val utilLint = Project( id = "util-lint", base = file("util-lint") ).settings( - sharedSettings -).settings( - name := "util-lint" -).dependsOn(utilCore) + sharedSettings + ).settings( + name := "util-lint" + ).dependsOn(utilCore) lazy val utilLogging = Project( id = "util-logging", base = file("util-logging") ).settings( - sharedSettings -).settings( - name := "util-logging" -).dependsOn(utilCore, utilApp, utilStats) + sharedSettings + ).settings( + name := "util-logging" + ).dependsOn(utilCore, utilApp, utilStats) lazy val utilSlf4jApi = Project( id = "util-slf4j-api", base = file("util-slf4j-api") ).settings( - sharedSettings -).settings( - name := "util-slf4j-api", - libraryDependencies ++= Seq( - slf4jApi, - "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" - ) -).dependsOn(utilCore % "test") + sharedSettings + ).settings( + name := "util-slf4j-api", + libraryDependencies ++= Seq( + slf4jApi, + "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" + ) + ).dependsOn(utilCore % "test") lazy val utilSlf4jJulBridge = Project( id = "util-slf4j-jul-bridge", base = file("util-slf4j-jul-bridge") ).settings( - sharedSettings -).settings( - name := "util-slf4j-jul-bridge", - libraryDependencies ++= Seq( - slf4jApi, - "org.slf4j" % "jul-to-slf4j" % slf4jVersion) -).dependsOn(utilCore, utilSlf4jApi) + sharedSettings + ).settings( + name := "util-slf4j-jul-bridge", + libraryDependencies ++= Seq(slf4jApi, "org.slf4j" % "jul-to-slf4j" % slf4jVersion) + ).dependsOn(utilCore, utilSlf4jApi) lazy val utilRegistry = Project( id = "util-registry", base = file("util-registry") ).settings( - sharedSettings -).settings( - name := "util-registry" -).dependsOn(utilCore) + sharedSettings + ).settings( + name := "util-registry" + ).dependsOn(utilCore) lazy val utilSecurity = Project( id = "util-security", base = file("util-security") ).settings( - sharedSettings -).settings( - name := "util-security" -).dependsOn(utilCore, utilLogging) + sharedSettings + ).settings( + name := "util-security" + ).dependsOn(utilCore, utilLogging) lazy val utilStats = Project( id = "util-stats", base = file("util-stats") ).settings( - sharedSettings -).settings( - name := "util-stats", - libraryDependencies ++= Seq(caffeineLib, jsr305Lib, scalacheckLib) ++ { - CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, major)) if major >= 13 => - Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "0.2.0" % "test") - case _ => - Seq() - } -} -).dependsOn(utilCore, utilLint) + sharedSettings + ).settings( + name := "util-stats", + libraryDependencies ++= Seq(caffeineLib, jsr305Lib, scalacheckLib) ++ { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, major)) if major >= 13 => + Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "0.2.0" % "test") + case _ => + Seq() + } + } + ).dependsOn(utilCore, utilLint) lazy val utilTest = Project( id = "util-test", base = file("util-test") ).settings( - sharedSettings -).settings( - name := "util-test", - libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % "3.0.0", - "org.mockito" % "mockito-all" % "1.10.19" - ) -).dependsOn(utilCore, utilLogging) + sharedSettings + ).settings( + name := "util-test", + libraryDependencies ++= Seq( + "org.scalatest" %% "scalatest" % "3.0.0", + "org.mockito" % "mockito-all" % "1.10.19" + ) + ).dependsOn(utilCore, utilLogging) lazy val utilThrift = Project( id = "util-thrift", base = file("util-thrift") ).settings( - sharedSettings -).settings( - name := "util-thrift", - libraryDependencies ++= Seq( - "org.apache.thrift" % "libthrift" % "0.10.0", - slf4jApi % "provided", - "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion - ) -).dependsOn(utilCodec) + sharedSettings + ).settings( + name := "util-thrift", + libraryDependencies ++= Seq( + "org.apache.thrift" % "libthrift" % "0.10.0", + slf4jApi % "provided", + "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion + ) + ).dependsOn(utilCodec) lazy val utilTunable = Project( id = "util-tunable", base = file("util-tunable") ).settings( - sharedSettings -).settings( - name := "util-tunable", - libraryDependencies ++= Seq( - "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion, - "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude("com.google.guava", "guava") - ) -).dependsOn(utilApp, utilCore) + sharedSettings + ).settings( + name := "util-tunable", + libraryDependencies ++= Seq( + "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion, + "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava") + ) + ).dependsOn(utilApp, utilCore) lazy val utilZk = Project( id = "util-zk", base = file("util-zk") ).settings( - sharedSettings -).settings( - name := "util-zk", - libraryDependencies += zkDependency -).dependsOn(utilCore, utilLogging) + sharedSettings + ).settings( + name := "util-zk", + libraryDependencies += zkDependency + ).dependsOn(utilCore, utilLogging) lazy val utilZkTest = Project( id = "util-zk-test", base = file("util-zk-test") ).settings( - sharedSettings -).settings( - name := "util-zk-test", - libraryDependencies += zkDependency -).dependsOn(utilCore % "test") + sharedSettings + ).settings( + name := "util-zk-test", + libraryDependencies += zkDependency + ).dependsOn(utilCore % "test") diff --git a/project/build.sbt b/project/build.sbt index 0c8c057a88..26f5fb29cc 100644 --- a/project/build.sbt +++ b/project/build.sbt @@ -2,5 +2,6 @@ scalacOptions ++= Seq( "-deprecation", "-unchecked", "-feature", - "-encoding", "utf8" + "-encoding", + "utf8" ) diff --git a/project/unidoc.sbt b/project/unidoc.sbt index 046b1130a6..5818ca1f65 100644 --- a/project/unidoc.sbt +++ b/project/unidoc.sbt @@ -1,2 +1 @@ addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1") - diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index 14f517798c..ccc4466aa8 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -251,9 +251,7 @@ trait App extends Closable with CloseAwaitably { // `close()` already called, we need to close this here, but only // after `close()` completes and `closing` is satisfied closing - .transform { _ => - closable.close(closeDeadline) - } + .transform { _ => closable.close(closeDeadline) } .by(shutdownTimer, closeDeadline) } } @@ -320,9 +318,7 @@ trait App extends Closable with CloseAwaitably { s"${errors.size} errors occurred on exit" } val exc = new CloseException(message) - errors.foreach { error => - exc.addSuppressed(error) - } + errors.foreach { error => exc.addSuppressed(error) } exc } @@ -362,9 +358,7 @@ trait App extends Closable with CloseAwaitably { final def nonExitingMain(args: Array[String]): Unit = { App.register(this) - loadServiceBindings.foreach { binding => - LoadService.bind(binding) - } + loadServiceBindings.foreach { binding => LoadService.bind(binding) } for (f <- inits) f() diff --git a/util-app/src/main/scala/com/twitter/app/ClassPath.scala b/util-app/src/main/scala/com/twitter/app/ClassPath.scala index bdbaa3ca85..261cfd4129 100644 --- a/util-app/src/main/scala/com/twitter/app/ClassPath.scala +++ b/util-app/src/main/scala/com/twitter/app/ClassPath.scala @@ -149,10 +149,11 @@ private[app] sealed abstract class ClassPath[CpInfo <: ClassPath.Info] { buf: Builder[CpInfo, Seq[CpInfo]], seenUris: mutable.Set[URI] ): Unit = { - val jarFile = try new JarFile(file) - catch { - case _: IOException => return // not a Jar file - } + val jarFile = + try new JarFile(file) + catch { + case _: IOException => return // not a Jar file + } try { for (uri <- jarClasspath(file, jarFile.getManifest)) { diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index 8a824a6747..2912db176e 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -99,27 +99,19 @@ object Flaggable { implicit val ofInt: Flaggable[Int] = mandatory(_.toInt) implicit val ofJavaInteger: Flaggable[JInteger] = - mandatory { s: String => - JInteger.valueOf(s.toInt) - } + mandatory { s: String => JInteger.valueOf(s.toInt) } implicit val ofLong: Flaggable[Long] = mandatory(_.toLong) implicit val ofJavaLong: Flaggable[JLong] = - mandatory { s: String => - JLong.valueOf(s.toLong) - } + mandatory { s: String => JLong.valueOf(s.toLong) } implicit val ofFloat: Flaggable[Float] = mandatory(_.toFloat) implicit val ofJavaFloat: Flaggable[JFloat] = - mandatory { s: String => - JFloat.valueOf(s.toFloat) - } + mandatory { s: String => JFloat.valueOf(s.toFloat) } implicit val ofDouble: Flaggable[Double] = mandatory(_.toDouble) implicit val ofJavaDouble: Flaggable[JDouble] = - mandatory { s: String => - JDouble.valueOf(s.toDouble) - } + mandatory { s: String => JDouble.valueOf(s.toDouble) } // Conversions for common non-primitive types and collections. implicit val ofDuration: Flaggable[Duration] = mandatory(Duration.parse(_)) diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index c2665e50fe..d8949dd832 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -140,12 +140,8 @@ class AppTest extends FunSuite { val a = new App { def main(): Unit = () } val p = new Promise[Unit] var n1, n2 = 0 - val c1 = Closable.make { _ => - n1 += 1; p - } - val c2 = Closable.make { _ => - n2 += 1; Future.Done - } + val c1 = Closable.make { _ => n1 += 1; p } + val c2 = Closable.make { _ => n2 += 1; Future.Done } a.closeOnExitLast(c2) a.closeOnExit(c1) val f = a.close() @@ -169,12 +165,8 @@ class AppTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => var n1, n2 = 0 - val c1 = Closable.make { _ => - n1 += 1; Future.never - } - val c2 = Closable.make { _ => - n2 += 1; Future.never - } + val c1 = Closable.make { _ => n1 += 1; Future.never } + val c2 = Closable.make { _ => n2 += 1; Future.never } a.closeOnExitLast(c2) a.closeOnExit(c1) val f = a.close(Time.now + 1.second) @@ -204,18 +196,10 @@ class AppTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => var n1, n2, n3, n4 = 0 - val c1 = Closable.make { _ => - n1 += 1; Future.never - } // Exit - val c2 = Closable.make { _ => - n2 += 1; Future.Done - } // ExitLast - val c3 = Closable.make { _ => - n3 += 1; Future.Done - } // Late Exit - val c4 = Closable.make { _ => - n4 += 1; Future.Done - } // Late ExitLast + val c1 = Closable.make { _ => n1 += 1; Future.never } // Exit + val c2 = Closable.make { _ => n2 += 1; Future.Done } // ExitLast + val c3 = Closable.make { _ => n3 += 1; Future.Done } // Late Exit + val c4 = Closable.make { _ => n4 += 1; Future.Done } // Late ExitLast a.closeOnExitLast(c2) a.closeOnExit(c1) val f = a.close(Time.now + 1.second) @@ -256,18 +240,10 @@ class AppTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => var n1, n2, n3, n4 = 0 - val c1 = Closable.make { _ => - n1 += 1; Future.never - } // Exit - val c2 = Closable.make { _ => - n2 += 1; Future.never - } // ExitLast - val c3 = Closable.make { _ => - n3 += 1; Future.never - } // Late Exit - val c4 = Closable.make { _ => - n4 += 1; Future.never - } // Late ExitLast + val c1 = Closable.make { _ => n1 += 1; Future.never } // Exit + val c2 = Closable.make { _ => n2 += 1; Future.never } // ExitLast + val c3 = Closable.make { _ => n3 += 1; Future.never } // Late Exit + val c4 = Closable.make { _ => n4 += 1; Future.never } // Late ExitLast a.closeOnExitLast(c2) a.closeOnExit(c1) val f = a.close(Time.now + 1.second) @@ -339,13 +315,9 @@ class AppTest extends FunSuite { throw new Exception("FORCED ON EXIT") } - closeOnExit(Closable.make { _ => - throw new Exception("FORCED CLOSE ON EXIT") - }) + closeOnExit(Closable.make { _ => throw new Exception("FORCED CLOSE ON EXIT") }) - closeOnExitLast(Closable.make { _ => - throw new Exception("FORCED CLOSE ON EXIT LAST") - }) + closeOnExitLast(Closable.make { _ => throw new Exception("FORCED CLOSE ON EXIT LAST") }) } } @@ -360,9 +332,7 @@ class AppTest extends FunSuite { // first fatal (InterruptedException) kills the app during close val app = new ErrorOnExitApp { def main(): Unit = { - closeOnExit(Closable.make { _ => - throw new InterruptedException("FORCED CLOSE ON EXIT") - }) + closeOnExit(Closable.make { _ => throw new InterruptedException("FORCED CLOSE ON EXIT") }) } } @@ -378,9 +348,7 @@ class AppTest extends FunSuite { throw new Exception("FORCED ON EXIT") } - closeOnExit(Closable.make { _ => - throw new Exception("FORCED CLOSE ON EXIT") - }) + closeOnExit(Closable.make { _ => throw new Exception("FORCED CLOSE ON EXIT") }) closeOnExitLast(Closable.make { _ => throw new InterruptedException("FORCED CLOSE ON EXIT LAST") diff --git a/util-app/src/test/scala/com/twitter/app/ColonizeTestApp.scala b/util-app/src/test/scala/com/twitter/app/ColonizeTestApp.scala index 0efaa06162..295788062a 100644 --- a/util-app/src/test/scala/com/twitter/app/ColonizeTestApp.scala +++ b/util-app/src/test/scala/com/twitter/app/ColonizeTestApp.scala @@ -50,7 +50,7 @@ object SolarSystemPlanets { type Kilogram = Double type Meter = Double - private val G = 6.67300E-11 // universal gravitational constant (m3 kg-1 s-2) + private val G = 6.67300e-11 // universal gravitational constant (m3 kg-1 s-2) } class ColonizeTestApp extends App { diff --git a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala index 2609ad28c8..1788c94cc8 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala @@ -259,7 +259,9 @@ class FlagsTest extends FunSuite { intercept[IllegalStateException] { testFlags .getAll() - .foreach(flag => if (flag.name == "something.id") flag.apply()) // this should fail, since all added flags should now fail fast. + .foreach(flag => + if (flag.name == "something.id") + flag.apply()) // this should fail, since all added flags should now fail fast. } // now parse and apply diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/ConduitSpscBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/ConduitSpscBenchmark.scala index ed43f656d1..a3ebdde41e 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/ConduitSpscBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/ConduitSpscBenchmark.scala @@ -83,9 +83,7 @@ class ConduitSpscBenchmark extends StdBenchAnnotations { val stream = mkAsyncStream(size) // Consume - stream.foldLeftF(false) { (_, buf) => - sink(buf) - } + stream.foldLeftF(false) { (_, buf) => sink(buf) } }) private[this] def feedBroker(n: Int): Future[Unit] = @@ -131,11 +129,7 @@ class ConduitSpscBenchmark extends StdBenchAnnotations { private[this] def consumeSpool[A](spool: Spool[A], b: Boolean): Future[Boolean] = if (spool.isEmpty) Future.value(b) else - sink(spool.head).flatMap { newB => - spool.tail.flatMap { tail => - consumeSpool(tail, newB) - } - } + sink(spool.head).flatMap { newB => spool.tail.flatMap { tail => consumeSpool(tail, newB) } } @Benchmark def spool: Boolean = @@ -144,8 +138,6 @@ class ConduitSpscBenchmark extends StdBenchAnnotations { val f = mkSpool(size) // Consume - f.flatMap { s => - consumeSpool(s, false) - } + f.flatMap { s => consumeSpool(s, false) } }) } diff --git a/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala index 04b6ca10ba..3b525092d4 100644 --- a/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala @@ -24,9 +24,7 @@ class CumulativeGaugeBenchmark extends StdBenchAnnotations { def setup(): Unit = { cg = new CGauge() gauges.clear() - 0.until(num).foreach { _ => - gauges.add(cg.addGauge(1)) - } + 0.until(num).foreach { _ => gauges.add(cg.addGauge(1)) } } @Benchmark diff --git a/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala b/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala index de524d6f89..cdb2f31bd1 100644 --- a/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala +++ b/util-benchmark/src/main/scala/com/twitter/hashing/KetamaDistributorBench.scala @@ -16,9 +16,8 @@ class KetamaDistributorBench extends StdBenchAnnotations { val numReps = 640 val numHosts = 310 - val hosts: scala.collection.immutable.IndexedSeq[String] = 0.until(numHosts).map { i => - "abcd-zzz-" + i + "-ab2.qwer.twitter.com" - } + val hosts: scala.collection.immutable.IndexedSeq[String] = + 0.until(numHosts).map { i => "abcd-zzz-" + i + "-ab2.qwer.twitter.com" } val nodes: scala.collection.immutable.IndexedSeq[HashNode[String]] = hosts map { h => HashNode[String](h + ":12021", 1, h) } diff --git a/util-benchmark/src/main/scala/com/twitter/jvm/CpuProfile.scala b/util-benchmark/src/main/scala/com/twitter/jvm/CpuProfile.scala index 8e86063ccb..808da933c7 100644 --- a/util-benchmark/src/main/scala/com/twitter/jvm/CpuProfile.scala +++ b/util-benchmark/src/main/scala/com/twitter/jvm/CpuProfile.scala @@ -60,9 +60,9 @@ object CpuProfileBenchmark { @Setup(Level.Iteration) def setUp(): Unit = { val stack = new Stack(new Random(rngSeed), stackMeanSize, stackStddev) - threads = for (_ <- 0 until nthreads) - yield - new Thread { + threads = + for (_ <- 0 until nthreads) + yield new Thread { override def run(): Unit = { try stack() catch { @@ -71,19 +71,13 @@ object CpuProfileBenchmark { } } - threads foreach { t => - t.start() - } + threads foreach { t => t.start() } } @TearDown(Level.Iteration) def tearDown(): Unit = { - threads foreach { t => - t.interrupt() - } - threads foreach { t => - t.join() - } + threads foreach { t => t.interrupt() } + threads foreach { t => t.join() } threads = Seq() } var threads = Seq[Thread]() diff --git a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala index 494cc4b758..4095af37ba 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala @@ -216,12 +216,8 @@ class FutureBenchmark extends StdBenchAnnotations { var sum = 0 var i = 0 - val fmap = { i: Int => - Future.value(i + 1) - } - val respond = { _: Try[Int] => - sum += 1 - } + val fmap = { i: Int => Future.value(i + 1) } + val respond = { _: Try[Int] => sum += 1 } while (i < state.depth) { next = i % 2 match { @@ -260,18 +256,10 @@ class FutureBenchmark extends StdBenchAnnotations { object FutureBenchmark { final val N = 10 - private val RespondFn: Try[Unit] => Unit = { _ => - () - } - private val FlatMapFn: Unit => Future[Unit] = { _: Unit => - Future.Unit - } - private val MapFn: Unit => Unit = { _: Unit => - () - } - private val TransformFn: Try[Unit] => Future[Unit] = { _: Try[Unit] => - Future.Done - } + private val RespondFn: Try[Unit] => Unit = { _ => () } + private val FlatMapFn: Unit => Future[Unit] = { _: Unit => Future.Unit } + private val MapFn: Unit => Unit = { _: Unit => () } + private val TransformFn: Try[Unit] => Future[Unit] = { _: Try[Unit] => Future.Done } @State(Scope.Benchmark) class ByState { @@ -297,9 +285,7 @@ object FutureBenchmark { @Setup def prepare(): Unit = { - futures = (0 until size).map { i => - Future.value(i) - }.toList + futures = (0 until size).map { i => Future.value(i) }.toList } } diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala b/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala index 1f98ebbd5a..dacafb1537 100644 --- a/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala @@ -51,9 +51,7 @@ object GuavaCache { */ def fromLoadingCache[K, V](cache: LoadingCache[K, Future[V]]): K => Future[V] = { val evicting = EvictingCache.lazily(new LoadingFutureCache(cache)); - { key: K => - evicting.get(key).get.interruptible() - } + { key: K => evicting.get(key).get.interruptible() } } /** diff --git a/util-cache/src/main/scala/com/twitter/cache/EvictingCache.scala b/util-cache/src/main/scala/com/twitter/cache/EvictingCache.scala index d56ee28cb8..3b0d28498e 100644 --- a/util-cache/src/main/scala/com/twitter/cache/EvictingCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/EvictingCache.scala @@ -5,9 +5,7 @@ import com.twitter.util.{Future, Throw} private[cache] class EvictingCache[K, V](underlying: FutureCache[K, V]) extends FutureCacheProxy[K, V](underlying) { private[this] def evictOnFailure(k: K, f: Future[V]): Future[V] = { - f.onFailure { _ => - evict(k, f) - } + f.onFailure { _ => evict(k, f) } f // we return the original future to make evict(k, f) easier to work with. } diff --git a/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala b/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala index 43a8bc1571..9646e664af 100644 --- a/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala @@ -107,9 +107,7 @@ object FutureCache { * @see [[standard]] for the equivalent Java API. */ def default[K, V](fn: K => Future[V], cache: FutureCache[K, V]): K => Future[V] = - AsyncMemoize(fn, new EvictingCache(cache)) andThen { f: Future[V] => - f.interruptible() - } + AsyncMemoize(fn, new EvictingCache(cache)) andThen { f: Future[V] => f.interruptible() } /** * Alias for [[default]] which can be called from Java. diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala b/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala index 75c7226e39..92e1eb1164 100644 --- a/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala @@ -52,9 +52,7 @@ object CaffeineCache { */ def fromLoadingCache[K, V](cache: LoadingCache[K, Future[V]]): K => Future[V] = { val evicting = EvictingCache.lazily(new LoadingFutureCache(cache)); - { key: K => - evicting.get(key).get.interruptible() - } + { key: K => evicting.get(key).get.interruptible() } } /** diff --git a/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala index 99bf6bed02..98bf953b2b 100644 --- a/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala @@ -10,8 +10,6 @@ class KeyEncodingCacheTest extends AbstractFutureCacheTest { val underlyingMap: ConcurrentHashMap[Int, Future[String]] = new ConcurrentHashMap() val underlyingCache: FutureCache[Int, String] = new ConcurrentMapCache(underlyingMap) val cache: FutureCache[String, String] = - new KeyEncodingCache({ num: String => - num.hashCode - }, underlyingCache) + new KeyEncodingCache({ num: String => num.hashCode }, underlyingCache) } } diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala index dfc3db8fa1..f48510da44 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala @@ -93,9 +93,7 @@ trait Offer[+T] { self => /** * Like {{map}}, but to a constant (call-by-name). */ - def const[U](f: => U): Offer[U] = map { _ => - f - } + def const[U](f: => U): Offer[U] = map { _ => f } /** * Java-friendly analog of `const()`. @@ -121,9 +119,7 @@ trait Offer[+T] { self => val ourTx = self.prepare() if (ourTx.isDefined) ourTx else { - ourTx.foreach { tx => - tx.nack() - } + ourTx.foreach { tx => tx.nack() } ourTx.raise(LostSynchronization) other.prepare() } diff --git a/util-core/src/main/scala-2.12-/com/twitter/util/HealthyQueue.scala b/util-core/src/main/scala-2.12-/com/twitter/util/HealthyQueue.scala index f6d8e1d0b2..04e56a656f 100644 --- a/util-core/src/main/scala-2.12-/com/twitter/util/HealthyQueue.scala +++ b/util-core/src/main/scala-2.12-/com/twitter/util/HealthyQueue.scala @@ -8,9 +8,7 @@ private[util] class HealthyQueue[A]( isHealthy: A => Boolean) extends mutable.Queue[Future[A]] { - 0.until(numItems) foreach { _ => - this += makeItem() - } + 0.until(numItems) foreach { _ => this += makeItem() } override def +=(elem: Future[A]): HealthyQueue.this.type = synchronized { super.+=(elem) diff --git a/util-core/src/main/scala-2.13+/com/twitter/util/HealthyQueue.scala b/util-core/src/main/scala-2.13+/com/twitter/util/HealthyQueue.scala index caef8b13a9..435b94d123 100644 --- a/util-core/src/main/scala-2.13+/com/twitter/util/HealthyQueue.scala +++ b/util-core/src/main/scala-2.13+/com/twitter/util/HealthyQueue.scala @@ -2,12 +2,13 @@ package com.twitter.util import scala.collection.mutable -private[util] class HealthyQueue[A](makeItem: () => Future[A], numItems: Int, isHealthy: A => Boolean) +private[util] class HealthyQueue[A]( + makeItem: () => Future[A], + numItems: Int, + isHealthy: A => Boolean) extends scala.collection.mutable.Queue[Future[A]] { - 0.until(numItems) foreach { _ => - this += makeItem() - } + 0.until(numItems) foreach { _ => this += makeItem() } override def addOne(elem: Future[A]): HealthyQueue.this.type = synchronized { super.addOne(elem) @@ -18,10 +19,10 @@ private[util] class HealthyQueue[A](makeItem: () => Future[A], numItems: Int, is } override def enqueue(elem: Future[A]) = synchronized { - super.enqueue(elem) + super.enqueue(elem) } override def enqueue(elem1: Future[A], elem2: Future[A], elems: Future[A]*) = synchronized { - super.enqueue(elem1, elem2, elems :_*) + super.enqueue(elem1, elem2, elems: _*) } override def dequeue(): Future[A] = synchronized { @@ -39,4 +40,4 @@ private[util] class HealthyQueue[A](makeItem: () => Future[A], numItems: Int, is } } } -} \ No newline at end of file +} diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala index b6a63104a3..af5a15b99d 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala @@ -138,11 +138,7 @@ object AsyncMeter { val seqWithoutLast: Seq[Future[Unit]] = (0 until num).map(_ => meter.await(meter.burstSize)) val seq = if (last == 0) seqWithoutLast else seqWithoutLast :+ meter.await(last) val result = Future.join(seq) - result.onFailure { exc => - seq.foreach { f: Future[Unit] => - f.raise(exc) - } - } + result.onFailure { exc => seq.foreach { f: Future[Unit] => f.raise(exc) } } result } else meter.await(permits) } diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala index 3b1f9bd05c..dd341de3dc 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala @@ -153,14 +153,15 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { */ def acquireAndRun[T](func: => Future[T]): Future[T] = acquire().flatMap { permit => - val f = try func - catch { - case NonFatal(e) => - Future.exception(e) - case e: Throwable => - permit.release() - throw e - } + val f = + try func + catch { + case NonFatal(e) => + Future.exception(e) + case e: Throwable => + permit.release() + throw e + } f.ensure { permit.release() } diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 4f70b2edd0..72975f3b9d 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -78,9 +78,7 @@ sealed abstract class AsyncStream[+A] { * Note: forces the stream. For infinite streams, the future never resolves. */ def foreach(f: A => Unit): Future[Unit] = - foldLeft(()) { (_, a) => - f(a) - } + foldLeft(()) { (_, a) => f(a) } /** * Execute the specified effect as each element of the resulting @@ -96,9 +94,7 @@ sealed abstract class AsyncStream[+A] { * whether the entire stream is consumed. */ def withEffect(f: A => Unit): AsyncStream[A] = - map { a => - f(a); a - } + map { a => f(a); a } /** * Maps each element of the stream to a Future action, resolving them from @@ -108,9 +104,7 @@ sealed abstract class AsyncStream[+A] { * Note: forces the stream. For infinite streams, the future never resolves. */ def foreachF(f: A => Future[Unit]): Future[Unit] = - foldLeftF(()) { (_, a) => - f(a) - } + foldLeftF(()) { (_, a) => f(a) } /** * Map over this stream with the given concurrency. The items will likely be @@ -151,9 +145,7 @@ sealed abstract class AsyncStream[+A] { this match { case Empty => empty case FromFuture(fa) => - Embed(fa.map { a => - if (p(a)) this else empty - }) + Embed(fa.map { a => if (p(a)) this else empty }) case Cons(fa, more) => Embed(fa.map { a => if (p(a)) Cons(fa, () => more().takeWhile(p)) @@ -175,9 +167,7 @@ sealed abstract class AsyncStream[+A] { this match { case Empty => empty case FromFuture(fa) => - Embed(fa.map { a => - if (p(a)) empty else this - }) + Embed(fa.map { a => if (p(a)) empty else this }) case Cons(fa, more) => Embed(fa.map { a => if (p(a)) more().dropWhile(p) @@ -253,9 +243,7 @@ sealed abstract class AsyncStream[+A] { this match { case Empty => empty case FromFuture(fa) => - Embed(fa.map { a => - if (p(a)) this else empty - }) + Embed(fa.map { a => if (p(a)) this else empty }) case Cons(fa, more) => Embed(fa.map { a => if (p(a)) Cons(fa, () => more().filter(p)) @@ -506,7 +494,8 @@ sealed abstract class AsyncStream[+A] { def fillBuffer( sizeRemaining: Int - )(s: => AsyncStream[A] + )( + s: => AsyncStream[A] ): Future[(Seq[A], () => AsyncStream[A])] = if (sizeRemaining < 1) Future.value((buffer.result, () => s)) else @@ -587,8 +576,7 @@ sealed abstract class AsyncStream[+A] { * stream to occur, but do not need to do anything with the resulting * values. */ - def force: Future[Unit] = foreach { _ => - } + def force: Future[Unit] = foreach { _ => } } object AsyncStream { @@ -606,7 +594,7 @@ object AsyncStream { new Cons(fa, next) def unapply[A](as: Cons[A]): Option[(Future[A], () => AsyncStream[A])] = - // note: pattern match returns the memoized value + // note: pattern match returns the memoized value Some((as.fa, () => as.more())) } diff --git a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala index 1f4c422ff2..c56a55c71e 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala @@ -160,9 +160,7 @@ sealed trait Spool[+A] { def mapFuture[B](f: A => Future[B]): Future[Spool[B]] = { if (isEmpty) Future.value(empty[B]) else { - f(head) map { h => - new LazyCons(h, tail flatMap (_ mapFuture f)) - } + f(head) map { h => new LazyCons(h, tail flatMap (_ mapFuture f)) } } } @@ -263,18 +261,14 @@ sealed trait Spool[+A] { val as = Vector.newBuilder[A] foreach { a => as += a - } map { _ => - as.result - } + } map { _ => as.result } } /** * Eagerly executes all computation represented by this Spool (presumably for * side-effects), and returns a Future representing its completion. */ - def force: Future[Unit] = foreach { _ => - () - } + def force: Future[Unit] = foreach { _ => () } } /** diff --git a/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala b/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala index 92ecdc2c70..e11160a4fc 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/SpoolSource.scala @@ -108,7 +108,8 @@ class SpoolSource[A](interruptHandler: PartialFunction[Throwable, Unit]) { @tailrec private[this] def updatingTailCall( newPromise: Promise[Spool[A]] - )(f: Promise[Spool[A]] => Unit + )( + f: Promise[Spool[A]] => Unit ): Unit = { val currentPromise = promiseRef.get // if the current promise is emptyPromise, then this source has already been closed diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 8a2f3b0af6..1d1774a684 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -995,7 +995,7 @@ object Buf { // Adapted from util-hashing. private[this] val UintMax: Long = 0xFFFFFFFFL private[this] val Fnv1a32Prime: Int = 16777619 - private[this] val Fnv1a32Init: Long = 0x811c9dc5L + private[this] val Fnv1a32Init: Long = 0x811C9DC5L private[this] def finishHash(hash: Long): Int = (hash & UintMax).toInt private[this] def hashBuf(buf: Buf, init: Long = Fnv1a32Init): Long = buf match { case b if b.isEmpty => init diff --git a/util-core/src/main/scala/com/twitter/io/ByteReader.scala b/util-core/src/main/scala/com/twitter/io/ByteReader.scala index 3d71044d35..fd39b231f5 100644 --- a/util-core/src/main/scala/com/twitter/io/ByteReader.scala +++ b/util-core/src/main/scala/com/twitter/io/ByteReader.scala @@ -425,9 +425,9 @@ private class ByteReaderImpl(buf: Buf) extends ByteReader { ret } - def readUnsignedIntBE(): Long = readIntBE() & 0xffffffffL + def readUnsignedIntBE(): Long = readIntBE() & 0xFFFFFFFFL - def readUnsignedIntLE(): Long = readIntLE() & 0xffffffffL + def readUnsignedIntLE(): Long = readIntLE() & 0xFFFFFFFFL // - Long - def readLongBE(): Long = { diff --git a/util-core/src/main/scala/com/twitter/io/Files.scala b/util-core/src/main/scala/com/twitter/io/Files.scala index a2dcaf9e72..e8263eb317 100644 --- a/util-core/src/main/scala/com/twitter/io/Files.scala +++ b/util-core/src/main/scala/com/twitter/io/Files.scala @@ -42,9 +42,7 @@ object Files { } else if (file.isFile) { file.delete() } else { - file.listFiles.foreach { f => - delete(f) - } + file.listFiles.foreach { f => delete(f) } file.delete() } } diff --git a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala b/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala index ed939121b7..82599622fe 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala +++ b/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala @@ -83,9 +83,7 @@ class CachingActivitySource[T](underlying: ActivitySource[T]) extends ActivitySo */ def get(name: String): Activity[T] = synchronized { gc() - Option(forward.get(name)) flatMap { wr => - Option(wr.get()) - } match { + Option(forward.get(name)) flatMap { wr => Option(wr.get()) } match { case Some(v) => v case None => val v = underlying.get(name) @@ -153,9 +151,7 @@ class FilePollingActivitySource private[exp] ( } } - Closable.make { _ => - Future { timerTask.cancel() } - } + Closable.make { _ => Future { timerTask.cancel() } } } Activity(v) diff --git a/util-core/src/main/scala/com/twitter/util/Activity.scala b/util-core/src/main/scala/com/twitter/util/Activity.scala index 0e3e475861..68a814e8af 100644 --- a/util-core/src/main/scala/com/twitter/util/Activity.scala +++ b/util-core/src/main/scala/com/twitter/util/Activity.scala @@ -87,10 +87,11 @@ case class Activity[+T](run: Var[Activity.State[T]]) { def flatMap[U](f: T => Activity[U]): Activity[U] = Activity(run flatMap { case Ok(v) => - val a = try f(v) - catch { - case NonFatal(exc) => Activity.exception(exc) - } + val a = + try f(v) + catch { + case NonFatal(exc) => Activity.exception(exc) + } a.run case Pending => Var.value(Activity.Pending) @@ -103,10 +104,11 @@ case class Activity[+T](run: Var[Activity.State[T]]) { */ def transform[U](f: Activity.State[T] => Activity[U]): Activity[U] = Activity(run flatMap { act => - val a = try f(act) - catch { - case NonFatal(exc) => Activity.exception(exc) - } + val a = + try f(act) + catch { + case NonFatal(exc) => Activity.exception(exc) + } a.run }) diff --git a/util-core/src/main/scala/com/twitter/util/Base64Long.scala b/util-core/src/main/scala/com/twitter/util/Base64Long.scala index 0d4af19c35..94e5bc5577 100644 --- a/util-core/src/main/scala/com/twitter/util/Base64Long.scala +++ b/util-core/src/main/scala/com/twitter/util/Base64Long.scala @@ -113,9 +113,7 @@ object Base64Long { new PartialFunction[Char, Int] { private[this] val maxChar = chars.max private[this] val reverse = Array.fill[Byte](maxChar.toInt + 1)(-1) - 0.until(AlphabetSize).foreach { i => - reverse(forward(i)) = i.toByte - } + 0.until(AlphabetSize).foreach { i => reverse(forward(i)) = i.toByte } def isDefinedAt(c: Char): Boolean = (c <= maxChar) && (reverse(c) != -1) def apply(c: Char): Int = reverse(c) } diff --git a/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala b/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala index 26e5c2deaf..b89be54dca 100644 --- a/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala +++ b/util-core/src/main/scala/com/twitter/util/BatchExecutor.scala @@ -92,8 +92,7 @@ private[util] class BatchExecutor[In, Out]( flushBatch() else { scheduleFlushIfNecessary() - () => - () + () => () } } diff --git a/util-core/src/main/scala/com/twitter/util/Closable.scala b/util-core/src/main/scala/com/twitter/util/Closable.scala index 7c06def274..d909bfc6d5 100644 --- a/util-core/src/main/scala/com/twitter/util/Closable.scala +++ b/util-core/src/main/scala/com/twitter/util/Closable.scala @@ -50,10 +50,9 @@ object Closable { /** Stop CollectClosables thread. */ def stopCollectClosablesThread(): Unit = { if (collectorThreadEnabled.compareAndSet(true, false)) - Try(collectorThread.interrupt).onFailure( - fatal => - logger - .log(Level.SEVERE, "Current thread cannot interrupt CollectClosables thread", fatal))() + Try(collectorThread.interrupt).onFailure(fatal => + logger + .log(Level.SEVERE, "Current thread cannot interrupt CollectClosables thread", fatal))() } /** Provide Java access to the [[com.twitter.util.Closable]] mixin. */ @@ -84,9 +83,7 @@ object Closable { */ @varargs def all(closables: Closable*): Closable = new Closable { def close(deadline: Time): Future[Unit] = { - val fs = closables.map { closable => - safeClose(closable, deadline) - } + val fs = closables.map { closable => safeClose(closable, deadline) } val iter = fs.iterator @tailrec def checkNext(): Future[Unit] = { diff --git a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala index bc8ddbd568..97097ba746 100644 --- a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala @@ -54,14 +54,15 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { val current = Local.save() if (current ne saved) Local.restore(saved) - val computed = try f(result) - catch { - case e: NonLocalReturnControl[_] => Future.exception(new FutureNonLocalReturnControl(e)) - case scala.util.control.NonFatal(e) => Future.exception(e) - case t: Throwable => - Monitor.handle(t) - throw t - } finally Local.restore(current) + val computed = + try f(result) + catch { + case e: NonLocalReturnControl[_] => Future.exception(new FutureNonLocalReturnControl(e)) + case scala.util.control.NonFatal(e) => Future.exception(e) + case t: Throwable => + Monitor.handle(t) + throw t + } finally Local.restore(current) p.become(computed) } }) diff --git a/util-core/src/main/scala/com/twitter/util/Credentials.scala b/util-core/src/main/scala/com/twitter/util/Credentials.scala index fba84ee62f..4e125dbe45 100644 --- a/util-core/src/main/scala/com/twitter/util/Credentials.scala +++ b/util-core/src/main/scala/com/twitter/util/Credentials.scala @@ -38,9 +38,7 @@ object Credentials { private[this] val value = ".+".r def auth: Parser[Tuple2[String, String]] = key ~ ":" ~ value ^^ { case k ~ ":" ~ v => (k, v) } - def content: Parser[Map[String, String]] = rep(auth) ^^ { auths => - Map(auths: _*) - } + def content: Parser[Map[String, String]] = rep(auth) ^^ { auths => Map(auths: _*) } def apply(in: String): Map[String, String] = { parseAll(content, in) match { diff --git a/util-core/src/main/scala/com/twitter/util/Diff.scala b/util-core/src/main/scala/com/twitter/util/Diff.scala index b1818c5175..6d1b972709 100644 --- a/util-core/src/main/scala/com/twitter/util/Diff.scala +++ b/util-core/src/main/scala/com/twitter/util/Diff.scala @@ -104,8 +104,9 @@ object Diffable { } else if (left.length > right.length) { diff(left.take(right.length), right) } else { - val insert = for (((x, y), i) <- left.zip(right).zipWithIndex if x != y) - yield (i -> y) + val insert = + for (((x, y), i) <- left.zip(right).zipWithIndex if x != y) + yield (i -> y) SeqDiff(left.length, insert.toMap) } diff --git a/util-core/src/main/scala/com/twitter/util/Disposable.scala b/util-core/src/main/scala/com/twitter/util/Disposable.scala index 9c3679657e..d2b502a0f0 100644 --- a/util-core/src/main/scala/com/twitter/util/Disposable.scala +++ b/util-core/src/main/scala/com/twitter/util/Disposable.scala @@ -97,13 +97,14 @@ trait Managed[+T] { selfT => def make() = new Disposable[U] { val t = selfT.make() - val u = try { - f(t.get).make() - } catch { - case e: Exception => - t.dispose() - throw e - } + val u = + try { + f(t.get).make() + } catch { + case e: Exception => + t.dispose() + throw e + } def get = u.get @@ -120,9 +121,7 @@ trait Managed[+T] { selfT => } } - def map[U](f: T => U): Managed[U] = flatMap { t => - Managed.const(f(t)) - } + def map[U](f: T => U): Managed[U] = flatMap { t => Managed.const(f(t)) } /** * Builds a resource. diff --git a/util-core/src/main/scala/com/twitter/util/Event.scala b/util-core/src/main/scala/com/twitter/util/Event.scala index 4dda24b9ec..ea6701aace 100644 --- a/util-core/src/main/scala/com/twitter/util/Event.scala +++ b/util-core/src/main/scala/com/twitter/util/Event.scala @@ -86,9 +86,7 @@ trait Event[+T] { self => */ def collect[U](f: PartialFunction[T, U]): Event[U] = new Event[U] { def register(s: Witness[U]): Closable = - self.respond { t => - f.runWith(s.notify)(t) - } + self.respond { t => f.runWith(s.notify)(t) } } /** @@ -153,9 +151,7 @@ trait Event[+T] { self => def mergeMap[U](f: T => Event[U]): Event[U] = new Event[U] { def register(s: Witness[U]): Closable = { @volatile var inners = Nil: List[Closable] - val outer = self.respond { el => - inners.synchronized { inners ::= f(el).register(s) } - } + val outer = self.respond { el => inners.synchronized { inners ::= f(el).register(s) } } Closable.make { deadline => outer.close(deadline) before { @@ -170,12 +166,8 @@ trait Event[+T] { self => */ def select[U](other: Event[U]): Event[Either[T, U]] = new Event[Either[T, U]] { def register(s: Witness[Either[T, U]]): Closable = Closable.all( - self.register(s.comap { t => - Left(t) - }), - other.register(s.comap { u => - Right(u) - }) + self.register(s.comap { t => Left(t) }), + other.register(s.comap { u => Right(u) }) ) } @@ -323,7 +315,8 @@ trait Event[+T] { self => * builder. A value containing the current version of the collection * is notified for each incoming event. */ - def build[U >: T, That <: Seq[U]](implicit factory: Factory[U, That]): Event[That] = buildAny(factory) + def build[U >: T, That <: Seq[U]](implicit factory: Factory[U, That]): Event[That] = + buildAny(factory) /** * A Future which is satisfied by the first value observed. @@ -391,9 +384,7 @@ trait Event[+T] { self => * Builds a new Event by keeping only the Events where * the previous and current values are not `==` to each other. */ - def dedup: Event[T] = dedupWith { (a, b) => - a == b - } + def dedup: Event[T] = dedupWith { (a, b) => a == b } } /** diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 7c52ed1788..68ce681a9f 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -2,11 +2,7 @@ package com.twitter.util import com.twitter.concurrent.{Offer, Tx} import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} -import java.util.concurrent.{ - CancellationException, - CompletableFuture, - Future => JavaFuture -} +import java.util.concurrent.{CancellationException, CompletableFuture, Future => JavaFuture} import java.util.function.BiConsumer import java.util.{List => JList} import scala.collection.compat.immutable.ArraySeq @@ -318,9 +314,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * do. */ def join[A, B, C](a: Future[A], b: Future[B], c: Future[C]): Future[(A, B, C)] = - join(Seq(a, b, c)).map { _ => - (Await.result(a), Await.result(b), Await.result(c)) - } + join(Seq(a, b, c)).map { _ => (Await.result(a), Await.result(b), Await.result(c)) } /** * Join 4 futures. The returned future is complete when all @@ -1140,9 +1134,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( if (fs.isEmpty) emptyMap else { val (keys, values) = fs.toSeq.unzip - Future.collect(values).map { seq => - keys.iterator.zip(seq.iterator).toMap: Map[A, B] - } + Future.collect(values).map { seq => keys.iterator.zip(seq.iterator).toMap: Map[A, B] } } /** @@ -1329,9 +1321,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( def whileDo[A](p: => Boolean)(f: => Future[A]): Future[Unit] = { def loop(): Future[Unit] = { if (p) { - f.flatMap { _ => - loop() - } + f.flatMap { _ => loop() } } else Future.Unit } @@ -1415,7 +1405,8 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( sizeThreshold: Int, timeThreshold: Duration = Duration.Top, sizePercentile: => Float = 1.0f - )(f: AnySeq[In] => Future[Seq[Out]] + )( + f: AnySeq[In] => Future[Seq[Out]] )( implicit timer: Timer ): Batcher[In, Out] = { @@ -1527,9 +1518,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * @see [[respond]] if you need the result of the computation for * usage in the side-effect. */ - def ensure(f: => Unit): Future[A] = respond { _ => - f - } + def ensure(f: => Unit): Future[A] = respond { _ => f } /** * Is the result of the Future available yet? @@ -2044,12 +2033,8 @@ abstract class Future[+A] extends Awaitable[A] { self => val p = Promise.interrupts[U](other, this) val a = Promise.attached(other) val b = Promise.attached(this) - a.respond { t => - if (p.updateIfEmpty(t)) b.detach() - } - b.respond { t => - if (p.updateIfEmpty(t)) a.detach() - } + a.respond { t => if (p.updateIfEmpty(t)) b.detach() } + b.respond { t => if (p.updateIfEmpty(t)) a.detach() } p } @@ -2139,9 +2124,7 @@ abstract class Future[+A] extends Awaitable[A] { self => s"Cannot call proxyTo on an already satisfied Promise: ${Await.result(other.liftToTry)}" ) } - respond { res => - other.update(res) - } + respond { res => other.update(res) } } /** @@ -2270,9 +2253,7 @@ abstract class Future[+A] extends Awaitable[A] { self => */ def willEqual[B](that: Future[B]): Future[Boolean] = { this.transform { thisResult => - that.transform { thatResult => - Future.value(thisResult == thatResult) - } + that.transform { thatResult => Future.value(thisResult == thatResult) } } } diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index ab730ef3d7..a56f723a6a 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -153,13 +153,14 @@ class ExecutorServiceFuturePool protected[this] ( // This is safe: the only thing that can call task.run() is // executor, the only thing that can raise an interrupt is the // receiver of this value, which will then be fully initialized. - val javaFuture = try executor.submit(task) - catch { - case e: RejectedExecutionException => - runOk.set(false) - p.setException(e) - null - } + val javaFuture = + try executor.submit(task) + catch { + case e: RejectedExecutionException => + runOk.set(false) + p.setException(e) + null + } p.setInterruptHandler { case cause => diff --git a/util-core/src/main/scala/com/twitter/util/Futures.scala b/util-core/src/main/scala/com/twitter/util/Futures.scala index 21184652bb..67d0199444 100644 --- a/util-core/src/main/scala/com/twitter/util/Futures.scala +++ b/util-core/src/main/scala/com/twitter/util/Futures.scala @@ -33,9 +33,8 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * underlying futures complete. It fails immediately if any of them * do. */ - def join[A, B](a: Future[A], b: Future[B]): Future[(A, B)] = Future.join(Seq(a, b)).map { _ => - (Await.result(a), Await.result(b)) - } + def join[A, B](a: Future[A], b: Future[B]): Future[(A, B)] = + Future.join(Seq(a, b)).map { _ => (Await.result(a), Await.result(b)) } /** * Join 3 futures. The returned future is complete when all @@ -43,9 +42,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * do. */ def join[A, B, C](a: Future[A], b: Future[B], c: Future[C]): Future[(A, B, C)] = - Future.join(Seq(a, b, c)).map { _ => - (Await.result(a), Await.result(b), Await.result(c)) - } + Future.join(Seq(a, b, c)).map { _ => (Await.result(a), Await.result(b), Await.result(c)) } /** * Join 4 futures. The returned future is complete when all diff --git a/util-core/src/main/scala/com/twitter/util/Local.scala b/util-core/src/main/scala/com/twitter/util/Local.scala index e11f7bad37..8f1b165926 100644 --- a/util-core/src/main/scala/com/twitter/util/Local.scala +++ b/util-core/src/main/scala/com/twitter/util/Local.scala @@ -4918,13 +4918,12 @@ object Local { */ def closed[R](fn: () => R): () => R = { val closure = Local.save() - () => - { - val save = Local.save() - Local.restore(closure) - try fn() - finally Local.restore(save) - } + () => { + val save = Local.save() + Local.restore(closure) + try fn() + finally Local.restore(save) + } } } diff --git a/util-core/src/main/scala/com/twitter/util/NetUtil.scala b/util-core/src/main/scala/com/twitter/util/NetUtil.scala index 4c87bd7710..86de343726 100644 --- a/util-core/src/main/scala/com/twitter/util/NetUtil.scala +++ b/util-core/src/main/scala/com/twitter/util/NetUtil.scala @@ -131,9 +131,7 @@ object NetUtil { isIpInBlock(inetAddressToInt(inetAddress), ipBlock) def isIpInBlocks(ip: Int, ipBlocks: Iterable[(Int, Int)]): Boolean = { - ipBlocks exists { ipBlock => - isIpInBlock(ip, ipBlock) - } + ipBlocks exists { ipBlock => isIpInBlock(ip, ipBlock) } } def isIpInBlocks(ip: String, ipBlocks: Iterable[(Int, Int)]): Boolean = { diff --git a/util-core/src/main/scala/com/twitter/util/Signal.scala b/util-core/src/main/scala/com/twitter/util/Signal.scala index 96de468b46..333af1588d 100644 --- a/util-core/src/main/scala/com/twitter/util/Signal.scala +++ b/util-core/src/main/scala/com/twitter/util/Signal.scala @@ -58,9 +58,7 @@ class SunSignalHandler extends SignalHandler { new InvocationHandler { def invoke(proxy: Object, method: Method, args: Array[Object]) = { if (method.getName() == "handle") { - handlers(signal).foreach { x => - x(nameMethod.invoke(args(0)).asInstanceOf[String]) - } + handlers(signal).foreach { x => x(nameMethod.invoke(args(0)).asInstanceOf[String]) } } null } diff --git a/util-core/src/main/scala/com/twitter/util/SlowProbeProxyTimer.scala b/util-core/src/main/scala/com/twitter/util/SlowProbeProxyTimer.scala index 4a4fa448be..1dbe4ef280 100644 --- a/util-core/src/main/scala/com/twitter/util/SlowProbeProxyTimer.scala +++ b/util-core/src/main/scala/com/twitter/util/SlowProbeProxyTimer.scala @@ -48,7 +48,8 @@ abstract class SlowProbeProxyTimer(maxRuntime: Duration) extends ProxyTimer { override protected def schedulePeriodically( when: Time, period: Duration - )(f: => Unit + )( + f: => Unit ): TimerTask = { checkSlowTask() self.schedule(when, period)(meterTask(f)) diff --git a/util-core/src/main/scala/com/twitter/util/Stopwatch.scala b/util-core/src/main/scala/com/twitter/util/Stopwatch.scala index fc96d2c6d0..acf756c333 100644 --- a/util-core/src/main/scala/com/twitter/util/Stopwatch.scala +++ b/util-core/src/main/scala/com/twitter/util/Stopwatch.scala @@ -47,7 +47,7 @@ object Stopwatch extends Stopwatch { Time.localGetTime() match { case Some(local) => local().inNanoseconds case None => systemNanos() - } + } /** * A function that returns a Long that can be used for measuring elapsed time @@ -71,7 +71,7 @@ object Stopwatch extends Stopwatch { Time.localGetTime() match { case Some(local) => local().inMicroseconds case None => systemMicros() - } + } /** * A function that returns a Long that can be used for measuring elapsed time @@ -95,17 +95,15 @@ object Stopwatch extends Stopwatch { Time.localGetTime() match { case Some(local) => local().inMilliseconds case None => systemMillis() - } + } def start(): Elapsed = Time.localGetTime() match { case Some(local) => val startAt: Time = local() - () => - local() - startAt + () => local() - startAt case None => val startAt: Long = systemNanos() - () => - Duration.fromNanoseconds(systemNanos() - startAt) + () => Duration.fromNanoseconds(systemNanos() - startAt) } /** diff --git a/util-core/src/main/scala/com/twitter/util/Timer.scala b/util-core/src/main/scala/com/twitter/util/Timer.scala index 6731d7eb18..bb4f1328c0 100644 --- a/util-core/src/main/scala/com/twitter/util/Timer.scala +++ b/util-core/src/main/scala/com/twitter/util/Timer.scala @@ -366,9 +366,7 @@ class MockTimer extends Timer { throw new IllegalStateException("timer is stopped already") val now = Time.now - val (toRun, toQueue) = tasks.partition { task => - task.when <= now - } + val (toRun, toQueue) = tasks.partition { task => task.when <= now } tasks = toQueue toRun.filterNot(_.isCancelled).foreach(_.runner()) } diff --git a/util-core/src/main/scala/com/twitter/util/Try.scala b/util-core/src/main/scala/com/twitter/util/Try.scala index 883e881a15..252ca36b24 100644 --- a/util-core/src/main/scala/com/twitter/util/Try.scala +++ b/util-core/src/main/scala/com/twitter/util/Try.scala @@ -23,7 +23,8 @@ object Try { val Void: Try[Void] = Try(null: Void) def apply[R](r: => R): Try[R] = { - try { Return(r) } catch { + try { Return(r) } + catch { case NonFatal(e) => Throw(e) } } @@ -57,9 +58,7 @@ object Try { if (ts.isEmpty) Return(Seq.empty[A]) else Try { - ts map { t => - t() - } + ts map { t => t() } } } @@ -207,9 +206,7 @@ sealed abstract class Try[+R] { * chained `this` as in `respond`. */ def ensure(f: => Unit): Try[R] = - respond { _ => - f - } + respond { _ => f } /** * Returns None if this is a Throw or a Some containing the value if this is a Return diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index a9b4e93fcc..1cbbd17018 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -132,9 +132,7 @@ trait Var[+T] { self => */ lazy val changes: Event[T] = new Event[T] { def register(s: Witness[T]) = - self.observe { newv => - s.notify(newv) - } + self.observe { newv => s.notify(newv) } } /** diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 7acdb373c5..0ac81b6e04 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -33,10 +33,11 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { def genLongStream(len: Int): AsyncStream[Int] = if (len == 0) { AsyncStream.of(1) - } else AsyncStream.of(1).flatMap { i => - count = count + 1 - i +:: genLongStream(len - 1) - } + } else + AsyncStream.of(1).flatMap { i => + count = count + 1 + i +:: genLongStream(len - 1) + } // a long stream created via flatMap does not stack overflow val s = genLongStream(1000000) ++ genLongStream(3) @@ -167,14 +168,10 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { s.uncons assert(!p.isDefined) - s.foldRight(Future.Done) { (_, _) => - Future.Done - } + s.foldRight(Future.Done) { (_, _) => Future.Done } assert(!p.isDefined) - s.scanLeft(Future.Done) { (_, _) => - Future.Done - } + s.scanLeft(Future.Done) { (_, _) => Future.Done } assert(!p.isDefined) s ++ s @@ -209,12 +206,7 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { new Ctx(s => s.withFilter(_ => true)) new Ctx(s => s.take(2)) new Ctx(s => s.takeWhile(_ => true)) - new Ctx( - s => - s.scanLeft(Future.Done) { (_, _) => - Future.Done - } - ) + new Ctx(s => s.scanLeft(Future.Done) { (_, _) => Future.Done }) new Ctx(s => s ++ s) } @@ -256,9 +248,7 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } test(s"$impl: ++") { - forAll { (a: List[Int], b: List[Int]) => - assert(toSeq(fromSeq(a) ++ fromSeq(b)) == a ++ b) - } + forAll { (a: List[Int], b: List[Int]) => assert(toSeq(fromSeq(a) ++ fromSeq(b)) == a ++ b) } } test(s"$impl: foldRight") { @@ -308,15 +298,11 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test(s"$impl: flatten") { val small = Gen.resize(10, Arbitrary.arbitrary[List[List[Int]]]) - forAll(small) { s => - assert(toSeq(fromSeq(s.map(fromSeq)).flatten) == s.flatten) - } + forAll(small) { s => assert(toSeq(fromSeq(s.map(fromSeq)).flatten) == s.flatten) } } test(s"$impl: head") { - forAll { (a: List[Int]) => - assert(await(fromSeq(a).head) == a.headOption) - } + forAll { (a: List[Int]) => assert(await(fromSeq(a).head) == a.headOption) } } test(s"$impl: isEmpty") { @@ -373,9 +359,7 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } test(s"$impl: toSeq") { - forAll { (as: List[Int]) => - assert(await(fromSeq(as).toSeq()) == as) - } + forAll { (as: List[Int]) => assert(await(fromSeq(as).toSeq()) == as) } } test(s"$impl: identity") { @@ -647,15 +631,11 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } test(s"$impl: sum") { - forAll { xs: List[Int] => - assert(xs.sum == await(fromSeq(xs).sum)) - } + forAll { xs: List[Int] => assert(xs.sum == await(fromSeq(xs).sum)) } } test(s"$impl: size") { - forAll { xs: List[Int] => - assert(xs.size == await(fromSeq(xs).size)) - } + forAll { xs: List[Int] => assert(xs.size == await(fromSeq(xs).size)) } } test(s"$impl: force") { @@ -763,9 +743,7 @@ class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { case Left(a) => tailRecM(a)(f) } - val stream = tailRecM(0) { i => - AsyncStream.of(if (i < n) Left(i + 1) else Right(i)) - } + val stream = tailRecM(0) { i => AsyncStream.of(if (i < n) Left(i + 1) else Right(i)) } assert(Await.result(stream.toSeq) == Seq(n)) } diff --git a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala index 69dac52d1f..971ed45051 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala @@ -35,13 +35,9 @@ class OfferTest extends WordSpec with MockitoSugar { assert(tx.ack() == (result)) val offer = spy(new SimpleOffer(tx)) - val mapped = offer map { i => - (i - 100).toString - } + val mapped = offer map { i => (i - 100).toString } - val f = mapped.prepare() flatMap { tx => - tx.ack() - } + val f = mapped.prepare() flatMap { tx => tx.ack() } assert(await(f) == Commit("23")) } @@ -49,12 +45,8 @@ class OfferTest extends WordSpec with MockitoSugar { "Offer.choose" should { class OfferSpecHelper { - val pendingTxs = 0 until 3 map { _ => - new Promise[Tx[Int]] - } - val offers = pendingTxs map { tx => - spy(new SimpleOffer(tx)) - } + val pendingTxs = 0 until 3 map { _ => new Promise[Tx[Int]] } + val offers = pendingTxs map { tx => spy(new SimpleOffer(tx)) } val offer = Offer.choose(offers: _*) } @@ -68,13 +60,9 @@ class OfferTest extends WordSpec with MockitoSugar { val h = new TxReadyHelper import h._ - offers foreach { of => - verify(of, never()).prepare() - } + offers foreach { of => verify(of, never()).prepare() } assert(offer.prepare().isDefined == true) - offers foreach { of => - verify(of).prepare() - } + offers foreach { of => verify(of).prepare() } } "select it" in { @@ -336,9 +324,7 @@ class OfferTest extends WordSpec with MockitoSugar { "synchronize on offers forever" in { val b = new Broker[Int] var count = 0 - b.recv foreach { _ => - count += 1 - } + b.recv foreach { _ => count += 1 } assert(count == 0) assert(b.send(1).sync().isDefined == true) assert(count == 1) @@ -362,12 +348,8 @@ class OfferTest extends WordSpec with MockitoSugar { "cancel timer tasks when losing" in Time.withTimeAt(Time.epoch) { tc => implicit val timer = new MockTimer - val e10 = Offer.timeout(10.seconds) map { _ => - 10 - } - val e5 = Offer.timeout(5.seconds) map { _ => - 5 - } + val e10 = Offer.timeout(10.seconds) map { _ => 10 } + val e5 = Offer.timeout(5.seconds) map { _ => 5 } val item = Offer.select(e5, e10) assert(item.poll == None) diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala index 787b03253a..ae4824f8fb 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala @@ -61,16 +61,12 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { } "fold left" in { - val fold = s.foldLeft(0) { (x, y) => - x + y - } + val fold = s.foldLeft(0) { (x, y) => x + y } assert(await(fold) == 0) } "reduce left" in { - val fold = s.reduceLeft { (x, y) => - x + y - } + val fold = s.reduceLeft { (x, y) => x + y } intercept[UnsupportedOperationException] { await(fold) } @@ -146,16 +142,12 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { } "fold left" in { - val fold = s.foldLeft(0) { (x, y) => - x + y - } + val fold = s.foldLeft(0) { (x, y) => x + y } assert(await(fold) == 3) } "reduce left" in { - val fold = s.reduceLeft { (x, y) => - x + y - } + val fold = s.reduceLeft { (x, y) => x + y } assert(await(fold) == 3) } @@ -184,9 +176,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { val seq = await(spool.toSeq) val flatSpool = - spool.flatMap { inner => - Future.value(inner.toSpool) - } + spool.flatMap { inner => Future.value(inner.toSpool) } assert(await(flatSpool.flatMap(_.toSeq)) == seq.flatten) } @@ -225,9 +215,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { "return with exception on error in callback" in { val xs = new ArrayBuffer[Option[Int]] - val f = s foreach { _ => - throw new Exception("sad panda") - } + val f = s foreach { _ => throw new Exception("sad panda") } intercept[Exception] { await(f) } @@ -235,9 +223,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { "return with exception on EOFException in callback" in { val xs = new ArrayBuffer[Option[Int]] - val f = s foreach { _ => - throw new EOFException("sad panda") - } + val f = s foreach { _ => throw new EOFException("sad panda") } intercept[EOFException] { await(f) } @@ -294,9 +280,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { import h._ val xs = new ArrayBuffer[Option[Int]] - val f = s foreach { _ => - throw new Exception("sad panda") - } + val f = s foreach { _ => throw new Exception("sad panda") } p() = Return(2 *:: p1) intercept[Exception] { await(f) @@ -308,9 +292,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { import h._ val xs = new ArrayBuffer[Option[Int]] - val f = s foreach { _ => - throw new EOFException("sad panda") - } + val f = s foreach { _ => throw new EOFException("sad panda") } p() = Return(2 *:: p1) intercept[EOFException] { await(f) @@ -371,9 +353,7 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { val h = new SimpleDelayedSpoolHelper import h._ - val f = s.foldLeft(0) { (x, y) => - x + y - } + val f = s.foldLeft(0) { (x, y) => x + y } assert(f.isDefined == false) p() = Return(2 *:: p1) @@ -420,23 +400,15 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { } "map lazily" in { - applyLazily { spool => - Future.value(spool.map(_ + 1)) - } + applyLazily { spool => Future.value(spool.map(_ + 1)) } } "mapFuture lazily" in { - applyLazily { spool => - spool.mapFuture(Future.value(_)) - } + applyLazily { spool => spool.mapFuture(Future.value(_)) } } "flatMap lazily" in { - applyLazily { spool => - spool.flatMap { item => - Future.value((item to (item + 5)).toSpool) - } - } + applyLazily { spool => spool.flatMap { item => Future.value((item to (item + 5)).toSpool) } } } "takeWhile lazily" in { @@ -456,16 +428,12 @@ class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { } "zip lazily" in { - applyLazily { spool => - Future.value(spool.zip(spool).map { case (a, b) => a + b }) - } + applyLazily { spool => Future.value(spool.zip(spool).map { case (a, b) => a + b }) } } "act eagerly when forced" in { val (spool, tailReached) = - applyLazily { spool => - Future.value(spool.map(_ + 1)) - } + applyLazily { spool => Future.value(spool.map(_ + 1)) } Await.ready { spool.map(_.force) } assert(tailReached.isDefined) } diff --git a/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala index 118d4d3829..10073e2d7a 100644 --- a/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala @@ -32,12 +32,10 @@ class PercentOpsTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } test("assorted percentages") { - forAll(arbitrary[Int]) { i => - assert(new RichPercent(i).percent == i / 100.0) - } + forAll(arbitrary[Int]) { i => assert(new RichPercent(i).percent == i / 100.0) } // We're not as accurate when we get into high double-digit exponents, but that's acceptable. - forAll(Gen.choose(-10000D, 10000D)) { d => + forAll(Gen.choose(-10000d, 10000d)) { d => assert(doubleEq(new RichPercent(d).percent, d / 100.0)) } } diff --git a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala index db43061cbc..25afa098c0 100644 --- a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala @@ -7,9 +7,7 @@ class U64OpsTest extends FunSuite with ScalaCheckDrivenPropertyChecks { import com.twitter.conversions.U64Ops._ test("toU64HextString") { - forAll { l: Long => - assert(l.toU64HexString == "%016x".format(l)) - } + forAll { l: Long => assert(l.toU64HexString == "%016x".format(l)) } } test("toU64Long") { diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index d149709d91..be11594823 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -91,9 +91,7 @@ class BufReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { val r = BufReader.framed(BufReader(Buf(buffersWithLength)), new BufReaderTest.U32BEFramer()) // read all of the frames - buffers.foreach { buf => - assert(await(r.read()).contains(buf)) - } + buffers.foreach { buf => assert(await(r.read()).contains(buf)) } // make sure the reader signals EOF assert(await(r.read()).isEmpty) diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index 370592857b..a0c7a73f29 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -670,9 +670,7 @@ class BufTest def slice(i: Int, j: Int): Buf = throw new Exception("not implemented") def length: Int = 12 def write(output: Array[Byte], off: Int): Unit = - (off until off + length) foreach { i => - output(i) = 'a'.toByte - } + (off until off + length) foreach { i => output(i) = 'a'.toByte } def write(output: ByteBuffer): Unit = ??? def get(index: Int): Byte = ??? @@ -999,12 +997,8 @@ class BufTest } yield Buf(Seq.fill(n)(buf)) forAll(Gen.listOf(bufGen)) { bufs: List[Buf] => - val concatLeft = bufs.foldLeft(Buf.Empty) { (l, r) => - l.concat(r) - } - val concatRight = bufs.foldRight(Buf.Empty) { (l, r) => - l.concat(r) - } + val concatLeft = bufs.foldLeft(Buf.Empty) { (l, r) => l.concat(r) } + val concatRight = bufs.foldRight(Buf.Empty) { (l, r) => l.concat(r) } val constructor = Buf(bufs) assert(constructor == concatLeft) assert(constructor == concatRight) diff --git a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala index 45ee50f88c..01f0084bef 100644 --- a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala @@ -167,12 +167,12 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test("readUnsignedIntBE")(forAll { i: Int => val br = new ByteReaderImpl(Buf.Empty) { override def readIntBE() = i } - assert(br.readUnsignedIntBE() == (i & 0xffffffffL)) + assert(br.readUnsignedIntBE() == (i & 0xFFFFFFFFL)) }) test("readUnsignedIntLE")(forAll { i: Int => val br = new ByteReaderImpl(Buf.Empty) { override def readIntLE() = i } - assert(br.readUnsignedIntLE() == (i & 0xffffffffL)) + assert(br.readUnsignedIntLE() == (i & 0xFFFFFFFFL)) }) val uInt64s: Gen[BigInt] = Gen diff --git a/util-core/src/test/scala/com/twitter/io/FilesTest.scala b/util-core/src/test/scala/com/twitter/io/FilesTest.scala index 0c09a18b55..03922201c7 100644 --- a/util-core/src/test/scala/com/twitter/io/FilesTest.scala +++ b/util-core/src/test/scala/com/twitter/io/FilesTest.scala @@ -19,9 +19,7 @@ class FilesTest extends WordSpec with TempFolder { assert(subfile.createNewFile() == true) assert(Files.delete(tempFolder) == true) - Seq(file, subfile, folder, tempFolder).foreach { x => - assert(!x.exists) - } + Seq(file, subfile, folder, tempFolder).foreach { x => assert(!x.exists) } } } diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index 351acb6b0d..dda7b2f6b6 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -290,9 +290,7 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck def closeNotSatisfiedUntillAllReadsDone = { val rw = new Pipe[Buf] val rf = rw.read() - val cf = rf.flatMap { _ => - rw.close() - } + val cf = rf.flatMap { _ => rw.close() } assert(!rf.isDefined) assert(!cf.isDone) diff --git a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala index 874d39f2b2..785e73c01b 100644 --- a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala +++ b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala @@ -77,9 +77,7 @@ class Base64LongTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } test("toBase64 uses the expected number of digits") { - BoundaryValues.foreach { n: Long => - assert(toBase64(n).length == expectedLength(n)) - } + BoundaryValues.foreach { n: Long => assert(toBase64(n).length == expectedLength(n)) } forAll((n: Long) => assert(toBase64(n).length == expectedLength(n))) val b = new StringBuilder @@ -91,9 +89,7 @@ class Base64LongTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } test("fromBase64 is the inverse of toBase64") { - BoundaryValues.foreach { l => - assert(l == fromBase64(toBase64(l))) - } + BoundaryValues.foreach { l => assert(l == fromBase64(toBase64(l))) } val b = new StringBuilder forAllLongs { (a, n) => diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index b742a797dd..925eb34959 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -178,9 +178,7 @@ class ClosableTest extends FunSuite with Eventually with IntegrationPatience { } test("Cloasable.make catches NonFatals and translates to failed Futures") { - val throwing = Closable.make { _ => - throw new Exception("lolz") - } + val throwing = Closable.make { _ => throw new Exception("lolz") } var failed = 0 val f = throwing.close().respond { diff --git a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala index d46ceb168e..a2d3bb6ea2 100644 --- a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala @@ -52,9 +52,7 @@ class CloseAwaitablyTest extends FunSuite { test("close awaitably with error") { val message = "FORCED EXCEPTION" - val closeFn: () => Future[Unit] = { () => - throw new Exception(message) - } + val closeFn: () => Future[Unit] = { () => throw new Exception(message) } val testClosable = new TestClosable(closeFn) val f = testClosable.close(Time.now) @@ -68,9 +66,7 @@ class CloseAwaitablyTest extends FunSuite { test("close awaitably with fatal error, fatal error blows up close()") { val message = "FORCED INTERRUPTED EXCEPTION" - val closeFn: () => Future[Unit] = { () => - throw new InterruptedException(message) - } + val closeFn: () => Future[Unit] = { () => throw new InterruptedException(message) } val testClosable = new TestClosable(closeFn) val e = intercept[InterruptedException] { diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 56d29e77f6..9b7f540382 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -105,15 +105,11 @@ class DurationTest extends { val ops: Duration.type = Duration } with TimeLikeSp } "fromNow" in { - Time.withCurrentTimeFrozen { _ => - assert(10.seconds.fromNow == (Time.now + 10.seconds)) - } + Time.withCurrentTimeFrozen { _ => assert(10.seconds.fromNow == (Time.now + 10.seconds)) } } "ago" in { - Time.withCurrentTimeFrozen { _ => - assert(10.seconds.ago == (Time.now - 10.seconds)) - } + Time.withCurrentTimeFrozen { _ => assert(10.seconds.ago == (Time.now - 10.seconds)) } } "compare" in { @@ -223,9 +219,7 @@ class DurationTest extends { val ops: Duration.type = Duration } with TimeLikeSp Duration.Bottom, Duration.Top, Duration.Undefined - ).foreach { d => - assert(Duration.parse(d.toString) == d) - } + ).foreach { d => assert(Duration.parse(d.toString) == d) } } "parse" in { diff --git a/util-core/src/test/scala/com/twitter/util/EventTest.scala b/util-core/src/test/scala/com/twitter/util/EventTest.scala index bea5ab5611..48d642cbc0 100644 --- a/util-core/src/test/scala/com/twitter/util/EventTest.scala +++ b/util-core/src/test/scala/com/twitter/util/EventTest.scala @@ -136,15 +136,11 @@ class EventTest extends FunSuite { def register(w: Witness[Int]) = { n += 1 w.notify(1) - Closable.make { _ => - n -= 1; Future.Done - } + Closable.make { _ => n -= 1; Future.Done } } } - val e12 = e1 mergeMap { _ => - e2 - } + val e12 = e1 mergeMap { _ => e2 } val ref = new AtomicReference(Seq.empty[Int]) val closable = e12.build.register(Witness(ref)) @@ -278,9 +274,7 @@ class EventTest extends FunSuite { } def ite[T](i: Var[Boolean], t: Var[T], e: Var[T]) = - i flatMap { i => - if (i) t else e - } + i flatMap { i => if (i) t else e } val b = Var(true) val x = Var(7) @@ -329,9 +323,7 @@ class EventTest extends FunSuite { test("Event.dedupWith") { val e = Event[Int]() val ref = new AtomicReference[Seq[Int]] - e.dedupWith { (a, b) => - a >= b - } + e.dedupWith { (a, b) => a >= b } .build .register(Witness(ref)) e.notify(0) diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index ac9eafab59..61b07a2cf6 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -94,12 +94,8 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert promise } iteration - .onSuccess { _ => - complete = true - } - .onFailure { _ => - failure = true - } + .onSuccess { _ => complete = true } + .onFailure { _ => failure = true } assert(!complete) assert(!failure) } @@ -176,12 +172,8 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert } iteration - .onSuccess { _ => - complete = true - } - .onFailure { _ => - failure = true - } + .onSuccess { _ => complete = true } + .onFailure { _ => failure = true } assert(!complete) assert(!failure) } @@ -905,12 +897,8 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert } } - testJavaFuture("toJavaFuture", { f: Future[Int] => - f.toJavaFuture - }) - testJavaFuture("toCompletableFuture", { f: Future[Int] => - f.toCompletableFuture - }) + testJavaFuture("toJavaFuture", { f: Future[Int] => f.toJavaFuture }) + testJavaFuture("toCompletableFuture", { f: Future[Int] => f.toCompletableFuture }) "monitored" should { trait MonitoredHelper { @@ -1002,9 +990,7 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert "map" which { "when it's all chill" in { - val f = Future(1).map { x => - x + 1 - } + val f = Future(1).map { x => x + 1 } assert(await(f) == 2) } @@ -1058,9 +1044,7 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert def ret(): String = { val f = const.value(1).transform { case Return(_) => - val fn = { () => - return "OK" - } + val fn = { () => return "OK" } fn() Future.value(ret()) case Throw(_) => const.value(0) @@ -1101,9 +1085,7 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert val actual = intercept[FatalException] { Monitor.using(m) { - const.value(1).transform { _ => - throw exc - } + const.value(1).transform { _ => throw exc } } } @@ -1240,17 +1222,12 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert testSequence( "flatMap", - (a, next) => - a.flatMap { _ => - next() - } + (a, next) => a.flatMap { _ => next() } ) testSequence("before", (a, next) => a.before { next() }) "flatMap (values)" should { - val f = Future(1).flatMap { x => - Future(x + 1) - } + val f = Future(1).flatMap { x => Future(x + 1) } "apply" in { assert(await(f) == 2) @@ -1370,9 +1347,7 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert "foreach" in { var wasCalledWith: Option[Int] = None val f = Future(1) - f.foreach { i => - wasCalledWith = Some(i) - } + f.foreach { i => wasCalledWith = Some(i) } assert(wasCalledWith.contains(1)) } @@ -1390,9 +1365,7 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert "when the result has not yet arrived it buffers computations" in { var wasCalledWith: Option[Int] = None val f = new Promise[Int] - f.foreach { i => - wasCalledWith = Some(i) - } + f.foreach { i => wasCalledWith = Some(i) } assert(wasCalledWith.isEmpty) f() = Return(1) assert(wasCalledWith.contains(1)) @@ -1481,9 +1454,7 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert val promise = new Promise[Option[Int]] local() = 123 - val done = promise.map { otherValue => - (otherValue, local()) - } + val done = promise.map { otherValue => (otherValue, local()) } val t = new Thread { override def run(): Unit = { @@ -1925,9 +1896,7 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert val i = Random.nextInt(ps.length) val e = new Exception("sad panda") val t = if (fail) Throw(e) else Return(i) - f.respond { _ => - () - } + f.respond { _ => () } assert(ps.map(_.waitqLength).sum == n) ps(i).update(t) assert(ps.map(_.waitqLength).sum == 0) @@ -1987,9 +1956,7 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert val i = Random.nextInt(ps.length) val e = new Exception("sad panda") val t = if (fail) Throw(e) else Return(i) - f.respond { _ => - () - } + f.respond { _ => () } assert(ps.map(_.waitqLength).sum == n) ps(i).update(t) assert(ps.map(_.waitqLength).sum == 0) @@ -2054,9 +2021,7 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert "terminate when 'next' throws" in { val exc = new Exception def next(): Future[Int] = throw exc - val done = Future.each(next()) { _ => - throw exc - } + val done = Future.each(next()) { _ => throw exc } assert(done.poll.contains(Throw(Future.NextThrewException(exc)))) } diff --git a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala index c13ab58134..4722544fbc 100644 --- a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala @@ -63,9 +63,7 @@ class MemoizeTest extends FunSuite { val ConcurrencyLevel = 5 val computations = - Future.collect(1 to ConcurrencyLevel map { _ => - FuturePool.unboundedPool(memoizer(5)) - }) + Future.collect(1 to ConcurrencyLevel map { _ => FuturePool.unboundedPool(memoizer(5)) }) startUpLatch.countDown() val results = Await.result(computations) diff --git a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala index 15ed8db4bd..35aeeddb1c 100644 --- a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala @@ -16,9 +16,7 @@ class MonitorTest extends FunSuite with MockitoSugar { class MonitorOrElseHelper { val m0, m1, m2 = spy(new MockMonitor()) - Seq(m0, m1, m2).foreach { m => - when(m.handle(any[Throwable])).thenReturn(true) - } + Seq(m0, m1, m2).foreach { m => when(m.handle(any[Throwable])).thenReturn(true) } val exc = new Exception val m = m0.orElse(m1).orElse(m2) } @@ -45,13 +43,9 @@ class MonitorTest extends FunSuite with MockitoSugar { val h = new MonitorOrElseHelper import h._ - Seq(m0, m1, m2) foreach { m => - when(m.handle(any[Throwable])).thenReturn(false) - } + Seq(m0, m1, m2) foreach { m => when(m.handle(any[Throwable])).thenReturn(false) } assert(!m.handle(exc)) - Seq(m0, m1, m2) foreach { m => - verify(m).handle(exc) - } + Seq(m0, m1, m2) foreach { m => verify(m).handle(exc) } } test("Monitor#orElse should wrap Monitor exceptions and pass them on") { diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index 46c302a8e5..d0dc983bb5 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -195,18 +195,14 @@ class PromiseTest extends FunSuite { test("ready") { // this verifies that we don't break the behavior regarding how // the `Scheduler.flush()` is a necessary in `Promise.ready` - Future.Done.map { _ => - Await.result(Future.Done.map(Predef.identity), 5.seconds) - } + Future.Done.map { _ => Await.result(Future.Done.map(Predef.identity), 5.seconds) } } test("maps use local state at the time of callback creation") { val local = new Local[String] local.set(Some("main thread")) val p = new Promise[String]() - val f = p.map { _ => - local().getOrElse("") - } + val f = p.map { _ => local().getOrElse("") } val thread = new Thread(new Runnable { def run(): Unit = { diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 3f9226d5ec..b159783d14 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -427,9 +427,7 @@ with IntegrationPatience { val t1 = t0 + 10.minutes Time.withTimeAt(t0) { _ => assert(Time.now == t0) - Time.withTimeAt(t1) { _ => - assert(Time.now == t1) - } + Time.withTimeAt(t1) { _ => assert(Time.now == t1) } assert(Time.now == t0) } assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 8e16c0d1b4..38acfbb32f 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -32,9 +32,7 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { assert(Var.sample(s) == "8923") var buf = mutable.Buffer[String]() - s.changes.register(Witness({ (v: String) => - buf += v; () - })) + s.changes.register(Witness({ (v: String) => buf += v; () })) assert(buf == Seq("8923")) v() = 111 assert(buf == Seq("8923", "111")) @@ -43,23 +41,13 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test("depth ordering") { val v0 = U(3) val v1 = U(2) - val v2 = v1 flatMap { i => - v1 - } - val v3 = v2 flatMap { i => - v1 - } - val v4 = v3 flatMap { i => - v0 - } + val v2 = v1 flatMap { i => v1 } + val v3 = v2 flatMap { i => v1 } + val v4 = v3 flatMap { i => v0 } var result = 1 - v4.changes.register(Witness({ (i: Int) => - result = result + 2 - })) // result = 3 - v0.changes.register(Witness({ (i: Int) => - result = result * 2 - })) // result = 6 + v4.changes.register(Witness({ (i: Int) => result = result + 2 })) // result = 3 + v0.changes.register(Witness({ (i: Int) => result = result * 2 })) // result = 6 assert(result == 6) result = 1 // reset the value, but this time the ordering will go v0, v4 because of depth @@ -73,15 +61,9 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { val v1 = Var(2) var result = 0 - val o1 = v1.changes.register(Witness({ (i: Int) => - result = result + i - })) // result = 2 - val o2 = v1.changes.register(Witness({ (i: Int) => - result = result * i * i - })) // result = 2 * 2 * 2 = 8 - val o3 = v1.changes.register(Witness({ (i: Int) => - result = result + result + i - })) // result = 8 + 8 + 2 = 18 + val o1 = v1.changes.register(Witness({ (i: Int) => result = result + i })) // result = 2 + val o2 = v1.changes.register(Witness({ (i: Int) => result = result * i * i })) // result = 2 * 2 * 2 = 8 + val o3 = v1.changes.register(Witness({ (i: Int) => result = result + result + i })) // result = 8 + 8 + 2 = 18 assert(result == 18) // ensure those three things happened in sequence @@ -200,9 +182,7 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { assert(vv == 111) assert(closed == Time.Zero) - val o1 = v.changes.register(Witness({ (v: Int) => - () - })) + val o1 = v.changes.register(Witness({ (v: Int) => () })) val t = Time.now val f = o.close(t) @@ -222,9 +202,7 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } test("Var.collect[Seq]") { - def ranged(n: Int) = Seq.tabulate(n) { i => - Var(i) - } + def ranged(n: Int) = Seq.tabulate(n) { i => Var(i) } for (i <- 1 to 10) { val vars = ranged(i) @@ -243,9 +221,7 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test("Var.collect[Set]") { val vars = Seq(Var(1), Var(2), Var(3)) - val coll = Var.collect(vars.map { v => - v: Var[Int] - }.toSet) + val coll = Var.collect(vars.map { v => v: Var[Int] }.toSet) val ref = new AtomicReference[Set[Int]] coll.changes.register(Witness(ref)) assert(ref.get == Set(1, 2, 3)) @@ -278,13 +254,11 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { intercept[java.lang.StackOverflowError] { val v = Var.collect(vars) - v.observe { x => - } + v.observe { x => } } val v = Var.collectIndependent(vars) - v.observe { x => - } + v.observe { x => } } /** @@ -294,9 +268,7 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { val contents = List(1, 2, 3, 4) val v1 = Var.value(contents) assert(Var.sample(v1) eq contents) - v1.changes.register(Witness({ (l: List[Int]) => - assert(contents eq l); () - })) + v1.changes.register(Witness({ (l: List[Int]) => assert(contents eq l); () })) } /** @@ -310,12 +282,8 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { Var.value(i * 2) } - val c1 = f.changes.register(Witness({ (i: Int) => - assert(i == 22); () - })) - val c2 = f.changes.register(Witness({ (i: Int) => - assert(i == 22); () - })) + val c1 = f.changes.register(Witness({ (i: Int) => assert(i == 22); () })) + val c2 = f.changes.register(Witness({ (i: Int) => assert(i == 22); () })) c1.close() c2.close() @@ -324,9 +292,7 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { v() = 22 // now it's safe to re-observe var observed = 3 - f.changes.register(Witness({ (i: Int) => - observed = i - })) + f.changes.register(Witness({ (i: Int) => observed = i })) assert(Var.sample(f) == 44) assert(Var.sample(v) == 22) @@ -345,12 +311,8 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { */ test("Var not executing until observed") { val x = Var(0) - val invertX = x map { i => - 1 / i - } - val result = x flatMap { i => - if (i == 0) Var(0) else invertX - } + val invertX = x map { i => 1 / i } + val result = x flatMap { i => if (i == 0) Var(0) else invertX } x() = 42 x() = 0 // this should not throw an exception because there are no observers @@ -379,13 +341,9 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test("Don't propagate up-to-date " + typ + "-valued Var observations") { val v = Var(123) val w = newVar(333) - val x = v flatMap { _ => - w - } + val x = v flatMap { _ => w } var buf = mutable.Buffer[Int]() - x.changes.register(Witness({ (v: Int) => - buf += v; () - })) + x.changes.register(Witness({ (v: Int) => buf += v; () })) assert(buf == Seq(333)) v() = 333 @@ -402,9 +360,7 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } var buf = mutable.Buffer[Int]() - x.changes.register(Witness({ (v: Int) => - buf += v; () - })) + x.changes.register(Witness({ (v: Int) => buf += v; () })) assert(buf == Seq(333)) v() = 333 diff --git a/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala b/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala index 03fa81e233..1e1436384a 100644 --- a/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala @@ -21,9 +21,7 @@ class WaitQueueTest extends FunSuite with ScalaCheckDrivenPropertyChecks { } yield genWaitQueueOf(size) test("size") { - forAll(Gen.choose(0, 100)) { size => - assert(genWaitQueueOf(size).size == size) - } + forAll(Gen.choose(0, 100)) { size => assert(genWaitQueueOf(size).size == size) } } test("contains") { diff --git a/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala b/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala index a7ab923bdf..f237bf8114 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala @@ -84,10 +84,10 @@ class ConsistentHashingDistributor[A]( val buffer = md5.digest() - underlying.put(byteArrayToLE(buffer, 0).toLong & 0xffffffffL, node) - underlying.put(byteArrayToLE(buffer, 4).toLong & 0xffffffffL, node) - underlying.put(byteArrayToLE(buffer, 8).toLong & 0xffffffffL, node) - underlying.put(byteArrayToLE(buffer, 12).toLong & 0xffffffffL, node) + underlying.put(byteArrayToLE(buffer, 0).toLong & 0xFFFFFFFFL, node) + underlying.put(byteArrayToLE(buffer, 4).toLong & 0xFFFFFFFFL, node) + underlying.put(byteArrayToLE(buffer, 8).toLong & 0xFFFFFFFFL, node) + underlying.put(byteArrayToLE(buffer, 12).toLong & 0xFFFFFFFFL, node) i += 1 } @@ -106,7 +106,7 @@ class ConsistentHashingDistributor[A]( // hashes are 32-bit because they are 32-bit on the libmemcached and // we need to maintain compatibility with libmemcached - private[this] def truncateHash(hash: Long): Long = hash & 0xffffffffL + private[this] def truncateHash(hash: Long): Long = hash & 0xFFFFFFFFL private def mapEntryForHash(hash: Long): java.util.Map.Entry[Long, HashNode[A]] = { val truncatedHash = truncateHash(hash) diff --git a/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala b/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala index 1eada3cdfc..0b9ca12d2d 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala @@ -17,15 +17,11 @@ trait Hashable[-T, +R] extends (T => R) { self => trait LowPriorityHashable { // XOR the high 32 bits into the low to get a int: implicit def toInt[T](implicit h: Hashable[T, Long]): Hashable[T, Int] = - h.andThen { long => - (long >> 32).toInt ^ long.toInt - } + h.andThen { long => (long >> 32).toInt ^ long.toInt } // Get the UTF-8 bytes of a string to hash it implicit def fromString[T](implicit h: Hashable[Array[Byte], T]): Hashable[String, T] = - h.compose { s: String => - s.getBytes - } + h.compose { s: String => s.getBytes } } object Hashable extends LowPriorityHashable { @@ -54,7 +50,7 @@ object Hashable extends LowPriorityHashable { val PRIME: Int = 16777619 var i = 0 val len = key.length - var rv: Long = 0x811c9dc5L + var rv: Long = 0x811C9DC5L while (i < len) { rv = (rv * PRIME) ^ (key(i) & 0xff) i += 1 @@ -74,7 +70,7 @@ object Hashable extends LowPriorityHashable { val PRIME: Int = 16777619 var i = 0 val len = key.length - var rv: Long = 0x811c9dc5L + var rv: Long = 0x811C9DC5L while (i < len) { rv = (rv ^ (key(i) & 0xff)) * PRIME i += 1 @@ -94,7 +90,7 @@ object Hashable extends LowPriorityHashable { val PRIME: Long = 1099511628211L var i = 0 val len = key.length - var rv: Long = 0xcbf29ce484222325L + var rv: Long = 0xCBF29CE484222325L while (i < len) { rv = (rv * PRIME) ^ (key(i) & 0xff) i += 1 @@ -114,7 +110,7 @@ object Hashable extends LowPriorityHashable { val PRIME: Long = 1099511628211L var i = 0 val len = key.length - var rv: Long = 0xcbf29ce484222325L + var rv: Long = 0xCBF29CE484222325L while (i < len) { rv = (rv ^ (key(i) & 0xff)) * PRIME i += 1 @@ -163,7 +159,7 @@ object Hashable extends LowPriorityHashable { var j = 0 while (j < 8) { if ((rv & 1) != 0) { - rv = (rv >> 1) ^ 0xedb88320L + rv = (rv >> 1) ^ 0xEDB88320L } else { rv >>= 1 } diff --git a/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala index 20b74c0feb..013a14b8c7 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala @@ -45,9 +45,7 @@ class ConsistentHashingDistributorTest extends WordSpec with ScalaCheckDrivenPro assert(expected.size == 99) // Test that ketamaClient.clientOf(key) == expected IP - val handleToIp = nodes.map { n => - n.handle -> n.identifier - }.toMap + val handleToIp = nodes.map { n => n.handle -> n.identifier }.toMap for (testcase <- expected) { val hash = KeyHasher.KETAMA.hashKey(testcase(0).getBytes) diff --git a/util-hashing/src/test/scala/com/twitter/hashing/DistributionTester.scala b/util-hashing/src/test/scala/com/twitter/hashing/DistributionTester.scala index 5688d70c8f..3f0ee37b58 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/DistributionTester.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/DistributionTester.scala @@ -15,13 +15,9 @@ class DistributionTester[A](distributor: Distributor[A]) { keysPerNode(key) += 1 } var frequencies = keysPerNode.values.toList - frequencies ++= 0 until (distributor.nodeCount - frequencies.size) map { _ => - 0 - } + frequencies ++= 0 until (distributor.nodeCount - frequencies.size) map { _ => 0 } val average = frequencies.sum.toDouble / frequencies.size - val diffs = frequencies.map { v => - math.pow((v - average), 2) - } + val diffs = frequencies.map { v => math.pow((v - average), 2) } val sd = math.sqrt(diffs.sum / (frequencies.size - 1)) sd / average } diff --git a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala index b2f7a31e78..de3ca360d7 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala @@ -18,9 +18,7 @@ class HashableTest extends FunSuite with ScalaCheckDrivenPropertyChecks { ) def testConsistency[T](algo: Hashable[Array[Byte], T]): Unit = { - forAll { input: Array[Byte] => - assert(algo(input) == algo(input)) - } + forAll { input: Array[Byte] => assert(algo(input) == algo(input)) } } algorithms foreach { algo => diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala b/util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala index 1b006ec4eb..1480ba50ee 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala @@ -25,16 +25,12 @@ class Demo extends FunSuite { } def someBusinessLogic(future: Future[Int]): Future[Int] = { - val result: Future[Int] = future.map { i => - i + 1 - } + val result: Future[Int] = future.map { i => i + 1 } moreBusinessLogic(result) } def moreBusinessLogic(future: Future[Int]): Future[Int] = { - val result: Future[Int] = future.map { i => - i * 2 - } + val result: Future[Int] = future.map { i => i * 2 } lordBusinessLogic(result) } diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala b/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala index 11ab3876e9..fc6597ff73 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala @@ -45,9 +45,7 @@ private[jvm] class Allocations(statsReceiver: StatsReceiver) { private[jvm] def start(): Unit = { edenPool - .flatMap { bean => - Option(bean.getUsage) - } + .flatMap { bean => Option(bean.getUsage) } .foreach { _ => ManagementFactory.getGarbageCollectorMXBeans.asScala.foreach { case bean: NotificationEmitter => diff --git a/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala b/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala index 5f1095876b..69041d5a0b 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala @@ -43,9 +43,7 @@ class ContentionSnapshot { if (deadlockThreadIds == null) Array.empty[ThreadInfo] else deadlockThreadIds.flatMap { id => - blocked.find { threadInfo => - threadInfo.getThreadId() == id - } + blocked.find { threadInfo => threadInfo.getThreadId() == id } } Snapshot( diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala b/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala index fefbae359a..c7e078f835 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Estimator.scala @@ -40,7 +40,7 @@ class Kalman(N: Int) { val mv = mvar val ev = evar if (mv + ev == 0) - weight = 1D + weight = 1d else weight = mv / (mv + ev) n += 1 @@ -60,13 +60,11 @@ class Kalman(N: Int) { private[this] def variance(samples: Array[Double]): Double = { if (samples.length == 1) - return 0D + return 0d val sum = samples.sum val mean = sum / samples.length - val diff = (samples map { x => - (x - mean) * (x - mean) - }).sum + val diff = (samples map { x => (x - mean) * (x - mean) }).sum diff / (samples.length - 1) } @@ -80,7 +78,7 @@ class Kalman(N: Int) { * value). */ class KalmanGaussianError(N: Int, range: Double) extends Kalman(N) with Estimator[Double] { - require(range >= 0D && range < 1D) + require(range >= 0d && range < 1d) private[this] val rng = new Random def measure(m: Double): Unit = { @@ -137,7 +135,7 @@ class WindowedMeans(N: Int, windows: Seq[(Int, Int)]) extends Estimator[Double] * See: https://web.mit.edu/saltzer/www/publications/instrumentation.html */ class LoadAverage(interval: Double) extends Estimator[Double] { - private[this] val a = math.exp(-1D / interval) + private[this] val a = math.exp(-1d / interval) private[this] var load = Double.NaN def measure(m: Double): Unit = { diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index 8a0779fdf0..a46cbfbfce 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -59,9 +59,7 @@ class Hotspot extends Jvm { private[this] def counters(pat: String) = { try { val cs = jvm.getInternalCounters(pat).asScala - cs.map { c => - c.getName() -> c - }.toMap + cs.map { c => c.getName() -> c }.toMap } catch { case e: OperationNotSupportedException => log.log( diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala index 119a329f42..fb1f92cf2b 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala @@ -131,8 +131,7 @@ trait Jvm { buffer = (gc :: buffer).takeWhile(_.timestamp > floor) } - since: Time => - buffer.takeWhile(_.timestamp > since) + since: Time => buffer.takeWhile(_.timestamp > since) } def forceGc(): Unit diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 6890604cd6..02f5151c94 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -77,9 +77,7 @@ object JvmStats { val currentMem = memStats.scope("current") val postGCStats = memStats.scope("postGC") memPool.foreach { pool => - val name = pool.getName.regexSub("""[^\w]""".r) { m => - "_" - } + val name = pool.getName.regexSub("""[^\w]""".r) { m => "_" } if (pool.getCollectionUsage != null) { def usage = pool.getCollectionUsage // this is a snapshot, we can't reuse the value gauges.add(postGCStats.addGauge(name, "used") { usage.getUsed }) @@ -126,9 +124,7 @@ object JvmStats { val gcPool = ManagementFactory.getGarbageCollectorMXBeans.asScala val gcStats = stats.scope("gc") gcPool.foreach { gc => - val name = gc.getName.regexSub("""[^\w]""".r) { m => - "_" - } + val name = gc.getName.regexSub("""[^\w]""".r) { m => "_" } gauges.add(gcStats.addGauge(name, "cycles") { gc.getCollectionCount }) gauges.add(gcStats.addGauge(name, "msec") { gc.getCollectionTime }) } diff --git a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala index fed1867e10..ff28e078a2 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala @@ -5,7 +5,7 @@ import org.scalatest.FunSuite class EstimatorTest extends FunSuite { test("LoadAverage") { // This makes LoadAverage.a = 1/2 for easy testing. - val interval = -1D / math.log(0.5) + val interval = -1d / math.log(0.5) val e = new LoadAverage(interval) assert(e.estimate.isNaN) diff --git a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala index 69afdfb70c..b36bfa2f5d 100644 --- a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala @@ -275,9 +275,7 @@ class FileHandler( if (examineRollTime) { // Only allow a single thread at a time to do a roll synchronized { - nextRollTime foreach { time => - if (Time.now.inMilliseconds > time) roll() - } + nextRollTime foreach { time => if (Time.now.inMilliseconds > time) roll() } } } diff --git a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala index 44adbd2015..874672f0bc 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala @@ -28,9 +28,7 @@ private[logging] object Formatter { private[logging] def formatStackTrace(t: Throwable, limit: Int): List[String] = { var out = new mutable.ListBuffer[String] if (limit > 0) { - out ++= t.getStackTrace.map { elem => - " at %s".format(elem.toString) - } + out ++= t.getStackTrace.map { elem => " at %s".format(elem.toString) } if (out.length > limit) { out.trimEnd(out.length - limit) out += " (...more...)" diff --git a/util-logging/src/main/scala/com/twitter/logging/Handler.scala b/util-logging/src/main/scala/com/twitter/logging/Handler.scala index d3db7599a3..5a7f6935ee 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Handler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Handler.scala @@ -25,9 +25,7 @@ import java.util.{logging => javalog} abstract class Handler(val formatter: Formatter, val level: Option[Level]) extends javalog.Handler { setFormatter(formatter) - level.foreach { x => - setLevel(x) - } + level.foreach { x => setLevel(x) } override def toString: String = { "<%s level=%s formatter=%s>".format(getClass.getName, getLevel, formatter.toString) diff --git a/util-logging/src/main/scala/com/twitter/logging/Logger.scala b/util-logging/src/main/scala/com/twitter/logging/Logger.scala index a703afb050..349501c2bb 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Logger.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Logger.scala @@ -211,14 +211,10 @@ object NullLogger object Logger extends Iterable[Logger] { private[this] val levelNamesMap: Map[String, Level] = - Level.AllLevels.map { level => - level.name -> level - }.toMap + Level.AllLevels.map { level => level.name -> level }.toMap private[this] val levelsMap: Map[Int, Level] = - Level.AllLevels.map { level => - level.value -> level - }.toMap + Level.AllLevels.map { level => level.value -> level }.toMap // A cache of scala Logger objects by name. // Using a low concurrencyLevel (2), with the assumption that we aren't ever creating too diff --git a/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala b/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala index 0d95a7747c..b94dd484d9 100644 --- a/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala +++ b/util-logging/src/main/scala/com/twitter/logging/LoggerFactory.scala @@ -48,12 +48,8 @@ case class LoggerFactory( def apply(): Logger = { val logger = Logger.get(node) logger.clearHandlers() - level.foreach { x => - logger.setLevel(x) - } - handlers.foreach { h => - logger.addHandler(h()) - } + level.foreach { x => logger.setLevel(x) } + handlers.foreach { h => logger.addHandler(h()) } logger.setUseParentHandlers(useParents) logger } diff --git a/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala b/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala index e617f71a97..9dad3f8b29 100644 --- a/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala @@ -144,9 +144,7 @@ class QueueingHandler(handler: Handler, val maxQueueSize: Int, inferClassNames: override def flush(): Unit = { // Publish all records in queue - queue.drain().map { records => - records.foreach(doPublish) - } + queue.drain().map { records => records.foreach(doPublish) } // Propagate flush super.flush() diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index e3937ce697..68ae4c1083 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -32,18 +32,16 @@ class FileHandlerTest extends WordSpec with TempFolder { new BufferedReader(new InputStreamReader(new FileInputStream(new File(folderName, filename)))) } - def writer(filename: String):OutputStreamWriter = { + def writer(filename: String): OutputStreamWriter = { new OutputStreamWriter(new FileOutputStream(new File(folderName, filename)), "UTF-8") } - private[this] val matcher = (name: String) => new BiPredicate[Path, BasicFileAttributes] { - override def test( - path: Path, - attributes: BasicFileAttributes - ): Boolean = { - path.toFile.getName.contains(name) + private[this] val matcher = (name: String) => + new BiPredicate[Path, BasicFileAttributes] { + override def test(path: Path, attributes: BasicFileAttributes): Boolean = { + path.toFile.getName.contains(name) + } } - } // Looks for files that contain `name` in the given `dirToSearch`. // This method is used in place of setting the `user.dir` @@ -240,9 +238,7 @@ class FileHandlerTest extends WordSpec with TempFolder { } // the target, 1 rotated file, and the short file should all remain - (1 to 5).foreach { _ => - flush() - } + (1 to 5).foreach { _ => flush() } val fileSet = new File(folderName).list().toSet assert(fileSet.contains(name)) assert(fileSet.contains(namePrefix)) diff --git a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala index 0181b3d583..ace293b2b0 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala @@ -157,21 +157,18 @@ class FormatterTest extends WordSpec { } def scrub(in: String) = { - in.regexSub("""FormatterTest.scala:\d+""".r) { m => - "FormatterTest.scala:NNN" - } - .regexSub("""FormatterTest\$[\w\\$]+""".r) { m => - "FormatterTest$$" - } + in.regexSub("""FormatterTest.scala:\d+""".r) { m => "FormatterTest.scala:NNN" } + .regexSub("""FormatterTest\$[\w\\$]+""".r) { m => "FormatterTest$$" } } "simple" in { - val exception = try { - ExceptionLooper.cycle(10) - null - } catch { - case t: Throwable => t - } + val exception = + try { + ExceptionLooper.cycle(10) + null + } catch { + case t: Throwable => t + } assert( Formatter.formatStackTrace(exception, 5).map { scrub(_) } == List( " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", @@ -185,12 +182,13 @@ class FormatterTest extends WordSpec { } "nested" in { - val exception = try { - ExceptionLooper.cycle2(2) - null - } catch { - case t: Throwable => t - } + val exception = + try { + ExceptionLooper.cycle2(2) + null + } catch { + case t: Throwable => t + } assert( Formatter.formatStackTrace(exception, 1).map { scrub(_) } == List( " at com.twitter.logging.FormatterTest$$.cycle2(FormatterTest.scala:NNN)", diff --git a/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala b/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala index 039c1937b7..5b6d6b1e80 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala @@ -6,9 +6,7 @@ import org.scalatest.FunSuite class LogRecordTest extends FunSuite { test("LogRecord should getMethod properly") { Logger.withLoggers(Nil) { - new LogRecordTestHelper({ r: JRecord => - r.getSourceMethodName() - }) { + new LogRecordTestHelper({ r: JRecord => r.getSourceMethodName() }) { def makingLogRecord() = { logger.log(Level.INFO, "OK") assert(handler.get == "makingLogRecord") @@ -36,10 +34,7 @@ abstract class LogRecordTestHelper(formats: JRecord => String) { logger.addHandler(handler) } -class Foo - extends LogRecordTestHelper({ r: JRecord => - r.getSourceClassName() - }) { +class Foo extends LogRecordTestHelper({ r: JRecord => r.getSourceClassName() }) { def makingLogRecord(): Unit = { logger.log(Level.INFO, "OK") } diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala index ff10443d7a..9dff6d0099 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala @@ -40,9 +40,7 @@ class ProxyTest extends WordSpec { "generate a factory for an interface" in { var called = 0 - val f1 = new ProxyFactory[TestInterface]({ m => - called += 1; m() - }) + val f1 = new ProxyFactory[TestInterface]({ m => called += 1; m() }) val obj = new TestImpl val proxied = f1(obj) @@ -56,9 +54,7 @@ class ProxyTest extends WordSpec { "generate a factory for a class with a default constructor" in { var called = 0 - val f2 = new ProxyFactory[TestClass]({ m => - called += 1; m() - }) + val f2 = new ProxyFactory[TestClass]({ m => called += 1; m() }) val obj = new TestClass val proxied = f2(obj) @@ -164,12 +160,8 @@ class ProxyTest extends WordSpec { results.sortWith((a, b) => a > b).slice(2, 5).sum / 3 } - val reflectConstructor = { () => - new ReferenceProxyFactory[TestInterface](_()) - } - val cglibConstructor = { () => - new ProxyFactory[TestInterface](_()) - } + val reflectConstructor = { () => new ReferenceProxyFactory[TestInterface](_()) } + val cglibConstructor = { () => new ProxyFactory[TestInterface](_()) } /* "maintains proxy creation speed" in { val repTimes = 40000 diff --git a/util-registry/src/main/scala/com/twitter/util/registry/Registry.scala b/util-registry/src/main/scala/com/twitter/util/registry/Registry.scala index 92eab2b039..006028f952 100644 --- a/util-registry/src/main/scala/com/twitter/util/registry/Registry.scala +++ b/util-registry/src/main/scala/com/twitter/util/registry/Registry.scala @@ -89,9 +89,7 @@ class SimpleRegistry extends Registry { } private[this] def sanitize(key: String): String = - key.filter { char => - char > 31 && char < 127 - } + key.filter { char => char > 31 && char < 127 } } /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala index d640071418..53a8b05bda 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala @@ -41,11 +41,7 @@ object ExceptionStatsHandler { Seq(exceptionChain, Nil) } - labels.flatMap { prefix => - suffixes.map { suffix => - prefix ++ suffix - } - } + labels.flatMap { prefix => suffixes.map { suffix => prefix ++ suffix } } } } @@ -151,8 +147,6 @@ private[finagle] class MultiCategorizingExceptionStatsHandler( val paths: Seq[Seq[String]] = statPaths(t, labels, rollup) if (flags.nonEmpty) statsReceiver.counter(parentLabel).incr() - paths.foreach { path => - statsReceiver.counter(path: _*).incr() - } + paths.foreach { path => statsReceiver.counter(path: _*).incr() } } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala index e9c91d5196..e932203353 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala @@ -18,9 +18,7 @@ class RollupStatsReceiver(protected val self: StatsReceiver) extends StatsReceiv Seq(s) case Seq(hd, tl @ _*) => - Seq(Seq(hd)) ++ (tails(tl) map { t => - Seq(hd) ++ t - }) + Seq(Seq(hd)) ++ (tails(tl) map { t => Seq(hd) ++ t }) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala index 1e358e5182..1aa75644c7 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala @@ -46,9 +46,7 @@ object Stat { def timeFuture[A](stat: Stat, unit: TimeUnit)(f: => Future[A]): Future[A] = { val start = Stopwatch.timeNanos() try { - f.respond { _ => - stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS)) - } + f.respond { _ => stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS)) } } catch { case NonFatal(e) => stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS)) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/WithHistogramDetails.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/WithHistogramDetails.scala index 1315f21950..d5da2f8632 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/WithHistogramDetails.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/WithHistogramDetails.scala @@ -22,8 +22,7 @@ private[stats] class AggregateWithHistogramDetails(underlying: Seq[WithHistogram */ def histogramDetails: Map[String, HistogramDetail] = underlying.foldLeft[Map[String, HistogramDetail]](Map.empty[String, HistogramDetail]) { - (acc, cur) => - cur.histogramDetails ++ acc + (acc, cur) => cur.histogramDetails ++ acc } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala index 5b931cd784..70f257bbca 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala @@ -74,9 +74,7 @@ class CumulativeGaugeTest extends FunSuite with Eventually with IntegrationPatie test("a CumulativeGauge should sum values across all registered gauges") { val gauge = new TestGauge() - val underlying = 0.until(100).map { _ => - gauge.addGauge { 10.0f } - } + val underlying = 0.until(100).map { _ => gauge.addGauge { 10.0f } } assert(gauge.getValue == (10.0f * 100)) } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index eafd7af4b2..372b5a5c03 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -49,9 +49,9 @@ class StatsReceiverTest extends FunSuite { assert(s1.values == Seq.empty) assert(s2.values == Seq.empty) - broadcastStat.add(1F) - assert(s1.values == Seq(1F)) - assert(s2.values == Seq(1F)) + broadcastStat.add(1f) + assert(s1.values == Seq(1f)) + assert(s2.values == Seq(1f)) } test("StatsReceiver time") { @@ -141,9 +141,7 @@ class StatsReceiverTest extends FunSuite { assert( "DenylistStatsReceiver(NullStatsReceiver)" == - new DenylistStatsReceiver(NullStatsReceiver, { _ => - false - }).toString + new DenylistStatsReceiver(NullStatsReceiver, { _ => false }).toString ) val inMem = new InMemoryStatsReceiver() diff --git a/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala b/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala index e179e0b2d2..4567e358f7 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala +++ b/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala @@ -257,7 +257,8 @@ trait ArgumentCapture { E: ClassTag, F: ClassTag, G: ClassTag - ](func: (A, B, C, D, E, F, G) => _ + ]( + func: (A, B, C, D, E, F, G) => _ ): Seq[(A, B, C, D, E, F, G)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -294,7 +295,8 @@ trait ArgumentCapture { E: ClassTag, F: ClassTag, G: ClassTag - ](func: (A, B, C, D, E, F, G) => _ + ]( + func: (A, B, C, D, E, F, G) => _ ): (A, B, C, D, E, F, G) = capturingAll[A, B, C, D, E, F, G](func).lastOption.getOrElse(noArgWasCaptured()) @@ -331,7 +333,8 @@ trait ArgumentCapture { F: ClassTag, G: ClassTag, H: ClassTag - ](func: (A, B, C, D, E, F, G, H) => _ + ]( + func: (A, B, C, D, E, F, G, H) => _ ): Seq[(A, B, C, D, E, F, G, H)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -372,7 +375,8 @@ trait ArgumentCapture { F: ClassTag, G: ClassTag, H: ClassTag - ](func: (A, B, C, D, E, F, G, H) => _ + ]( + func: (A, B, C, D, E, F, G, H) => _ ): (A, B, C, D, E, F, G, H) = capturingAll[A, B, C, D, E, F, G, H](func).lastOption.getOrElse(noArgWasCaptured()) @@ -412,7 +416,8 @@ trait ArgumentCapture { G: ClassTag, H: ClassTag, I: ClassTag - ](func: (A, B, C, D, E, F, G, H, I) => _ + ]( + func: (A, B, C, D, E, F, G, H, I) => _ ): Seq[(A, B, C, D, E, F, G, H, I)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -457,7 +462,8 @@ trait ArgumentCapture { G: ClassTag, H: ClassTag, I: ClassTag - ](func: (A, B, C, D, E, F, G, H, I) => _ + ]( + func: (A, B, C, D, E, F, G, H, I) => _ ): (A, B, C, D, E, F, G, H, I) = capturingAll[A, B, C, D, E, F, G, H, I](func).lastOption.getOrElse(noArgWasCaptured()) @@ -500,7 +506,8 @@ trait ArgumentCapture { H: ClassTag, I: ClassTag, J: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -549,7 +556,8 @@ trait ArgumentCapture { H: ClassTag, I: ClassTag, J: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J) => _ ): (A, B, C, D, E, F, G, H, I, J) = capturingAll[A, B, C, D, E, F, G, H, I, J](func).lastOption.getOrElse(noArgWasCaptured()) @@ -598,7 +606,8 @@ trait ArgumentCapture { I: ClassTag, J: ClassTag, K: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -651,7 +660,8 @@ trait ArgumentCapture { I: ClassTag, J: ClassTag, K: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K) => _ ): (A, B, C, D, E, F, G, H, I, J, K) = capturingAll[A, B, C, D, E, F, G, H, I, J, K](func).lastOption.getOrElse(noArgWasCaptured()) @@ -703,7 +713,8 @@ trait ArgumentCapture { J: ClassTag, K: ClassTag, L: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -760,7 +771,8 @@ trait ArgumentCapture { J: ClassTag, K: ClassTag, L: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L](func).lastOption.getOrElse(noArgWasCaptured()) @@ -815,7 +827,8 @@ trait ArgumentCapture { K: ClassTag, L: ClassTag, M: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -876,7 +889,8 @@ trait ArgumentCapture { K: ClassTag, L: ClassTag, M: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M](func).lastOption .getOrElse(noArgWasCaptured()) @@ -935,7 +949,8 @@ trait ArgumentCapture { L: ClassTag, M: ClassTag, N: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1015,7 +1030,8 @@ trait ArgumentCapture { L: ClassTag, M: ClassTag, N: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1077,7 +1093,8 @@ trait ArgumentCapture { M: ClassTag, N: ClassTag, O: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1162,7 +1179,8 @@ trait ArgumentCapture { M: ClassTag, N: ClassTag, O: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1227,7 +1245,8 @@ trait ArgumentCapture { N: ClassTag, O: ClassTag, P: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1317,7 +1336,8 @@ trait ArgumentCapture { N: ClassTag, O: ClassTag, P: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1385,7 +1405,8 @@ trait ArgumentCapture { O: ClassTag, P: ClassTag, Q: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1480,7 +1501,8 @@ trait ArgumentCapture { O: ClassTag, P: ClassTag, Q: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1554,7 +1576,8 @@ trait ArgumentCapture { P: ClassTag, Q: ClassTag, R: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1654,7 +1677,8 @@ trait ArgumentCapture { P: ClassTag, Q: ClassTag, R: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1731,7 +1755,8 @@ trait ArgumentCapture { Q: ClassTag, R: ClassTag, S: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -1836,7 +1861,8 @@ trait ArgumentCapture { Q: ClassTag, R: ClassTag, S: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S](func).lastOption .getOrElse(noArgWasCaptured()) @@ -1922,7 +1948,8 @@ trait ArgumentCapture { R: ClassTag, S: ClassTag, T: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -2032,7 +2059,8 @@ trait ArgumentCapture { R: ClassTag, S: ClassTag, T: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T](func).lastOption .getOrElse(noArgWasCaptured()) @@ -2124,7 +2152,8 @@ trait ArgumentCapture { S: ClassTag, T: ClassTag, U: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -2239,7 +2268,8 @@ trait ArgumentCapture { S: ClassTag, T: ClassTag, U: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U](func).lastOption .getOrElse(noArgWasCaptured()) @@ -2340,7 +2370,8 @@ trait ArgumentCapture { T: ClassTag, U: ClassTag, V: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => _ ): Seq[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)] = { val argCaptorA = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) val argCaptorB = ArgumentCaptor.forClass(classTag[B].runtimeClass.asInstanceOf[Class[B]]) @@ -2460,7 +2491,8 @@ trait ArgumentCapture { T: ClassTag, U: ClassTag, V: ClassTag - ](func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => _ + ]( + func: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => _ ): (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) = capturingAll[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V](func).lastOption .getOrElse(noArgWasCaptured()) diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala index 99cf886ae2..6dfc4007f2 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala @@ -110,9 +110,7 @@ final class JsonTunableMapper(deserializers: Seq[JsonDeserializer[_ <: Any]]) { import JsonTunableMapper._ private[this] object DeserializationModule extends SimpleModule { - deserializers.foreach { jd => - addDeserializer(jd.handledType().asInstanceOf[Class[Any]], jd) - } + deserializers.foreach { jd => addDeserializer(jd.handledType().asInstanceOf[Class[Any]], jd) } } private[this] val mapper: ObjectMapper = diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala index 49d770a60c..f5d72a00eb 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala @@ -163,9 +163,7 @@ object TunableMap { private[tunable] def replace(newMap: TunableMap): Unit = { // clear keys no longer present - (entries.map(_.key).toSet -- newMap.entries.map(_.key).toSet).foreach { key => - clear(key) - } + (entries.map(_.key).toSet -- newMap.entries.map(_.key).toSet).foreach { key => clear(key) } // add/update new tunable values ++=(newMap) diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/TunableMapTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/TunableMapTest.scala index 6b819a96c0..78cdfedf43 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/TunableMapTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/TunableMapTest.scala @@ -336,9 +336,7 @@ class MutableTest extends FunSuite { map1.put("key2", 5) assert(combined.entries.size == 1) - assert(combined.entries.exists { entry => - entry.key.id == "key2" && entry.value == 5 - }) + assert(combined.entries.exists { entry => entry.key.id == "key2" && entry.value == 5 }) } test("orElse: entries returns entries from both maps") { @@ -353,12 +351,8 @@ class MutableTest extends FunSuite { map2.put("key2", 5) assert(combined.entries.size == 2) - assert(combined.entries.exists { entry => - entry.key.id == "key1" && entry.value == "hello1" - }) - assert(combined.entries.exists { entry => - entry.key.id == "key2" && entry.value == 5 - }) + assert(combined.entries.exists { entry => entry.key.id == "key1" && entry.value == "hello1" }) + assert(combined.entries.exists { entry => entry.key.id == "key2" && entry.value == 5 }) } test("orElse on two NullTunableMaps produces NullTunableMap") { diff --git a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala index 8627612099..ef6755a404 100644 --- a/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala +++ b/util-zk/src/main/scala/com/twitter/zk/NativeConnector.scala @@ -50,18 +50,14 @@ case class NativeConnector( serialized { connection.getOrElse { val c = mkConnection - c.sessionEvents foreach { event => - sessionBroker.send(event()).sync() - } + c.sessionEvents foreach { event => sessionBroker.send(event()).sync() } connection = Some(c) c }.apply }.flatten .rescue { case e: NativeConnector.ConnectTimeoutException => - release() flatMap { _ => - Future.exception(e) - } + release() flatMap { _ => Future.exception(e) } } /** diff --git a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala index 600415779d..5b5fe3fb58 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZNode.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZNode.scala @@ -91,9 +91,7 @@ trait ZNode { zkClient.retrying { zk => val result = new StringCallbackPromise zk.create(creatingPath, data, acls.asJava, mode, result, null) - result map { newPath => - zkClient(newPath) - } + result map { newPath => zkClient(newPath) } } } @@ -101,9 +99,7 @@ trait ZNode { def delete(version: Int = 0): Future[ZNode] = zkClient.retrying { zk => val result = new UnitCallbackPromise zk.delete(path, version, result, null) - result map { _ => - this - } + result map { _ => this } } /** Returns a Future that is satisfied with this ZNode with its metadata and data */ @@ -117,9 +113,7 @@ trait ZNode { def sync(): Future[ZNode] = zkClient.retrying { zk => val result = new UnitCallbackPromise zk.sync(path, result, null) - result map { _ => - this - } + result map { _ => this } } /** Provides access to this node's children. */ @@ -215,9 +209,7 @@ trait ZNode { /** Pipe events from a subtree's monitor to this broker. */ def pipeSubTreeUpdates(next: Offer[ZNode.TreeUpdate]): Unit = { - next.sync().flatMap(broker ! _).onSuccess { _ => - pipeSubTreeUpdates(next) - } + next.sync().flatMap(broker ! _).onSuccess { _ => pipeSubTreeUpdates(next) } } /** Monitor a watch on this node. */ @@ -242,9 +234,7 @@ trait ZNode { log.debug("updating %s with %d children", path, treeUpdate.added.size) broker.send(treeUpdate).sync.onSuccess { _ => log.debug("updated %s with %d children", path, treeUpdate.added.size) - treeUpdate.added foreach { z => - pipeSubTreeUpdates(z.monitorTree()) - } + treeUpdate.added foreach { z => pipeSubTreeUpdates(z.monitorTree()) } eventUpdate onSuccess { event => log.debug("event received on %s: %s", path, event) } onSuccess { diff --git a/util-zk/src/main/scala/com/twitter/zk/ZOp.scala b/util-zk/src/main/scala/com/twitter/zk/ZOp.scala index 889fa2acbe..3b9e6d0465 100644 --- a/util-zk/src/main/scala/com/twitter/zk/ZOp.scala +++ b/util-zk/src/main/scala/com/twitter/zk/ZOp.scala @@ -23,11 +23,7 @@ trait ZOp[T <: ZNode.Exists] { def setWatch(): Unit = { watch() onSuccess { case ZNode.Watch(result, update) => - broker ! result onSuccess { _ => - update onSuccess { _ => - setWatch() - } - } + broker ! result onSuccess { _ => update onSuccess { _ => setWatch() } } } } setWatch() diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala index 6075fd1554..4b36b732ae 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala @@ -57,9 +57,7 @@ class ShardCoordinator(zk: ZkClient, path: String, numShards: Int) { def acquire(): Future[ShardPermit] = { semaphore.acquire flatMap { permit => shardNodes() map { nodes => - nodes map { node => - shardIdOf(node.path) - } + nodes map { node => shardIdOf(node.path) } } map { ids => (0 until numShards) filterNot { ids contains _ } } flatMap { availableIds => @@ -84,9 +82,7 @@ class ShardCoordinator(zk: ZkClient, path: String, numShards: Int) { case err: LackOfConsensusException => Future.exception(SemaphoreError(err)) case err: PermitMismatchException => Future.exception(SemaphoreError(err)) case err: PermitNodeException => Future.exception(SemaphoreError(err)) - } onFailure { err => - permit.release() - } + } onFailure { err => permit.release() } } } diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala index a52c68eff3..9ce310cd42 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ZkAsyncSemaphore.scala @@ -74,9 +74,7 @@ class ZkAsyncSemaphore( val mySequenceNumber = sequenceNumberOf(permitNode.path) permitNodes() flatMap { permits => - val sequenceNumbers = permits map { child => - sequenceNumberOf(child.path) - } + val sequenceNumbers = permits map { child => sequenceNumberOf(child.path) } getConsensusNumPermits(permits) flatMap { consensusNumPermits => if (consensusNumPermits != numPermits) { throw ZkAsyncSemaphore.PermitMismatchException( @@ -124,9 +122,7 @@ class ZkAsyncSemaphore( zk onSessionEvent { case StateEvent.Expired => rejectWaitQueue() case StateEvent.Connected => { - permitNodes() map { nodes => - checkWaiters(nodes) - } + permitNodes() map { nodes => checkWaiters(nodes) } monitorSemaphore(semaphoreNode) } } @@ -179,11 +175,7 @@ class ZkAsyncSemaphore( */ private[this] def monitorSemaphore(node: ZNode) = { val monitor = node.getChildren.monitor() - monitor foreach { tryChildren => - tryChildren map { zop => - checkWaiters(zop.children.toSeq) - } - } + monitor foreach { tryChildren => tryChildren map { zop => checkWaiters(zop.children.toSeq) } } } /** @@ -205,9 +197,7 @@ class ZkAsyncSemaphore( val permits = nodes filter { child => child.path.startsWith(permitNodePathPrefix) } sortBy (child => sequenceNumberOf(child.path)) - val ids = permits map { child => - sequenceNumberOf(child.path) - } + val ids = permits map { child => sequenceNumberOf(child.path) } val waitqIterator = waitq.iterator() while (waitqIterator.hasNext) { val (promise, permitNode) = waitqIterator.next() @@ -257,9 +247,7 @@ class ZkAsyncSemaphore( Future.collect(permits map numPermitsOf) map { purportedNumPermits => val groupedByNumPermits = purportedNumPermits filter { i => 0 < i - } groupBy { i => - i - } + } groupBy { i => i } val permitsToBelievers = groupedByNumPermits map { case (permits, believers) => (permits, believers.size) } diff --git a/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala b/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala index 05717650f3..e9f4badef7 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala @@ -23,9 +23,7 @@ class ConnectorTest extends WordSpec with MockitoSugar { connector } val nConnectors = 3 - val connectors = 1 to nConnectors map { _ => - mockConnector - } + val connectors = 1 to nConnectors map { _ => mockConnector } val connector = Connector.RoundRobin(connectors: _*) } @@ -33,30 +31,18 @@ class ConnectorTest extends WordSpec with MockitoSugar { val h = new ConnectorSpecHelper import h._ - connectors foreach { x => - assert(x.apply() == Future.never) - } - (1 to 2 * nConnectors) foreach { _ => - connector() - } - connectors foreach { c => - verify(c, times(3)).apply() - } + connectors foreach { x => assert(x.apply() == Future.never) } + (1 to 2 * nConnectors) foreach { _ => connector() } + connectors foreach { c => verify(c, times(3)).apply() } } "release" in { val h = new ConnectorSpecHelper import h._ - connectors foreach { x => - assert(x.release() == Future.never) - } - (1 to 2) foreach { _ => - connector.release() - } - connectors foreach { c => - verify(c, times(3)).release() - } + connectors foreach { x => assert(x.release() == Future.never) } + (1 to 2) foreach { _ => connector.release() } + connectors foreach { c => verify(c, times(3)).release() } } } } diff --git a/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala index 9f944522cc..1e31581e33 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala @@ -30,9 +30,7 @@ class ZNodeTest extends WordSpec with MockitoSugar { val h = new ZNodeSpecHelper import h._ - val zs = (0 to 1) map { _ => - ZNode(zk, "/some/path") - } + val zs = (0 to 1) map { _ => ZNode(zk, "/some/path") } val table = Map(zs(0) -> true) assert(table.keys.toList.contains(zs(0))) assert(table.keys.toList.contains(zs(1))) diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index bc5fb0be41..569cf5c5ee 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -55,7 +55,8 @@ class ZkClientTest extends WordSpec with MockitoSugar { data: Array[Byte] = "".getBytes, acls: Seq[ACL] = zkClient.acl, mode: CreateMode = zkClient.mode - )(wait: => Future[String] + )( + wait: => Future[String] ): Unit = { when( zk.create( @@ -147,8 +148,10 @@ class ZkClientTest extends WordSpec with MockitoSugar { def watchChildren( path: String - )(children: => Future[ZNode.Children] - )(update: => Future[WatchedEvent] + )( + children: => Future[ZNode.Children] + )( + update: => Future[WatchedEvent] ): Unit = { val w = ArgumentCaptor.forClass(classOf[Watcher]) val cb = ArgumentCaptor.forClass(classOf[AsyncCallback.Children2Callback]) @@ -183,8 +186,10 @@ class ZkClientTest extends WordSpec with MockitoSugar { def watchData( path: String - )(result: => Future[ZNode.Data] - )(update: => Future[WatchedEvent] + )( + result: => Future[ZNode.Data] + )( + update: => Future[WatchedEvent] ): Unit = { val w = ArgumentCaptor.forClass(classOf[Watcher]) val cb = ArgumentCaptor.forClass(classOf[AsyncCallback.DataCallback]) @@ -258,9 +263,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { i += 1 Future.exception(connectionLoss) } - .onSuccess { _ => - fail("Unexpected success") - } + .onSuccess { _ => fail("Unexpected success") } .handle { case e: KeeperException.ConnectionLossException => assert(e == connectionLoss) @@ -278,9 +281,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { i += 1 Future.Done } - .onSuccess { _ => - assert(i == 1) - } + .onSuccess { _ => assert(i == 1) } ) } @@ -294,9 +295,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { i += 1 throw rex } - .onSuccess { _ => - fail("Unexpected success") - } + .onSuccess { _ => fail("Unexpected success") } .handle { case e: RuntimeException => assert(e == rex) @@ -313,9 +312,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { i += 1 Future.exception(connectionLoss) } - .onSuccess { _ => - fail("Shouldn't have succeeded") - } + .onSuccess { _ => fail("Shouldn't have succeeded") } .handle { case e: KeeperException.ConnectionLossException => assert(i == 1) @@ -544,16 +541,10 @@ class ZkClientTest extends WordSpec with MockitoSugar { class MonitorHelper extends ExistHelper { val deleted = NodeEvent.Deleted(znode.path) def expectZNodes(n: Int): Unit = { - val results = 0 until n map { _ => - ZNode.Exists(znode, new Stat) - } - results foreach { r => - watch(znode.path)(Future(r.stat))(Future(deleted)) - } + val results = 0 until n map { _ => ZNode.Exists(znode, new Stat) } + results foreach { r => watch(znode.path)(Future(r.stat))(Future(deleted)) } val update = znode.exists.monitor() - results foreach { result => - assert(update.syncWait().get() == result) - } + results foreach { result => assert(update.syncWait().get() == result) } } } @@ -748,9 +739,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { "In every generation there is a chosen one.", "She alone will stand against the vampires the demons and the forces of darkness.", "She is the slayer." - ) map { text => - ZNode.Data(znode, new Stat, text.getBytes) - } + ) map { text => ZNode.Data(znode, new Stat, text.getBytes) } results foreach { result => watchData(znode.path)(Future(result)) { Future(NodeEvent.ChildrenChanged(znode.path)) @@ -781,14 +770,8 @@ class ZkClientTest extends WordSpec with MockitoSugar { val treeRoot = ZNode.Children(zkClient("/arboreal"), new Stat, 'a' to 'e' map { _.toString }) val treeChildren = treeRoot +: ('a' to 'e') - .map { c => - ZNode.Children(treeRoot(c.toString), new Stat, 'a' to c map { _.toString }) - } - .flatMap { z => - z +: z.children.map { c => - ZNode.Children(c, new Stat, Nil) - } - } + .map { c => ZNode.Children(treeRoot(c.toString), new Stat, 'a' to c map { _.toString }) } + .flatMap { z => z +: z.children.map { c => ZNode.Children(c, new Stat, Nil) } } // Lay out node updates for the tree: Add a 'z' node to all nodes named 'a' val updateTree = treeChildren.collect { @@ -800,9 +783,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { }.flatten // Initially, we should get a ZNode.TreeUpdate for each node in the tree with only added nodes - val expectedByPath = treeChildren.map { z => - z.path -> ZNode.TreeUpdate(z, z.children.toSet) - }.toMap + val expectedByPath = treeChildren.map { z => z.path -> ZNode.TreeUpdate(z, z.children.toSet) }.toMap val updatesByPath = updateTree.map { z => val prior: Set[ZNode] = expectedByPath.get(z.path).map { _.added }.getOrElse(Set.empty) @@ -813,9 +794,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { // Create promises for each node in the tree -- satisfying a promise will fire a // ChildrenChanged event for its associated node. val updatePromises = treeChildren.map { _.path -> new Promise[WatchedEvent] }.toMap - treeChildren foreach { z => - watchChildren(z.path)(Future(z))(updatePromises(z.path)) - } + treeChildren foreach { z => watchChildren(z.path)(Future(z))(updatePromises(z.path)) } val offer = treeRoot.monitorTree() treeChildren foreach { _ => @@ -844,9 +823,7 @@ class ZkClientTest extends WordSpec with MockitoSugar { } "ok" in okUpdates { NodeEvent.ChildrenChanged(_) } - "be resilient to disconnect" in okUpdates { _ => - StateEvent.Disconnected() - } + "be resilient to disconnect" in okUpdates { _ => StateEvent.Disconnected() } "stop on session expiration" in { treeChildren foreach { z => diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala index 6fe820b1bc..54b207c643 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala @@ -33,9 +33,7 @@ class ZkAsyncSemaphoreTest extends WordSpec with MockitoSugar with AsyncAssertio } def acquire(sem: ZkAsyncSemaphore) = { - sem.acquire() onSuccess { permit => - permits add permit - } + sem.acquire() onSuccess { permit => permits add permit } } "provide a shared 2-permit semaphore and" should { From 76f9543c641068819447ca9a6676fdb6b9398e2c Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Mon, 30 Mar 2020 20:05:31 +0000 Subject: [PATCH 299/796] finagle-stats: Changed CumulativeGauge to use FuturePool.defaultExecutor Problem Finagle uses FJPool.commonPool for managing its gauges, which adds an extra thread pool to worry about. Solution It's unlikely that gauge management will be a huge burden, so let's just use FuturePool#defaultExecutor and call it a day. JIRA Issues: CSL-8357 Differential Revision: https://phabricator.twitter.biz/D455975 --- util-core/src/main/scala/com/twitter/util/FuturePool.scala | 2 +- .../scala/com/twitter/finagle/stats/CumulativeGauge.scala | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index a56f723a6a..4790f5f6ae 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -69,7 +69,7 @@ object FuturePool { override def toString: String = "FuturePool.immediatePool" } - private lazy val defaultExecutor = Executors.newCachedThreadPool( + private[twitter] lazy val defaultExecutor = Executors.newCachedThreadPool( new NamedPoolThreadFactory("UnboundedFuturePool", makeDaemons = true) ) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index 0232c6b15f..6e004fdd43 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -134,9 +134,12 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => case _ => whenNotPresent(verbosity) } + /** The executor that will be used for expiring gauges */ + def executor: Executor = ForkJoinPool.commonPool() + private[this] def whenNotPresent(verbosity: Verbosity) = new JFunction[Seq[String], CumulativeGauge] { - def apply(key: Seq[String]): CumulativeGauge = new CumulativeGauge { + def apply(key: Seq[String]): CumulativeGauge = new CumulativeGauge(executor) { self.registerGauge(verbosity, key, getValue) // The number of registers starts at `0` because every new gauge will cause a From f6c858f140728cf0ab56650db8506e493d513732 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 1 Apr 2020 17:41:14 +0000 Subject: [PATCH 300/796] Twitter-oss: Prepare OSS libraries for release 20.4.0 Problem We want to release the next versions of our Twitter OSS libraries 20.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9579 Differential Revision: https://phabricator.twitter.biz/D459190 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dde3c86dc8..b18f9da097 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.4.0 +------ + New Features ~~~~~~~~~~~~ * util-core: When looking to add idempotent close behavior, users should mix in `CloseOnce` to diff --git a/README.md b/README.md index 26c94519fb..a71552612a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.3.0" +val utilCore = "com.twitter" %% "util-core" % "20.4.0" ``` # Units diff --git a/build.sbt b/build.sbt index d7161281c2..ca95a86f06 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.4.0-SNAPSHOT" +val releaseVersion = "20.4.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 6c3beaa3de445b463cb6ce60955737331544987b Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 1 Apr 2020 20:20:22 +0000 Subject: [PATCH 301/796] compiler_option_sets and tags are lists not sets This change is a no-op - nothing should be observably different. Some of our tools are more comfortable with lists than sets, so use them more broadly. This change was generated with: ``` rg -g BUILD -l compiler_option_sets | xargs perl -0 -pi -e 's#compiler_option_sets = {([^}]*)}#compiler_option_sets = [\1]#mg' rg -g BUILD -l tags | xargs perl -0 -pi -e 's#tags = {([^}]*)}#tags = [\1]#mg' ``` Differential Revision: https://phabricator.twitter.biz/D459529 --- util-app/src/main/java/com/twitter/app/BUILD | 2 +- util-app/src/main/scala/BUILD | 2 +- util-app/src/test/java/com/twitter/app/BUILD | 2 +- util-app/src/test/scala/com/twitter/app/BUILD | 2 +- util-benchmark/src/main/scala/BUILD | 2 +- util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD | 2 +- util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD | 2 +- util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD | 2 +- util-cache/src/main/scala/com/twitter/cache/BUILD | 2 +- util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD | 2 +- util-cache/src/test/java/com/twitter/cache/BUILD | 2 +- util-cache/src/test/scala/com/twitter/cache/BUILD | 4 ++-- util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD | 2 +- util-codec/src/main/scala/com/twitter/util/BUILD | 2 +- util-codec/src/test/scala/com/twitter/util/BUILD | 2 +- util-core/BUILD | 2 +- util-core/src/main/java/com/twitter/concurrent/BUILD | 2 +- util-core/src/main/java/com/twitter/io/BUILD | 2 +- util-core/src/main/java/com/twitter/service/BUILD | 2 +- util-core/src/main/java/com/twitter/util/BUILD | 2 +- util-core/src/main/java/com/twitter/util/javainterop/BUILD | 2 +- util-core/src/main/scala/com/twitter/concurrent/BUILD | 2 +- util-core/src/main/scala/com/twitter/conversions/BUILD | 2 +- util-core/src/main/scala/com/twitter/io/BUILD | 2 +- util-core/src/main/scala/com/twitter/io/exp/BUILD | 2 +- util-core/src/test/java/com/twitter/concurrent/BUILD | 2 +- util-core/src/test/java/com/twitter/io/BUILD | 2 +- util-core/src/test/java/com/twitter/service/BUILD | 2 +- util-core/src/test/java/com/twitter/util/BUILD | 4 ++-- util-core/src/test/java/com/twitter/util/javainterop/BUILD | 2 +- util-core/src/test/scala/com/twitter/concurrent/BUILD | 2 +- util-core/src/test/scala/com/twitter/conversions/BUILD | 2 +- util-core/src/test/scala/com/twitter/io/BUILD | 2 +- util-core/src/test/scala/com/twitter/io/exp/BUILD | 2 +- util-core/src/test/scala/com/twitter/util/BUILD | 2 +- util-function/src/main/java/com/twitter/function/BUILD | 2 +- util-hashing/src/main/scala/com/twitter/hashing/BUILD | 2 +- util-hashing/src/test/scala/com/twitter/hashing/BUILD | 2 +- .../src/test/scala/com/twitter/util/capturepoints/BUILD | 2 +- util-jvm/src/main/scala/com/twitter/jvm/BUILD | 2 +- util-jvm/src/test/java/com/twitter/jvm/BUILD | 2 +- util-jvm/src/test/scala/com/twitter/jvm/BUILD | 4 ++-- util-lint/src/main/scala/com/twitter/util/lint/BUILD | 2 +- util-lint/src/test/scala/com/twitter/util/lint/BUILD | 2 +- util-logging/src/main/scala/com/twitter/logging/BUILD | 2 +- util-logging/src/test/java/com/twitter/logging/BUILD | 2 +- util-logging/src/test/scala/com/twitter/logging/BUILD | 2 +- util-reflect/BUILD | 4 ++-- util-reflect/src/main/scala/com/twitter/util/reflect/BUILD | 2 +- util-reflect/src/test/scala/com/twitter/util/reflect/BUILD | 2 +- util-registry/src/main/scala/com/twitter/util/registry/BUILD | 2 +- util-registry/src/test/java/com/twitter/util/registry/BUILD | 2 +- util-registry/src/test/scala/com/twitter/util/registry/BUILD | 2 +- util-security/src/main/scala/com/twitter/util/security/BUILD | 2 +- util-security/src/test/java/com/twitter/util/security/BUILD | 2 +- util-security/src/test/scala/com/twitter/util/security/BUILD | 2 +- util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD | 2 +- util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD | 2 +- util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD | 2 +- .../src/main/scala/com/twitter/util/logging/BUILD | 2 +- util-stats/src/main/scala/com/twitter/finagle/stats/BUILD | 2 +- util-stats/src/test/java/com/twitter/finagle/stats/BUILD | 2 +- util-stats/src/test/scala/com/twitter/finagle/stats/BUILD | 2 +- util-test/src/main/scala/com/twitter/logging/BUILD | 2 +- util-test/src/main/scala/com/twitter/util/testing/BUILD | 2 +- util-test/src/test/scala/com/twitter/util/testing/BUILD | 2 +- util-thrift/src/main/scala/com/twitter/util/BUILD | 2 +- util-thrift/src/test/java/com/twitter/util/BUILD | 2 +- util-thrift/src/test/scala/com/twitter/util/BUILD | 2 +- util-tunable/src/main/scala/com/twitter/util/tunable/BUILD | 2 +- .../src/main/scala/com/twitter/util/tunable/linter/BUILD | 2 +- util-tunable/src/test/java/com/twitter/util/tunable/BUILD | 2 +- util-tunable/src/test/scala/com/twitter/util/tunable/BUILD | 2 +- util-zk-test/src/main/scala/com/twitter/zk/BUILD | 2 +- util-zk-test/src/test/java/com/twitter/zk/BUILD | 2 +- util-zk-test/src/test/scala/com/twitter/zk/BUILD | 2 +- util-zk/src/main/scala/com/twitter/zk/BUILD | 2 +- util-zk/src/main/scala/com/twitter/zk/coordination/BUILD | 2 +- util-zk/src/test/scala/com/twitter/zk/BUILD | 2 +- util-zk/src/test/scala/com/twitter/zk/coordination/BUILD | 2 +- 80 files changed, 84 insertions(+), 84 deletions(-) diff --git a/util-app/src/main/java/com/twitter/app/BUILD b/util-app/src/main/java/com/twitter/app/BUILD index edb0841aab..07b35e6f1a 100644 --- a/util-app/src/main/java/com/twitter/app/BUILD +++ b/util-app/src/main/java/com/twitter/app/BUILD @@ -1,5 +1,5 @@ java_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = artifact( org = "com.twitter", name = "util-app-java", diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index 03f4556539..69d5617962 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-app", diff --git a/util-app/src/test/java/com/twitter/app/BUILD b/util-app/src/test/java/com/twitter/app/BUILD index 5dbf5bdfe1..13ab487e8f 100644 --- a/util-app/src/test/java/com/twitter/app/BUILD +++ b/util-app/src/test/java/com/twitter/app/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-app/src/test/scala/com/twitter/app/BUILD b/util-app/src/test/scala/com/twitter/app/BUILD index 088de4a208..9a2f51a5ff 100644 --- a/util-app/src/test/scala/com/twitter/app/BUILD +++ b/util-app/src/test/scala/com/twitter/app/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index d325c96201..c0b4b95339 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -1,6 +1,6 @@ scala_library( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/org/openjdk/jmh:jmh-core", diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD index a2be9cee82..4060da9e15 100644 --- a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-cache-guava", diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD index 5b90d260c3..af19ef0670 100644 --- a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/google/guava", "3rdparty/jvm/junit", diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD index a8f0c5da2d..190c4c3270 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/google/code/findbugs:jsr305", "3rdparty/jvm/com/google/guava", diff --git a/util-cache/src/main/scala/com/twitter/cache/BUILD b/util-cache/src/main/scala/com/twitter/cache/BUILD index bc4e13b68f..baf8b94bd0 100644 --- a/util-cache/src/main/scala/com/twitter/cache/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-cache-cache", diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD index 81597a3ee8..1f304f1b38 100644 --- a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-cache-caffeine", diff --git a/util-cache/src/test/java/com/twitter/cache/BUILD b/util-cache/src/test/java/com/twitter/cache/BUILD index c48082bc18..6c0aff781e 100644 --- a/util-cache/src/test/java/com/twitter/cache/BUILD +++ b/util-cache/src/test/java/com/twitter/cache/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index 48c3388627..fc2504e691 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -4,7 +4,7 @@ junit_tests( "!AbstractLoadingFutureCacheTest.scala", "*.scala", ], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ ":abstract_tests", "3rdparty/jvm/com/github/ben-manes/caffeine", @@ -23,7 +23,7 @@ scala_library( "AbstractFutureCacheTest.scala", "AbstractLoadingFutureCacheTest.scala", ], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index f85bfc8509..8a2e55a065 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/com/google/code/findbugs:jsr305", diff --git a/util-codec/src/main/scala/com/twitter/util/BUILD b/util-codec/src/main/scala/com/twitter/util/BUILD index 74e11682df..e500b175e8 100644 --- a/util-codec/src/main/scala/com/twitter/util/BUILD +++ b/util-codec/src/main/scala/com/twitter/util/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-codec", diff --git a/util-codec/src/test/scala/com/twitter/util/BUILD b/util-codec/src/test/scala/com/twitter/util/BUILD index b8b5baa9da..6eebe20b6a 100644 --- a/util-codec/src/test/scala/com/twitter/util/BUILD +++ b/util-codec/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-core/BUILD b/util-core/BUILD index 4d74fc6d90..9561881e0f 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -23,7 +23,7 @@ scala_library( "src/main/scala-2.12-/com/twitter/util/*.scala", "src/main/scala/com/twitter/util/*.scala", ], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-core-util", diff --git a/util-core/src/main/java/com/twitter/concurrent/BUILD b/util-core/src/main/java/com/twitter/concurrent/BUILD index 04fa4aba73..136832acd6 100644 --- a/util-core/src/main/java/com/twitter/concurrent/BUILD +++ b/util-core/src/main/java/com/twitter/concurrent/BUILD @@ -1,5 +1,5 @@ java_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = artifact( org = "com.twitter", name = "util-core-java-concurrent", diff --git a/util-core/src/main/java/com/twitter/io/BUILD b/util-core/src/main/java/com/twitter/io/BUILD index e2b53131c0..745732e107 100644 --- a/util-core/src/main/java/com/twitter/io/BUILD +++ b/util-core/src/main/java/com/twitter/io/BUILD @@ -1,5 +1,5 @@ java_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = artifact( org = "com.twitter", name = "util-core-java-io", diff --git a/util-core/src/main/java/com/twitter/service/BUILD b/util-core/src/main/java/com/twitter/service/BUILD index 10e7dfa119..927ce868bd 100644 --- a/util-core/src/main/java/com/twitter/service/BUILD +++ b/util-core/src/main/java/com/twitter/service/BUILD @@ -1,6 +1,6 @@ java_library( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = artifact( org = "com.twitter", name = "util-core-java-service", diff --git a/util-core/src/main/java/com/twitter/util/BUILD b/util-core/src/main/java/com/twitter/util/BUILD index b6bf59859c..e3560f9bfc 100644 --- a/util-core/src/main/java/com/twitter/util/BUILD +++ b/util-core/src/main/java/com/twitter/util/BUILD @@ -1,5 +1,5 @@ java_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = artifact( org = "com.twitter", name = "util-core-java", diff --git a/util-core/src/main/java/com/twitter/util/javainterop/BUILD b/util-core/src/main/java/com/twitter/util/javainterop/BUILD index f610b2f422..5e3cf913ce 100644 --- a/util-core/src/main/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/main/java/com/twitter/util/javainterop/BUILD @@ -1,5 +1,5 @@ java_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = artifact( org = "com.twitter", name = "util-core-java-javainterop", diff --git a/util-core/src/main/scala/com/twitter/concurrent/BUILD b/util-core/src/main/scala/com/twitter/concurrent/BUILD index d6406755a6..ae50cf42a2 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/main/scala/com/twitter/concurrent/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-core-concurrent", diff --git a/util-core/src/main/scala/com/twitter/conversions/BUILD b/util-core/src/main/scala/com/twitter/conversions/BUILD index 3684ab4eed..02bf679037 100644 --- a/util-core/src/main/scala/com/twitter/conversions/BUILD +++ b/util-core/src/main/scala/com/twitter/conversions/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-core-conversions", diff --git a/util-core/src/main/scala/com/twitter/io/BUILD b/util-core/src/main/scala/com/twitter/io/BUILD index 92de774949..f858ce31e0 100644 --- a/util-core/src/main/scala/com/twitter/io/BUILD +++ b/util-core/src/main/scala/com/twitter/io/BUILD @@ -1,6 +1,6 @@ scala_library( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-core-io", diff --git a/util-core/src/main/scala/com/twitter/io/exp/BUILD b/util-core/src/main/scala/com/twitter/io/exp/BUILD index d972e0c479..16fab5a7b7 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/BUILD +++ b/util-core/src/main/scala/com/twitter/io/exp/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-core-io-exp", diff --git a/util-core/src/test/java/com/twitter/concurrent/BUILD b/util-core/src/test/java/com/twitter/concurrent/BUILD index d03eadbe88..f561361353 100644 --- a/util-core/src/test/java/com/twitter/concurrent/BUILD +++ b/util-core/src/test/java/com/twitter/concurrent/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD index 2ff51af03b..360630eb47 100644 --- a/util-core/src/test/java/com/twitter/io/BUILD +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/test/java/com/twitter/service/BUILD b/util-core/src/test/java/com/twitter/service/BUILD index c6780f9939..d7702ba8a7 100644 --- a/util-core/src/test/java/com/twitter/service/BUILD +++ b/util-core/src/test/java/com/twitter/service/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "util/util-core/src/main/java/com/twitter/service", diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index dad78b3a3c..5fb41ef273 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -1,7 +1,7 @@ java_library( name = "object_size_calculator", sources = ["ObjectSizeCalculator.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", ], @@ -12,7 +12,7 @@ junit_tests( "!ObjectSizeCalculator.java", "*.java", ], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ ":object_size_calculator", "3rdparty/jvm/junit", diff --git a/util-core/src/test/java/com/twitter/util/javainterop/BUILD b/util-core/src/test/java/com/twitter/util/javainterop/BUILD index 04f9eeac81..06308b494c 100644 --- a/util-core/src/test/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/test/java/com/twitter/util/javainterop/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index b24a60bffe..858b0adb5f 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -8,7 +8,7 @@ junit_tests( # # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` # for this target, we could try enabling this again. - compiler_option_sets = {}, + compiler_option_sets = [], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD index c931f97e67..9ec4a74df7 100644 --- a/util-core/src/test/scala/com/twitter/conversions/BUILD +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -8,7 +8,7 @@ junit_tests( # # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` # for this target, we could try enabling this again. - compiler_option_sets = {}, + compiler_option_sets = [], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 18cfcc7c35..5503eeae99 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -8,7 +8,7 @@ junit_tests( # # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` # for this target, we could try enabling this again. - compiler_option_sets = {}, + compiler_option_sets = [], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-core/src/test/scala/com/twitter/io/exp/BUILD b/util-core/src/test/scala/com/twitter/io/exp/BUILD index b874b8d55c..0614a7c935 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/BUILD +++ b/util-core/src/test/scala/com/twitter/io/exp/BUILD @@ -8,7 +8,7 @@ junit_tests( # # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` # for this target, we could try enabling this again. - compiler_option_sets = {}, + compiler_option_sets = [], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index a27f7556d4..2acf98fd13 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -8,7 +8,7 @@ junit_tests( # # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` # for this target, we could try enabling this again. - compiler_option_sets = {}, + compiler_option_sets = [], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-function/src/main/java/com/twitter/function/BUILD b/util-function/src/main/java/com/twitter/function/BUILD index 70397fbe4d..5ca28b5f22 100644 --- a/util-function/src/main/java/com/twitter/function/BUILD +++ b/util-function/src/main/java/com/twitter/function/BUILD @@ -1,5 +1,5 @@ java_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = artifact( org = "com.twitter", name = "util-function-java", diff --git a/util-hashing/src/main/scala/com/twitter/hashing/BUILD b/util-hashing/src/main/scala/com/twitter/hashing/BUILD index d5368648b1..df6e6067dd 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/main/scala/com/twitter/hashing/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-hashing", diff --git a/util-hashing/src/test/scala/com/twitter/hashing/BUILD b/util-hashing/src/test/scala/com/twitter/hashing/BUILD index 7c2793112a..3c75b82c7b 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/test/scala/com/twitter/hashing/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD index 4af58246ec..ea9323131b 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ "3rdparty/jvm/org/scalatest", diff --git a/util-jvm/src/main/scala/com/twitter/jvm/BUILD b/util-jvm/src/main/scala/com/twitter/jvm/BUILD index a3931e358d..800d969db7 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/main/scala/com/twitter/jvm/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-jvm", diff --git a/util-jvm/src/test/java/com/twitter/jvm/BUILD b/util-jvm/src/test/java/com/twitter/jvm/BUILD index a282525ce0..8f76149ddd 100644 --- a/util-jvm/src/test/java/com/twitter/jvm/BUILD +++ b/util-jvm/src/test/java/com/twitter/jvm/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "util/util-jvm/src/main/scala/com/twitter/jvm", diff --git a/util-jvm/src/test/scala/com/twitter/jvm/BUILD b/util-jvm/src/test/scala/com/twitter/jvm/BUILD index b6029ae29e..5eeb707bdd 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/test/scala/com/twitter/jvm/BUILD @@ -3,7 +3,7 @@ junit_tests( "!EstimatorApp.scala", "*.scala", ], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", @@ -19,7 +19,7 @@ junit_tests( scala_library( name = "estimator_app", sources = ["EstimatorApp.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "util/util-jvm/src/main/scala/com/twitter/jvm", ], diff --git a/util-lint/src/main/scala/com/twitter/util/lint/BUILD b/util-lint/src/main/scala/com/twitter/util/lint/BUILD index 58c0943416..bff1eb9a74 100644 --- a/util-lint/src/main/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/main/scala/com/twitter/util/lint/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-lint", diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index 74348f4430..d0829af8a8 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-logging/src/main/scala/com/twitter/logging/BUILD b/util-logging/src/main/scala/com/twitter/logging/BUILD index b2a4b92dfa..e38a8a3f39 100644 --- a/util-logging/src/main/scala/com/twitter/logging/BUILD +++ b/util-logging/src/main/scala/com/twitter/logging/BUILD @@ -1,6 +1,6 @@ scala_library( sources = ["**/*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-logging", diff --git a/util-logging/src/test/java/com/twitter/logging/BUILD b/util-logging/src/test/java/com/twitter/logging/BUILD index 6b1c43505c..7c8d12286a 100644 --- a/util-logging/src/test/java/com/twitter/logging/BUILD +++ b/util-logging/src/test/java/com/twitter/logging/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "util/util-logging/src/main/scala/com/twitter/logging", diff --git a/util-logging/src/test/scala/com/twitter/logging/BUILD b/util-logging/src/test/scala/com/twitter/logging/BUILD index b41fbadbc2..1329c71e27 100644 --- a/util-logging/src/test/scala/com/twitter/logging/BUILD +++ b/util-logging/src/test/scala/com/twitter/logging/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-reflect/BUILD b/util-reflect/BUILD index e5ef1bc6f2..6aa8fcd0cf 100644 --- a/util-reflect/BUILD +++ b/util-reflect/BUILD @@ -1,8 +1,8 @@ target( - tags = { + tags = [ # Legacy code that shouldn't be used for new services. "deprecated", - }, + ], dependencies = [ "util/util-reflect/src/main/scala", ], diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD index 0277d95b02..3cf145b79b 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-reflect", diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD index 0dda289399..01f19435a2 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-registry/src/main/scala/com/twitter/util/registry/BUILD b/util-registry/src/main/scala/com/twitter/util/registry/BUILD index e2d19187f2..4f0e8e24f7 100644 --- a/util-registry/src/main/scala/com/twitter/util/registry/BUILD +++ b/util-registry/src/main/scala/com/twitter/util/registry/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-registry", diff --git a/util-registry/src/test/java/com/twitter/util/registry/BUILD b/util-registry/src/test/java/com/twitter/util/registry/BUILD index d71095d706..a9142d0144 100644 --- a/util-registry/src/test/java/com/twitter/util/registry/BUILD +++ b/util-registry/src/test/java/com/twitter/util/registry/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-registry/src/test/scala/com/twitter/util/registry/BUILD b/util-registry/src/test/scala/com/twitter/util/registry/BUILD index 2643c0082c..df58be17b1 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/BUILD +++ b/util-registry/src/test/scala/com/twitter/util/registry/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-security/src/main/scala/com/twitter/util/security/BUILD b/util-security/src/main/scala/com/twitter/util/security/BUILD index 7e8bcf3479..6af38ad159 100644 --- a/util-security/src/main/scala/com/twitter/util/security/BUILD +++ b/util-security/src/main/scala/com/twitter/util/security/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-security", diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD index 37a0ef71d1..a02a91fe87 100644 --- a/util-security/src/test/java/com/twitter/util/security/BUILD +++ b/util-security/src/test/java/com/twitter/util/security/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "util/util-core/src/main/scala/com/twitter/util", diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index e50f35069b..18cf84c4c0 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD index e6d6b8578d..6103f9e7d2 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-slf4j", diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD index 772f8de291..07b1519143 100644 --- a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/slf4j:slf4j-api", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index c18a516ee0..565d7b52df 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -3,7 +3,7 @@ junit_tests( "*.java", "*.scala", ], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD index 94adec6123..59aee29399 100644 --- a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-slf4j-jul-bridge", diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD index 61e9b8ad0d..2683c9c9f8 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-stats", diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/BUILD b/util-stats/src/test/java/com/twitter/finagle/stats/BUILD index bee09e61dd..6ff135f5bb 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/test/java/com/twitter/finagle/stats/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "util/util-core/src/main/scala/com/twitter/util", diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD index 8efd337c48..1a07beeade 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-test/src/main/scala/com/twitter/logging/BUILD b/util-test/src/main/scala/com/twitter/logging/BUILD index 723ed1f465..3713dce38b 100644 --- a/util-test/src/main/scala/com/twitter/logging/BUILD +++ b/util-test/src/main/scala/com/twitter/logging/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-test-logging", diff --git a/util-test/src/main/scala/com/twitter/util/testing/BUILD b/util-test/src/main/scala/com/twitter/util/testing/BUILD index 43f24f67a7..eef5aca25d 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/main/scala/com/twitter/util/testing/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-test-testing", diff --git a/util-test/src/test/scala/com/twitter/util/testing/BUILD b/util-test/src/test/scala/com/twitter/util/testing/BUILD index f90a796615..f73116776c 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/test/scala/com/twitter/util/testing/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-thrift/src/main/scala/com/twitter/util/BUILD b/util-thrift/src/main/scala/com/twitter/util/BUILD index 84d006fc2b..30467df3bc 100644 --- a/util-thrift/src/main/scala/com/twitter/util/BUILD +++ b/util-thrift/src/main/scala/com/twitter/util/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-thrift", diff --git a/util-thrift/src/test/java/com/twitter/util/BUILD b/util-thrift/src/test/java/com/twitter/util/BUILD index 58e22a2a2a..a86c35f9ff 100644 --- a/util-thrift/src/test/java/com/twitter/util/BUILD +++ b/util-thrift/src/test/java/com/twitter/util/BUILD @@ -1,5 +1,5 @@ java_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/org/apache/thrift:libthrift", ], diff --git a/util-thrift/src/test/scala/com/twitter/util/BUILD b/util-thrift/src/test/scala/com/twitter/util/BUILD index bc48263b0b..7824a410fe 100644 --- a/util-thrift/src/test/scala/com/twitter/util/BUILD +++ b/util-thrift/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD index 396f9779ac..7a1363362d 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-tunable", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD index 0584e836b5..6956498758 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD @@ -13,7 +13,7 @@ scala_library( jvm_binary( name = "configuration-linter", main = "com.twitter.util.tunable.linter.ConfigurationLinter", - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ ":scala", ], diff --git a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD index fea37d9a99..7ed727c998 100644 --- a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD index 813a604796..c6bd4cfdd1 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-zk-test/src/main/scala/com/twitter/zk/BUILD b/util-zk-test/src/main/scala/com/twitter/zk/BUILD index 3353aa20fd..a85889c44f 100644 --- a/util-zk-test/src/main/scala/com/twitter/zk/BUILD +++ b/util-zk-test/src/main/scala/com/twitter/zk/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-zk-test", diff --git a/util-zk-test/src/test/java/com/twitter/zk/BUILD b/util-zk-test/src/test/java/com/twitter/zk/BUILD index 20568f96f6..ac7f94a3bf 100644 --- a/util-zk-test/src/test/java/com/twitter/zk/BUILD +++ b/util-zk-test/src/test/java/com/twitter/zk/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.java"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "util/util-zk-test/src/main/scala/com/twitter/zk", diff --git a/util-zk-test/src/test/scala/com/twitter/zk/BUILD b/util-zk-test/src/test/scala/com/twitter/zk/BUILD index 8c832f4ed0..b3b809ca20 100644 --- a/util-zk-test/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk-test/src/test/scala/com/twitter/zk/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", diff --git a/util-zk/src/main/scala/com/twitter/zk/BUILD b/util-zk/src/main/scala/com/twitter/zk/BUILD index 126654895e..aab569e76b 100644 --- a/util-zk/src/main/scala/com/twitter/zk/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-zk-zk", diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD index 7df377b309..b3d22bf1a7 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-zk-coordination", diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index f6effdd571..cdf567b65b 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index 1dad482043..0c1a60dfb4 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = ["*.scala"], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", From adb600e68582216124bbf999c2f4c3ab3cfc0f32 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 2 Apr 2020 18:56:55 +0000 Subject: [PATCH 302/796] Remove some extra BUILD files from util-core Move these targets into the main util-core BUILD file, because these files aren't providing much value, and their targets can exist in the place they're already tightly coupled into. NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D460349 --- util-app/src/main/scala/BUILD | 4 +-- util-app/src/test/java/com/twitter/app/BUILD | 2 +- util-app/src/test/scala/com/twitter/app/BUILD | 2 +- util-benchmark/src/main/scala/BUILD | 2 +- .../main/scala/com/twitter/cache/guava/BUILD | 2 +- .../test/java/com/twitter/cache/guava/BUILD | 2 +- .../test/scala/com/twitter/cache/guava/BUILD | 2 +- util-cache/src/main/scala/BUILD | 4 +-- .../src/main/scala/com/twitter/cache/BUILD | 2 +- .../scala/com/twitter/cache/caffeine/BUILD | 2 +- .../src/test/java/com/twitter/cache/BUILD | 2 +- .../src/test/scala/com/twitter/cache/BUILD | 4 +-- .../scala/com/twitter/cache/caffeine/BUILD | 2 +- util-core/BUILD | 27 ++++++++++++++++++- .../main/scala => }/PantsWorkaroundCore.scala | 0 .../main/java/com/twitter/concurrent/BUILD | 4 +-- util-core/src/main/java/com/twitter/io/BUILD | 4 +-- .../src/main/java/com/twitter/util/BUILD | 4 +-- util-core/src/main/scala/BUILD | 23 ---------------- .../main/scala/com/twitter/concurrent/BUILD | 2 +- .../main/scala/com/twitter/conversions/BUILD | 2 +- util-core/src/main/scala/com/twitter/io/BUILD | 2 +- .../src/main/scala/com/twitter/io/exp/BUILD | 2 +- .../src/main/scala/com/twitter/util/BUILD | 4 --- util-core/src/test/java/com/twitter/io/BUILD | 2 +- .../src/test/java/com/twitter/util/BUILD | 2 +- .../src/test/scala/com/twitter/util/BUILD | 2 +- .../com/twitter/util/capturepoints/BUILD | 2 +- util-jvm/src/main/scala/com/twitter/jvm/BUILD | 4 +-- util-jvm/src/test/scala/com/twitter/jvm/BUILD | 2 +- .../main/scala/com/twitter/util/lint/BUILD | 2 +- .../src/main/scala/com/twitter/logging/BUILD | 2 +- .../src/test/scala/com/twitter/logging/BUILD | 2 +- .../main/scala/com/twitter/util/reflect/BUILD | 2 +- .../test/scala/com/twitter/util/reflect/BUILD | 2 +- .../scala/com/twitter/util/registry/BUILD | 2 +- .../scala/com/twitter/util/security/BUILD | 2 +- .../test/java/com/twitter/util/security/BUILD | 2 +- .../scala/com/twitter/util/security/BUILD | 2 +- .../scala/com/twitter/finagle/stats/BUILD | 2 +- .../test/java/com/twitter/finagle/stats/BUILD | 2 +- .../scala/com/twitter/finagle/stats/BUILD | 2 +- .../main/scala/com/twitter/util/tunable/BUILD | 2 +- .../com/twitter/util/tunable/linter/BUILD | 2 +- .../test/scala/com/twitter/util/tunable/BUILD | 2 +- util-zk/src/main/scala/com/twitter/zk/BUILD | 4 +-- .../scala/com/twitter/zk/coordination/BUILD | 4 +-- util-zk/src/test/scala/com/twitter/zk/BUILD | 2 +- .../scala/com/twitter/zk/coordination/BUILD | 2 +- 49 files changed, 80 insertions(+), 82 deletions(-) rename util-core/{src/main/scala => }/PantsWorkaroundCore.scala (100%) delete mode 100644 util-core/src/main/scala/BUILD delete mode 100644 util-core/src/main/scala/com/twitter/util/BUILD diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index 69d5617962..b46efc9d04 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -9,12 +9,12 @@ scala_library( dependencies = [ "util/util-app/src/main/java/com/twitter/app", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-registry/src/main/scala/com/twitter/util/registry", ], exports = [ "util/util-app/src/main/java/com/twitter/app", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-app/src/test/java/com/twitter/app/BUILD b/util-app/src/test/java/com/twitter/app/BUILD index 13ab487e8f..caa454b38c 100644 --- a/util-app/src/test/java/com/twitter/app/BUILD +++ b/util-app/src/test/java/com/twitter/app/BUILD @@ -6,7 +6,7 @@ junit_tests( "3rdparty/jvm/org/scala-lang:scala-library", "util/util-app/src/main/java/com/twitter/app", "util/util-app/src/main/scala", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-function/src/main/java/com/twitter/function", ], ) diff --git a/util-app/src/test/scala/com/twitter/app/BUILD b/util-app/src/test/scala/com/twitter/app/BUILD index 9a2f51a5ff..625b5e5f3c 100644 --- a/util-app/src/test/scala/com/twitter/app/BUILD +++ b/util-app/src/test/scala/com/twitter/app/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/scalatest", "util/util-app/src/main/scala", "util/util-app/src/test/resources", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index c0b4b95339..5f995d2941 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -5,7 +5,7 @@ scala_library( "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/org/openjdk/jmh:jmh-core", "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/scala", + "util/util-core:scala", "util/util-hashing/src/main/scala", "util/util-stats/src/main/scala", ], diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD index 4060da9e15..cbd02c37cc 100644 --- a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD @@ -8,6 +8,6 @@ scala_library( dependencies = [ "3rdparty/jvm/com/google/guava", "util/util-cache/src/main/scala/com/twitter/cache", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD index af19ef0670..508db84ad5 100644 --- a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD @@ -7,6 +7,6 @@ junit_tests( "3rdparty/jvm/org/scala-lang:scala-library", "util/util-cache-guava/src/main/scala/com/twitter/cache/guava", "util/util-cache/src/main/scala/com/twitter/cache", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD index 190c4c3270..49a911f633 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD @@ -9,6 +9,6 @@ junit_tests( "util/util-cache-guava/src/main/scala/com/twitter/cache/guava", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-cache/src/main/scala/BUILD b/util-cache/src/main/scala/BUILD index b563354662..9fade1261a 100644 --- a/util-cache/src/main/scala/BUILD +++ b/util-cache/src/main/scala/BUILD @@ -9,11 +9,11 @@ scala_library( dependencies = [ "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], exports = [ "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-cache/src/main/scala/com/twitter/cache/BUILD b/util-cache/src/main/scala/com/twitter/cache/BUILD index baf8b94bd0..3a3ff184c8 100644 --- a/util-cache/src/main/scala/com/twitter/cache/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/BUILD @@ -6,6 +6,6 @@ scala_library( repo = artifactory, ), dependencies = [ - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD index 1f304f1b38..3e53ea41d0 100644 --- a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD @@ -8,6 +8,6 @@ scala_library( dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "util/util-cache/src/main/scala/com/twitter/cache", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-cache/src/test/java/com/twitter/cache/BUILD b/util-cache/src/test/java/com/twitter/cache/BUILD index 6c0aff781e..84243db2a9 100644 --- a/util-cache/src/test/java/com/twitter/cache/BUILD +++ b/util-cache/src/test/java/com/twitter/cache/BUILD @@ -6,6 +6,6 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scala-lang:scala-library", "util/util-cache/src/main/scala/com/twitter/cache", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index fc2504e691..3fec9f1e3d 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -13,7 +13,7 @@ junit_tests( "3rdparty/jvm/org/scalatest", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) @@ -29,6 +29,6 @@ scala_library( "3rdparty/jvm/org/scalatest", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index 8a2e55a065..c34ffeecc7 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -10,6 +10,6 @@ junit_tests( "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-core/BUILD b/util-core/BUILD index 9561881e0f..3b228be402 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -1,7 +1,7 @@ target( dependencies = [ + ":scala", "util/util-core/src/main/java", - "util/util-core/src/main/scala", ], ) @@ -41,3 +41,28 @@ scala_library( "util/util-function/src/main/java", ], ) + +scala_library( + # This target exists only to act as an aggregator jar. + name = "scala", + sources = ["PantsWorkaroundCore.scala"], + provides = scala_artifact( + org = "com.twitter", + name = "util-core", + repo = artifactory, + ), + dependencies = [ + ":util-core-util", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-core/src/main/scala/com/twitter/io/exp", + ], + exports = [ + ":util-core-util", + "util/util-core/src/main/scala/com/twitter/concurrent", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-core/src/main/scala/com/twitter/io/exp", + ], +) diff --git a/util-core/src/main/scala/PantsWorkaroundCore.scala b/util-core/PantsWorkaroundCore.scala similarity index 100% rename from util-core/src/main/scala/PantsWorkaroundCore.scala rename to util-core/PantsWorkaroundCore.scala diff --git a/util-core/src/main/java/com/twitter/concurrent/BUILD b/util-core/src/main/java/com/twitter/concurrent/BUILD index 136832acd6..a89617e9a8 100644 --- a/util-core/src/main/java/com/twitter/concurrent/BUILD +++ b/util-core/src/main/java/com/twitter/concurrent/BUILD @@ -7,10 +7,10 @@ java_library( ), dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/scala", + "util/util-core:scala", ], exports = [ "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/scala", + "util/util-core:scala", ], ) diff --git a/util-core/src/main/java/com/twitter/io/BUILD b/util-core/src/main/java/com/twitter/io/BUILD index 745732e107..92ea031f28 100644 --- a/util-core/src/main/java/com/twitter/io/BUILD +++ b/util-core/src/main/java/com/twitter/io/BUILD @@ -7,10 +7,10 @@ java_library( ), dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/scala", + "util/util-core:scala", ], exports = [ "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/scala", + "util/util-core:scala", ], ) diff --git a/util-core/src/main/java/com/twitter/util/BUILD b/util-core/src/main/java/com/twitter/util/BUILD index e3560f9bfc..2af6b2cb13 100644 --- a/util-core/src/main/java/com/twitter/util/BUILD +++ b/util-core/src/main/java/com/twitter/util/BUILD @@ -7,10 +7,10 @@ java_library( ), dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/scala", + "util/util-core:scala", ], exports = [ "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/scala", + "util/util-core:scala", ], ) diff --git a/util-core/src/main/scala/BUILD b/util-core/src/main/scala/BUILD deleted file mode 100644 index afe099a500..0000000000 --- a/util-core/src/main/scala/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -scala_library( - # This target exists only to act as an aggregator jar. - sources = ["PantsWorkaroundCore.scala"], - provides = scala_artifact( - org = "com.twitter", - name = "util-core", - repo = artifactory, - ), - dependencies = [ - "util/util-core/src/main/scala/com/twitter/concurrent", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/io", - "util/util-core/src/main/scala/com/twitter/io/exp", - "util/util-core/src/main/scala/com/twitter/util", - ], - exports = [ - "util/util-core/src/main/scala/com/twitter/concurrent", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/io", - "util/util-core/src/main/scala/com/twitter/io/exp", - "util/util-core/src/main/scala/com/twitter/util", - ], -) diff --git a/util-core/src/main/scala/com/twitter/concurrent/BUILD b/util-core/src/main/scala/com/twitter/concurrent/BUILD index ae50cf42a2..b01b7650c4 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/main/scala/com/twitter/concurrent/BUILD @@ -6,6 +6,6 @@ scala_library( repo = artifactory, ), dependencies = [ - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-core/src/main/scala/com/twitter/conversions/BUILD b/util-core/src/main/scala/com/twitter/conversions/BUILD index 02bf679037..a7f5379883 100644 --- a/util-core/src/main/scala/com/twitter/conversions/BUILD +++ b/util-core/src/main/scala/com/twitter/conversions/BUILD @@ -6,6 +6,6 @@ scala_library( repo = artifactory, ), dependencies = [ - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-core/src/main/scala/com/twitter/io/BUILD b/util-core/src/main/scala/com/twitter/io/BUILD index f858ce31e0..76baf480e2 100644 --- a/util-core/src/main/scala/com/twitter/io/BUILD +++ b/util-core/src/main/scala/com/twitter/io/BUILD @@ -8,6 +8,6 @@ scala_library( ), dependencies = [ "util/util-core/src/main/scala/com/twitter/concurrent", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-core/src/main/scala/com/twitter/io/exp/BUILD b/util-core/src/main/scala/com/twitter/io/exp/BUILD index 16fab5a7b7..0c58937e7c 100644 --- a/util-core/src/main/scala/com/twitter/io/exp/BUILD +++ b/util-core/src/main/scala/com/twitter/io/exp/BUILD @@ -7,6 +7,6 @@ scala_library( ), dependencies = [ "util/util-core/src/main/scala/com/twitter/io", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-core/src/main/scala/com/twitter/util/BUILD b/util-core/src/main/scala/com/twitter/util/BUILD deleted file mode 100644 index 3709da9555..0000000000 --- a/util-core/src/main/scala/com/twitter/util/BUILD +++ /dev/null @@ -1,4 +0,0 @@ -alias( - name = "util", - target = "util/util-core:util-core-util", -) diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD index 360630eb47..ca525eb773 100644 --- a/util-core/src/test/java/com/twitter/io/BUILD +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -4,7 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core:scala", "util/util-core/src/main/java/com/twitter/io", - "util/util-core/src/main/scala", ], ) diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index 5fb41ef273..9c53e14d93 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -18,6 +18,6 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", "util/util-core/src/main/java/com/twitter/util", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index 2acf98fd13..f94ee19552 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -15,7 +15,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-core/src/test/java/com/twitter/util:object_size_calculator", ], ) diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD index ea9323131b..6c621f2dca 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD @@ -4,7 +4,7 @@ junit_tests( strict_deps = True, dependencies = [ "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-intellij/src/main/resources", ], ) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/BUILD b/util-jvm/src/main/scala/com/twitter/jvm/BUILD index 800d969db7..f45ac1032f 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/main/scala/com/twitter/jvm/BUILD @@ -8,10 +8,10 @@ scala_library( dependencies = [ "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], exports = [ - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/BUILD b/util-jvm/src/test/scala/com/twitter/jvm/BUILD index 5eeb707bdd..87df917b93 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/test/scala/com/twitter/jvm/BUILD @@ -9,7 +9,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", "util/util-app/src/main/scala", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-jvm/src/main/scala/com/twitter/jvm", "util/util-logging/src/main/scala", "util/util-test/src/main/scala", diff --git a/util-lint/src/main/scala/com/twitter/util/lint/BUILD b/util-lint/src/main/scala/com/twitter/util/lint/BUILD index bff1eb9a74..ee4416b514 100644 --- a/util-lint/src/main/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/main/scala/com/twitter/util/lint/BUILD @@ -6,6 +6,6 @@ scala_library( repo = artifactory, ), dependencies = [ - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-logging/src/main/scala/com/twitter/logging/BUILD b/util-logging/src/main/scala/com/twitter/logging/BUILD index e38a8a3f39..fcf8ff1edd 100644 --- a/util-logging/src/main/scala/com/twitter/logging/BUILD +++ b/util-logging/src/main/scala/com/twitter/logging/BUILD @@ -10,7 +10,7 @@ scala_library( "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], exports = [ diff --git a/util-logging/src/test/scala/com/twitter/logging/BUILD b/util-logging/src/test/scala/com/twitter/logging/BUILD index 1329c71e27..20e0ad38c4 100644 --- a/util-logging/src/test/scala/com/twitter/logging/BUILD +++ b/util-logging/src/test/scala/com/twitter/logging/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/scalatest", "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/io", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-logging/src/main/scala/com/twitter/logging", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD index 3cf145b79b..5f40d5e8d0 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD @@ -7,7 +7,7 @@ scala_library( ), dependencies = [ "3rdparty/jvm/cglib", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], exports = [ "3rdparty/jvm/cglib", diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD index 01f19435a2..bdf3e30aca 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -4,7 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-reflect/src/main/scala/com/twitter/util/reflect", ], ) diff --git a/util-registry/src/main/scala/com/twitter/util/registry/BUILD b/util-registry/src/main/scala/com/twitter/util/registry/BUILD index 4f0e8e24f7..4f4cbd55e1 100644 --- a/util-registry/src/main/scala/com/twitter/util/registry/BUILD +++ b/util-registry/src/main/scala/com/twitter/util/registry/BUILD @@ -6,6 +6,6 @@ scala_library( repo = artifactory, ), dependencies = [ - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-security/src/main/scala/com/twitter/util/security/BUILD b/util-security/src/main/scala/com/twitter/util/security/BUILD index 6af38ad159..f60689e460 100644 --- a/util-security/src/main/scala/com/twitter/util/security/BUILD +++ b/util-security/src/main/scala/com/twitter/util/security/BUILD @@ -6,7 +6,7 @@ scala_library( repo = artifactory, ), dependencies = [ - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-logging/src/main/scala/com/twitter/logging", ], ) diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD index a02a91fe87..55596161b8 100644 --- a/util-security/src/test/java/com/twitter/util/security/BUILD +++ b/util-security/src/test/java/com/twitter/util/security/BUILD @@ -3,7 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-security/src/main/scala/com/twitter/util/security", ], ) diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index 18cf84c4c0..6bf546126a 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -5,7 +5,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", "util/util-core/src/main/scala/com/twitter/io", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-security/src/main/scala/com/twitter/util/security", "util/util-security/src/test/resources", ], diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD index 2683c9c9f8..0ba5e8cbc2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD @@ -13,7 +13,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/com/google/code/findbugs:jsr305", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-lint/src/main/scala/com/twitter/util/lint", ], exports = [ diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/BUILD b/util-stats/src/test/java/com/twitter/finagle/stats/BUILD index 6ff135f5bb..e01b10ddd7 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/test/java/com/twitter/finagle/stats/BUILD @@ -3,7 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD index 1a07beeade..2116031fbb 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD index 7a1363362d..8e00fbff68 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD @@ -12,7 +12,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/concurrent", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], exports = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD index 6956498758..69e32e255e 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD @@ -5,7 +5,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "util/util-app/src/main/scala", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-tunable/src/main/scala/com/twitter/util/tunable", ], ) diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD index c6bd4cfdd1..f6d67c888e 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -8,7 +8,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-tunable/src/main/scala/com/twitter/util/tunable", "util/util-tunable/src/test/resources", ], diff --git a/util-zk/src/main/scala/com/twitter/zk/BUILD b/util-zk/src/main/scala/com/twitter/zk/BUILD index aab569e76b..8bb00996d4 100644 --- a/util-zk/src/main/scala/com/twitter/zk/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/BUILD @@ -9,13 +9,13 @@ scala_library( "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-logging/src/main/scala/com/twitter/logging", ], exports = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD index b3d22bf1a7..470902026b 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD @@ -8,13 +8,13 @@ scala_library( dependencies = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", "util/util-core/src/main/scala/com/twitter/concurrent", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-zk/src/main/scala/com/twitter/zk", ], exports = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", "util/util-core/src/main/scala/com/twitter/concurrent", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-zk/src/main/scala/com/twitter/zk", ], ) diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index cdf567b65b..b233b1d006 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-logging/src/main/scala/com/twitter/logging", "util/util-zk/src/main/scala/com/twitter/zk", ], diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index 0c1a60dfb4..ebe6b6fc15 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", "util/util-zk/src/main/scala/com/twitter/zk", "util/util-zk/src/main/scala/com/twitter/zk/coordination", ], From d24c28e8a72c44f8b70a071e3a8e3e1763d1cfef Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 2 Apr 2020 20:27:18 +0000 Subject: [PATCH 303/796] finagle-stats: Caches the JsonExporter's regex matching Problem finagle-stats doesn't cache which metrics JsonExporter matches against, which may lead to reprocessing a lot of regex, if you have many metrics or a complicated regex. Solution Cache the metrics that JsonExporter matches against, and provide a utility in util-stats to make it easy to do such a thing. JIRA Issues: CSL-9544 Differential Revision: https://phabricator.twitter.biz/D459391 --- CHANGELOG.rst | 8 +++ .../twitter/finagle/stats/CachedRegex.scala | 53 +++++++++++++++++++ .../finagle/stats/CachedRegexTest.scala | 41 ++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala create mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/CachedRegexTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b18f9da097..9da979e31b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,14 @@ Breaking API Changes `c.t.hashing.KetamaDistributor` => `ConsistentHashingDistributor`. ``PHAB_ID=D449929`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-stats: Provide CachedRegex, a function that filters a + Map[String, Number] => Map[String, Number] and caches which keys to filter on + based on a regex. Useful for filtering down metrics in the style that Finagle + typically recommends ``PHAB_ID=D459391``. + 20.3.0 ------ diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala new file mode 100644 index 0000000000..d34b475380 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala @@ -0,0 +1,53 @@ +package com.twitter.finagle.stats + +import java.util.concurrent.ConcurrentHashMap +import java.util.function.Consumer +import scala.util.matching.Regex + +/** + * Caches the results of evaluating a given regex + * + * Caches whether the regex matches each of a list of strings. When a map is + * passed to the CachedRegex, it will strip out all of the strings that match + * the regex. + */ +private[stats] class CachedRegex(regex: Regex) + extends (collection.Map[String, Number] => collection.Map[String, Number]) { + // public for testing + // a mapping from a metric key to whether the regex matches it or not + // we allow this to race as long as we're confident that the result is correct + // since a given key will always compute the same value, this is safe to race + val regexMatchCache = new ConcurrentHashMap[String, java.lang.Boolean] + + private[this] val filterFn: String => Boolean = { key => + // we unfortunately need to rely on escape analysis to shave this allocation + // because otherwise java.util.Map#get that returns a null for a plausibly + // primitive type in scala gets casted to a null + // + // the alternative is to first check containsKey and then do a subsequent + // lookup, but that risks an actually potentially bad race condition + // (a potential false negative) + val cached = regexMatchCache.get(key) + val matched = if (cached ne null) { + Boolean.unbox(cached) + } else { + val result = regex.pattern.matcher(key).matches() + // adds new entries + regexMatchCache.put(key, result) + result + } + !matched + } + + def apply(samples: collection.Map[String, Number]): collection.Map[String, Number] = { + // don't really want this to be executed in parallel + regexMatchCache.forEachKey(Long.MaxValue, new Consumer[String] { + def accept(key: String): Unit = { + if (!samples.contains(key)) { + regexMatchCache.remove(key) + } + } + }) + samples.filterKeys(filterFn).toMap + } +} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CachedRegexTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CachedRegexTest.scala new file mode 100644 index 0000000000..b7aecd4628 --- /dev/null +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CachedRegexTest.scala @@ -0,0 +1,41 @@ +package com.twitter.finagle.stats + +import org.scalatest.FunSuite +import scala.jdk.CollectionConverters._ + +class CachedRegexTest extends FunSuite { + test("Lets through things it should let through") { + val cachedRegex = new CachedRegex("foo".r) + val original: Map[String, Number] = Map("bar" -> 3) + assert(cachedRegex(original) == original) + assert(cachedRegex.regexMatchCache.asScala == Map("bar" -> false)) + } + + test("Doesn't let through things it shouldn't let through") { + val cachedRegex = new CachedRegex("foo".r) + val original: Map[String, Number] = Map("foo" -> 3) + assert(cachedRegex(original) == Map.empty) + assert(cachedRegex.regexMatchCache.asScala == Map("foo" -> true)) + } + + test("Keeps track of things that are repeated") { + val cachedRegex = new CachedRegex("foo".r) + val original: Map[String, Number] = Map("foo" -> 3) + assert(cachedRegex(original) == Map.empty) + assert(cachedRegex.regexMatchCache.asScala == Map("foo" -> true)) + + assert(cachedRegex(original) == Map.empty) + assert(cachedRegex.regexMatchCache.asScala == Map("foo" -> true)) + } + + test("Strips keys that don't reappear") { + val cachedRegex = new CachedRegex("foo".r) + val original: Map[String, Number] = Map("foo" -> 3) + assert(cachedRegex(original) == Map.empty) + assert(cachedRegex.regexMatchCache.asScala == Map("foo" -> true)) + + val updated: Map[String, Number] = Map("bar" -> 4) + assert(cachedRegex(updated) == updated) + assert(cachedRegex.regexMatchCache.asScala == Map("bar" -> false)) + } +} From a4442215216fc69aedc701f3ef648a37c5c0fe7a Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 2 Apr 2020 23:15:32 +0000 Subject: [PATCH 304/796] Twitter-oss: update OSS libraries post-release to 20.5.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D460793 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a71552612a..08f8599dc6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index ca95a86f06..3fc06a742b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.4.0" +val releaseVersion = "20.5.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 1d8dc66d6e03783316e17cc8109f8959cec5b159 Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Thu, 2 Apr 2020 23:42:55 +0000 Subject: [PATCH 305/796] util-core: Add missing deps Problem It looks like we implicitly leak util-core onto the compile classpath, so theses have been compiling without explicit deps for a bit. Solution Soon we will stop that leaking, so explicitly add the deps. Differential Revision: https://phabricator.twitter.biz/D460263 --- util-core/src/test/java/com/twitter/util/BUILD | 3 ++- util-core/src/test/scala/com/twitter/concurrent/BUILD | 1 + util-core/src/test/scala/com/twitter/conversions/BUILD | 1 + util-core/src/test/scala/com/twitter/io/BUILD | 2 ++ util-core/src/test/scala/com/twitter/io/exp/BUILD | 2 ++ 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index 9c53e14d93..1176d163f7 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -15,9 +15,10 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ ":object_size_calculator", + "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-core/src/main/java/com/twitter/util", "util/util-core:util-core-util", + "util/util-core/src/main/java/com/twitter/util", ], ) diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index 858b0adb5f..335dfbe783 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -14,6 +14,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/test/java/com/twitter/util:object_size_calculator", diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD index 9ec4a74df7..647433c77c 100644 --- a/util-core/src/test/scala/com/twitter/conversions/BUILD +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -13,6 +13,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", ], ) diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 5503eeae99..c79cbd83ad 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -14,6 +14,8 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/main/scala/com/twitter/io", ], diff --git a/util-core/src/test/scala/com/twitter/io/exp/BUILD b/util-core/src/test/scala/com/twitter/io/exp/BUILD index 0614a7c935..dae848176c 100644 --- a/util-core/src/test/scala/com/twitter/io/exp/BUILD +++ b/util-core/src/test/scala/com/twitter/io/exp/BUILD @@ -13,7 +13,9 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io", "util/util-core/src/main/scala/com/twitter/io/exp", ], ) From 6d258f76cdf9ffa101600c988db6fefde343ae2e Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 7 Apr 2020 19:01:01 +0000 Subject: [PATCH 306/796] python 2 to 3 updates Problem/Solution We have many resources that are Python 2 based, which is no longer supported. We need to move them to Python 3. JIRA Issues: CSL-9429 Differential Revision: https://phabricator.twitter.biz/D437372 --- doc/src/sphinx/conf.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/src/sphinx/conf.py b/doc/src/sphinx/conf.py index 6369da3243..65dc7b6c35 100644 --- a/doc/src/sphinx/conf.py +++ b/doc/src/sphinx/conf.py @@ -29,8 +29,8 @@ 'index_logo': None } -project = u'Util' -copyright = u'{} Twitter, Inc'.format(datetime.datetime.now().year) +project = 'Util' +copyright = '{} Twitter, Inc'.format(datetime.datetime.now().year) htmlhelp_basename = "util" release = sbt_versions.find_release(os.path.abspath('../../../project/Build.scala')) version = sbt_versions.release_to_version(release) @@ -41,12 +41,12 @@ try: __import__('flask_theme_support') except ImportError as e: - print '-' * 74 - print 'Warning: Flask themes unavailable. Building with default theme' - print 'If you want the Flask themes, run this command and build again:' - print - print ' git submodule update --init' - print '-' * 74 + print('-' * 74) + print('Warning: Flask themes unavailable. Building with default theme') + print('If you want the Flask themes, run this command and build again:') + print() + print(' git submodule update --init') + print('-' * 74) pygments_style = 'tango' html_theme = 'default' From 6355644a00a2ac852c9e839e9c7057fce1beabb3 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 7 Apr 2020 20:02:02 +0000 Subject: [PATCH 307/796] util-routing: introduce experimental generic routing interfaces Problem Throughout the Twitter stack, we have multiple implementations and representations of "routing" - taking an input and searching for an associated destination. While the existing implementations exist primarily at the `c.t.finagle.Service` layer, the utility of a `Router` should not be limited to a `c.t.finagle.Service`. Solution We introduce the `Router` and `RouterBuilder` traits that can be used as a generic foundation for defining and building routers. JIRA Issues: CSL-8943 Differential Revision: https://phabricator.twitter.biz/D455757 --- build.sbt | 10 ++ util-routing/README.md | 5 + .../twitter/util/routing/AbstractRouter.scala | 43 +++++ .../main/scala/com/twitter/util/routing/BUILD | 12 ++ .../util/routing/ClosedRouterException.scala | 13 ++ .../routing/RouteValidationException.scala | 10 ++ .../com/twitter/util/routing/Router.scala | 107 ++++++++++++ .../twitter/util/routing/RouterBuilder.scala | 70 ++++++++ .../test/java/com/twitter/util/routing/BUILD | 13 ++ .../util/routing/RouterCompilationTest.java | 112 ++++++++++++ .../test/scala/com/twitter/util/routing/BUILD | 12 ++ .../util/routing/RouterBuilderTest.scala | 54 ++++++ .../com/twitter/util/routing/RouterTest.scala | 163 ++++++++++++++++++ .../com/twitter/util/routing/dynamic/BUILD | 7 + .../util/routing/dynamic/DynamicRoute.scala | 12 ++ .../util/routing/dynamic/DynamicRouter.scala | 30 ++++ .../twitter/util/routing/dynamic/Method.scala | 8 + .../util/routing/dynamic/Request.scala | 3 + .../twitter/util/routing/dynamic/Result.scala | 8 + .../com/twitter/util/routing/simple/BUILD | 7 + .../util/routing/simple/SimpleRoute.scala | 3 + .../util/routing/simple/SimpleRouter.scala | 23 +++ 22 files changed, 725 insertions(+) create mode 100644 util-routing/README.md create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/BUILD create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/ClosedRouterException.scala create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/RouteValidationException.scala create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/Router.scala create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala create mode 100644 util-routing/src/test/java/com/twitter/util/routing/BUILD create mode 100644 util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/BUILD create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRoute.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/dynamic/Method.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/dynamic/Request.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/dynamic/Result.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRoute.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRouter.scala diff --git a/build.sbt b/build.sbt index 3fc06a742b..8e35186ea4 100644 --- a/build.sbt +++ b/build.sbt @@ -203,6 +203,7 @@ lazy val util = Project( utilLogging, utilReflect, utilRegistry, + utilRouting, utilSecurity, utilSlf4jApi, utilSlf4jJulBridge, @@ -447,6 +448,15 @@ lazy val utilStats = Project( } ).dependsOn(utilCore, utilLint) +lazy val utilRouting = Project( + id = "util-routing", + base = file("util-routing") +).settings( + sharedSettings + ).settings( + name := "util-routing" + ).dependsOn(utilCore, utilLogging) + lazy val utilTest = Project( id = "util-test", base = file("util-test") diff --git a/util-routing/README.md b/util-routing/README.md new file mode 100644 index 0000000000..fb0efce121 --- /dev/null +++ b/util-routing/README.md @@ -0,0 +1,5 @@ +# util-routing + +`util-routing` is a project containing base interfaces for routing an input to a destination. +This project should be considered experimental and the likelihood of API changes which result +in code breakage or removal is high. diff --git a/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala b/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala new file mode 100644 index 0000000000..f4307dea69 --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala @@ -0,0 +1,43 @@ +package com.twitter.util.routing + +import com.twitter.logging.Logger +import com.twitter.util.Try +import java.lang.{Iterable => JIterable} +import java.util.Optional +import scala.jdk.CollectionConverters._ + +/** + * Java-friendly version of [[Router]] + * + * @param label A label used for identifying this Router (i.e. for distinguishing between [[Router]] + * instances in error messages or for StatsReceiver scope). + * @param jRoutes All of the [[Route]] routes contained within this [[Router]], represented as + * a Java [[JIterable]]. + * @tparam Input The Input type used to determine a route destination + * @tparam Route The output/resulting route type + */ +abstract class AbstractRouter[Input, Route](val label: String, jRoutes: JIterable[Route]) + extends Router[Input, Route] { + + /** The [[Logger]] to use for this [[Router]]. Can be overridden to customize. */ + def getLogger(): Logger = Router.Log + + override final def logger: Logger = getLogger() + + /** Java-friendly version of [[apply]] */ + final def route(input: Input): Optional[Route] = apply(input) match { + case Some(route) => Optional.of(route) + case _ => Optional.empty() + } + + /** Java-friendly version of [[find]] */ + protected def findAny(input: Input): Optional[Route] + + override final protected def find(input: Input): Option[Route] = { + val found = findAny(input) + Try(found.get()).toOption + } + + override final val routes: Iterable[Route] = jRoutes.asScala + +} diff --git a/util-routing/src/main/scala/com/twitter/util/routing/BUILD b/util-routing/src/main/scala/com/twitter/util/routing/BUILD new file mode 100644 index 0000000000..4b6436347d --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/BUILD @@ -0,0 +1,12 @@ +scala_library( + compiler_option_sets = {"fatal_warnings"}, + provides = scala_artifact( + org = "com.twitter", + name = "util-routing", + repo = artifactory, + ), + dependencies = [ + "util/util-core:util-core-util", + "util/util-logging/src/main/scala", + ], +) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/ClosedRouterException.scala b/util-routing/src/main/scala/com/twitter/util/routing/ClosedRouterException.scala new file mode 100644 index 0000000000..d6ae1f5a53 --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/ClosedRouterException.scala @@ -0,0 +1,13 @@ +package com.twitter.util.routing + +import scala.util.control.NoStackTrace + +/** + * Exception thrown when an attempt to route an `Input` to a [[Router]] where [[close()]] has been + * initiated on the [[Router]]. + * + * @param router The [[Router]] where [[close()]] has already been initiated. + */ +case class ClosedRouterException(router: Router[_, _]) + extends RuntimeException(s"Attempting to route on a closed router with label '${router.label}'") + with NoStackTrace diff --git a/util-routing/src/main/scala/com/twitter/util/routing/RouteValidationException.scala b/util-routing/src/main/scala/com/twitter/util/routing/RouteValidationException.scala new file mode 100644 index 0000000000..ddfa222c64 --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/RouteValidationException.scala @@ -0,0 +1,10 @@ +package com.twitter.util.routing + +/** + * Exception thrown when a [[RouterBuilder]] encounters a `Route` that is not valid for + * the [[Router]] type it is building. + */ +case class RouteValidationException(msg: String, cause: Throwable) extends Exception(msg, cause) { + def this(msg: String) = this(msg, null) + def this(cause: Throwable) = this(null, cause) +} diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Router.scala b/util-routing/src/main/scala/com/twitter/util/routing/Router.scala new file mode 100644 index 0000000000..ad30a5b4b4 --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/Router.scala @@ -0,0 +1,107 @@ +package com.twitter.util.routing + +import com.twitter.logging.Logger +import com.twitter.util.{Closable, ClosableOnce, Future, Time} +import scala.util.control.NonFatal + +object Router { + val Log = Logger.get(Router.getClass) +} + +/** + * A generic interface for routing an input to an optional matching route. + * + * @tparam Input The Input type used when determining a route destination + * @tparam Route The resulting route type. A [[Route]] may have dynamic + * properties and may be found to satisfy multiple [[Input]] + * instances. As there may not be a 1-to-1 mapping of + * [[Input]] to [[Route]], it is encouraged that a [[Route]] + * encapsulates its relationship to an [[Input]]. + * + * An example to consider would be an HTTP [[Router]]. + * An HTTP request contains a Method (i.e. GET, POST) and + * a URI. A [[Router]] for a REST API may match "GET /users/{id}" + * for multiple HTTP requests (i.e. "GET /users/123" AND "GET /users/456" would map + * to the same destination [[Route]]). As a result, the [[Route]] in this example + * should be aware of the method, path, and and the destination/logic responsible + * for handling an [[Input]] that matches. + * + * Another property to consider for a [[Route]] is "uniqueness". It may or may not + * be desirable for a [[Router]] to contain multiple routes that correspond to + * the same [[Input]]. + * + * @note A [[Router]] should be considered immutable unless explicitly noted. + */ +trait Router[Input, Route] extends (Input => Option[Route]) with ClosableOnce { + import Router.Log + + /** The [[Logger]] to use for this [[Router]]. Can be overridden to customize. */ + protected def logger: Logger = Log + + /** + * A label used for identifying this Router (i.e. for distinguishing between [[Router]] instances + * in error messages or for StatsReceiver scope). + */ + def label: String + + /** + * All of the [[Route]] routes contained within this [[Router]] + * + * @note This property is used to linearly determine the routes when [[closed()]] is called. + * It may also be used as a way to determine what routes are defined within a [[Router]] + * (i.e. for generating meaningful error messages). This property does not imply + * any relationship with [[apply]] or [[find]] or that a [[Router]]'s runtime behavior + * needs to be linear. + */ + def routes: Iterable[Route] + + /** + * Attempt to route an [[Input]] to a [[Route]] route known by this Router. + * + * @param input The [[Input]] to use for determining an available route + * + * @return `Some(Route)` if a route is found to match, `None` otherwise + * + * @note A [[ClosedRouterException]] will be thrown if [[close]] has been initiated on this + * [[router]] and subsequent routing attempts are received. + */ + final def apply(input: Input): Option[Route] = + if (isClosed) throw ClosedRouterException(this) + else find(input) + + /** + * Logic used to determine if this [[Router]] contains a [[Route]] + * for an [[Input]]. + * + * @param input The [[Input]] to determine a route for. + * + * @return `Some(Route)` if a matching route is defined, `None` if a route destination is + * not defined for the [[Input]]. + * + * @note This method is only meant to be called within the [[Router]]'s [[apply]], which + * handles lifecycle concerns. It should not be accessed directly. + */ + protected def find(input: Input): Option[Route] + + /** + * @inheritdoc + * @note NonFatal exceptions encountered when calling [[close()]] on a [[Route]] will be + * suppressed and Fatal exceptions will take on the same exception behavior of a + * [[Future.join]]. + */ + override protected def closeOnce(deadline: Time): Future[Unit] = + Future.join(routes.map(closeIfClosable(_, deadline)).toSeq) + + private[this] def closeIfClosable(route: Route, deadline: Time): Future[Unit] = route match { + case closable: Closable => + closable.close(deadline).rescue { + case NonFatal(e) => + logger.warning( + e, + s"Error encountered when attempting to close route '$route' in router '$label'") + Future.Done + } + case _ => + Future.Done + } +} diff --git a/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala b/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala new file mode 100644 index 0000000000..b779c9fc4d --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala @@ -0,0 +1,70 @@ +package com.twitter.util.routing + +import scala.collection.mutable.ArrayBuffer + +/** + * Contract for a [[Router]] builder. The resulting [[RouterType]] + * should be considered immutable, unless the [[RouterType]] implementation + * explicitly states otherwise. + * + * @param routeValidator A function that will be used to validate a [[Route]] when it is + * added to this builder via [[withRoute]]. If a [[Route]] is not + * valid, the [[routeValidator]] should throw a [[RouteValidationException]]. + * + * Ex: + * {{{ + * case class ExampleRoute(in: Int, out: Int) + * + * val routeValidator: ExampleRoute => Unit = { + * case ExampleRoute(in, out) if in < out => + * // route is valid + * () + * case ExampleRoute(in, out) => + * // route is not valid + * throw InvalidRouteException(s"'$in' is not less than '$out'") + * } + * }}} + * @tparam Input The [[Router]]'s `Input` type. + * @tparam Route The [[Router]]'s destination `Route` type. It is recommended that the `Route` + * is a self-contained/self-describing type for the purpose of validation via + * the [[routeValidator]]. Put differently, the `Route` should know of the + * `Input` that maps to itself. + * @tparam RouterType The type of [[Router]] to build. + */ +abstract class RouterBuilder[Input, Route, RouterType <: Router[Input, Route]]( + routeValidator: Route => Unit) { + + def this() = this((_: Route) => ()) + + private[this] val routes: ArrayBuffer[Route] = new ArrayBuffer[Route]() + + /** + * Add the [[route]] to the routes that will be present in + * the [[RouterType]] when [[newRouter()]] is called. + * + * @param route The [[Route]] to be added. + * + * @return + * + * @note The [[routeValidator]] will be exercised with the [[route]] + * and may throw an [[RouteValidationException]] + */ + def withRoute(route: Route): this.type = synchronized { + routeValidator(route) + routes += route + this + } + + /** + * Generate a new [[RouterType]] from the defined [[routes]]. + * @param routes The [[Route]]s to be used for building the resulting [[RouterType]]. + * @return The resulting [[RouterType]] made up of the underlying [[routes]]. + */ + protected def newRouter(routes: Iterable[Route]): RouterType + + /** + * @return A new [[RouterType]] made up of this builder's defined routes. + */ + def newRouter(): RouterType = newRouter(routes) + +} diff --git a/util-routing/src/test/java/com/twitter/util/routing/BUILD b/util-routing/src/test/java/com/twitter/util/routing/BUILD new file mode 100644 index 0000000000..774806955a --- /dev/null +++ b/util-routing/src/test/java/com/twitter/util/routing/BUILD @@ -0,0 +1,13 @@ +junit_tests( + sources = [ + "*.java", + ], + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-core:util-core-util", + "util/util-logging/src/main/scala", + "util/util-routing/src/main/scala/com/twitter/util/routing", + ], +) diff --git a/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java b/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java new file mode 100644 index 0000000000..b1a3a3c2eb --- /dev/null +++ b/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java @@ -0,0 +1,112 @@ +package com.twitter.util.routing; + +import org.junit.Assert; +import org.junit.Test; + +import scala.collection.Iterable; + +import java.util.Map; +import java.util.HashMap; +import java.util.Optional; + +import static com.twitter.util.Function.exfunc; +import static com.twitter.util.Function.func; + +public class RouterCompilationTest { + + private static final class StringRoute { + public final String input; + public final String output; + + StringRoute(String input, String output) { + this.input = input; + this.output = output; + } + } + + private static final class StringRouter extends AbstractRouter { + static StringRouterBuilder newBuilder() { return new StringRouterBuilder(); } + + private final Map routeMap; + + public StringRouter(Map routeMap) { + super("strings", routeMap.values()); + this.routeMap = routeMap; + } + + @Override + public Optional findAny(String input) { + StringRoute found = routeMap.get(input); + return Optional.ofNullable(found); + } + + } + + private static final class StringRouterBuilder + extends RouterBuilder { + + @Override + public StringRouter newRouter(Iterable routes) { + Map routeMap = new HashMap<>(); + routes.foreach(func(r -> routeMap.put(r.input, r))); + return new StringRouter(routeMap); + } + + } + + private static final class ValidatingStringRouterBuilder + extends RouterBuilder { + + ValidatingStringRouterBuilder() { + super(exfunc(r -> { + if (r.input.equals("invalid")) throw new RouteValidationException("INVALID!"); + return null; + })); + } + + @Override + public StringRouter newRouter(Iterable routes) { + Map routeMap = new HashMap<>(); + routes.foreach(func(r -> routeMap.put(r.input, r))); + return new StringRouter(routeMap); + } + + } + + @Test + public void testRouter() { + StringRoute hello = new StringRoute("hello", "abc"); + StringRoute goodbye = new StringRoute("goodbye", "123"); + + Map routes = new HashMap<>(); + routes.put(hello.input, hello); + routes.put(goodbye.input, goodbye); + + AbstractRouter router = new StringRouter(routes); + Assert.assertEquals(router.route("hello"), Optional.of(hello)); + Assert.assertEquals(router.route("goodbye"), Optional.of(goodbye)); + Assert.assertEquals(router.route("oh-no"), Optional.empty()); + } + + @Test + public void testRouterBuilder() { + StringRoute hello = new StringRoute("hello", "abc"); + StringRoute goodbye = new StringRoute("goodbye", "123"); + + AbstractRouter router = StringRouter.newBuilder() + .withRoute(hello) + .withRoute(goodbye) + .newRouter(); + + Assert.assertEquals(router.route("hello"), Optional.of(hello)); + Assert.assertEquals(router.route("goodbye"), Optional.of(goodbye)); + Assert.assertEquals(router.route("oh-no"), Optional.empty()); + } + + @Test(expected = RouteValidationException.class) + public void testValidatingRouterBuilder() { + new ValidatingStringRouterBuilder() + .withRoute(new StringRoute("invalid", "other")); + } + +} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD new file mode 100644 index 0000000000..6d7425ebe9 --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -0,0 +1,12 @@ +junit_tests( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-routing/src/main/scala/com/twitter/util/routing", + "util/util-routing/src/test/scala/com/twitter/util/routing/dynamic", + "util/util-routing/src/test/scala/com/twitter/util/routing/simple", + ], +) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala new file mode 100644 index 0000000000..29a2521be6 --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala @@ -0,0 +1,54 @@ +package com.twitter.util.routing + +import com.twitter.util.routing.simple.{SimpleRoute, SimpleRouter} +import org.scalatest.FunSuite + +class RouterBuilderTest extends FunSuite { + + private object TestRouter { + def newBuilder(): TestRouterBuilder = new TestRouterBuilder() + } + + private class TestRouterBuilder private[RouterBuilderTest] () + extends RouterBuilder[String, SimpleRoute, SimpleRouter] { + + protected def newRouter(routes: Iterable[SimpleRoute]): SimpleRouter = + new SimpleRouter(routes.map { r => r.in -> r }.toMap) + + } + + private class ValidatingTestRouterBuilder private[RouterBuilderTest] () + extends RouterBuilder[String, SimpleRoute, SimpleRouter](routeValidator = r => + if (r.in == "invalid") throw new IllegalArgumentException("INVALID!")) { + + protected def newRouter(routes: Iterable[SimpleRoute]): SimpleRouter = + new SimpleRouter(routes.map { r => r.in -> r }.toMap) + + } + + test("can build routes") { + val router = TestRouter + .newBuilder() + .withRoute(SimpleRoute("x", true)) + .withRoute(SimpleRoute("y", false)) + .withRoute(SimpleRoute("z", true)) + .newRouter() + + assert(router("x") == Some(SimpleRoute("x", true))) + assert(router("y") == Some(SimpleRoute("y", false))) + assert(router("z") == Some(SimpleRoute("z", true))) + assert(router("a") == None) + } + + test("throws when invalid route is passed") { + val router = new ValidatingTestRouterBuilder() + .withRoute(SimpleRoute("x", true)) + .withRoute(SimpleRoute("y", false)) + .withRoute(SimpleRoute("z", true)) + + intercept[IllegalArgumentException] { + router.withRoute(SimpleRoute("invalid", false)) + } + } + +} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala new file mode 100644 index 0000000000..6ad177903a --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala @@ -0,0 +1,163 @@ +package com.twitter.util.routing + +import com.twitter.conversions.DurationOps._ +import com.twitter.util.routing.dynamic.{DynamicRouter, DynamicRoute, Method, Request, Result} +import com.twitter.util.routing.simple.{SimpleRoute, SimpleRouter} +import com.twitter.util.{Await, Awaitable, Closable, Future, Time} +import java.util.concurrent.atomic.AtomicBoolean +import org.scalatest.FunSuite + +class RouterTest extends FunSuite { + + private def await[T](awaitable: Awaitable[T]): T = + Await.result(awaitable, 2.seconds) + + test("can define a router of generic types") { + val helloRoute = SimpleRoute("hello", true) + val goodbyeRoute = SimpleRoute("goodbye", false) + + val router = new SimpleRouter(Map("hello" -> helloRoute, "goodbye" -> goodbyeRoute)) + + assert(router("hello") == Some(helloRoute)) + assert(router("goodbye") == Some(goodbyeRoute)) + assert(router("something else") == None) + } + + test("test a router with dynamic routes") { + val abcReadHandler = + DynamicRoute(Method.Read, _.query.startsWith("abc/"), _ => Future.value(Result.Success)) + val xyzReadHandler = + DynamicRoute(Method.Read, _.query.startsWith("xyz/"), _ => Future.value(Result.Failure)) + val abcWriteHandler = + DynamicRoute(Method.Write, _.query.startsWith("abc/"), _ => Future.value(Result.Success)) + val defWriteHandler = + DynamicRoute(Method.Write, _.query == "def", _ => Future.value(Result.Failure)) + + val router = DynamicRouter + .newBuilder() + .withRoute(abcReadHandler) + .withRoute(xyzReadHandler) + .withRoute(abcWriteHandler) + .withRoute(defWriteHandler) + .newRouter() + + assert(router(Request(Method.Read, "abc/123/456")) == Some(abcReadHandler)) + assert(router(Request(Method.Read, "def/123/456")) == None) + assert(router(Request(Method.Read, "xyz/123/456")) == Some(xyzReadHandler)) + assert(router(Request(Method.Write, "abc/123/456")) == Some(abcWriteHandler)) + assert(router(Request(Method.Write, "def")) == Some(defWriteHandler)) + assert(router(Request(Method.Write, "def/123/456")) == None) + assert(router(Request(Method.Write, "xyz/123/456")) == None) + + await(router.close()) + assert(router.isClosed) + router.routes.foreach(r => assert(r.isClosed)) + } + + test("multiple close() calls on a router only executes close once") { + val router = new SimpleRouter(Map.empty) + + assert(router.closedTimes.get() == 0) + await(router.close()) + assert(router.closedTimes.get() == 1) + await(router.close()) + assert(router.closedTimes.get() == 1) + await(router.close()) + assert(router.closedTimes.get() == 1) + } + + test("routing to closed router throws") { + val router = new SimpleRouter(Map.empty) + await(router.close()) + intercept[ClosedRouterException] { + router("hello") + } + } + + test("router closes routes that are closable") { + class AtomicClosable extends Closable { + val isClosed = new AtomicBoolean(false); + override def close(deadline: Time): Future[Unit] = { + isClosed.compareAndSet(false, true) + Future.Done + } + } + + val closableA = new AtomicClosable + val closableB = new AtomicClosable + val closableC = new AtomicClosable + + class ClosableRouter extends Router[Boolean, Closable] { + override def label: String = "closable" + override def routes: Iterable[Closable] = Seq(closableA, closableB, closableC) + override protected def find(input: Boolean): Option[Closable] = None + } + + val router = new ClosableRouter + + assert(closableA.isClosed.get() == false) + assert(closableB.isClosed.get() == false) + assert(closableC.isClosed.get() == false) + + await(router.close()) + + assert(closableA.isClosed.get() == true) + assert(closableB.isClosed.get() == true) + assert(closableC.isClosed.get() == true) + + } + + test("a router is marked as closed before underlying routes complete closing") { + val closableA = Closable.make(_ => Future.never) + val closableB = Closable.make(_ => Future.Done) + + class ClosableRouter extends Router[Boolean, Closable] { + override def label: String = "closable" + override def routes: Iterable[Closable] = Seq(closableA, closableB) + override protected def find(input: Boolean): Option[Closable] = None + } + + val router = new ClosableRouter + + router(true) // route doesn't throw, router is still open + + val closeFuture = router.close() // close the router, any further requests will fail + + assert(closeFuture.isDefined == false) + intercept[ClosedRouterException] { + router(true) + } + assert(closeFuture.isDefined == false) // the future never completes + } + + test("a router suppresses non-fatal exceptions on close") { + val closableA = Closable.make(_ => throw new IllegalStateException("BOOM")) + + class ClosableRouter extends Router[Boolean, Closable] { + override def label: String = "closable" + override def routes: Iterable[Closable] = Seq(closableA) + override protected def find(input: Boolean): Option[Closable] = None + } + + val router = new ClosableRouter + await(router.close()) // close without exception + } + + test("a router bubbles up fatal exceptions on close") { + val closableA = Closable.make(_ => throw new OutOfMemoryError("BOOM")) + val closableB = Closable.make(_ => throw new OutOfMemoryError("BAM")) + + class ClosableRouter extends Router[Boolean, Closable] { + override def label: String = "closable" + override def routes: Iterable[Closable] = Seq(closableA, closableB) + override protected def find(input: Boolean): Option[Closable] = None + } + + val router = new ClosableRouter + val e = intercept[OutOfMemoryError] { + await(router.close()) + } + assert(e.getMessage == "BOOM") + } + +} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD new file mode 100644 index 0000000000..83fcd5c38b --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD @@ -0,0 +1,7 @@ +scala_library( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "util/util-routing/src/main/scala/com/twitter/util/routing", + ], +) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRoute.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRoute.scala new file mode 100644 index 0000000000..8ce2567805 --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRoute.scala @@ -0,0 +1,12 @@ +package com.twitter.util.routing.dynamic + +import com.twitter.util.{ClosableOnce, Future, Time} + +// example of a Route for a dynamic input +private[routing] case class DynamicRoute( + method: Method, + canHandle: Request => Boolean, + handle: Request => Future[Result]) + extends ClosableOnce { + override protected def closeOnce(deadline: Time): Future[Unit] = Future.Done +} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala new file mode 100644 index 0000000000..5e3c86e477 --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala @@ -0,0 +1,30 @@ +package com.twitter.util.routing.dynamic + +import com.twitter.util.routing.{Router, RouterBuilder} + +private[routing] object DynamicRouter { + def newBuilder(): DynamicTestRouterBuilder = new DynamicTestRouterBuilder() + + class DynamicTestRouterBuilder extends RouterBuilder[Request, DynamicRoute, DynamicRouter] { + override protected def newRouter(routes: Iterable[DynamicRoute]): DynamicRouter = + new DynamicRouter(routes.toSeq) + } +} + +// example to show how a router might be built using dynamic inputs +private[routing] class DynamicRouter(handlers: Seq[DynamicRoute]) + extends Router[Request, DynamicRoute] { + + // example showing how we can optimize dynamic routes to lookup by a property of the input + private[this] val handlersByMethod: Map[Method, Seq[DynamicRoute]] = handlers.groupBy(_.method) + + override def label: String = "dynamic-test-router" + + override def routes: Iterable[DynamicRoute] = handlers + + override protected def find(input: Request): Option[DynamicRoute] = + handlersByMethod.get(input.method) match { + case Some(handlers) => handlers.find(_.canHandle(input)) + case _ => None + } +} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Method.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Method.scala new file mode 100644 index 0000000000..1d2257d85d --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Method.scala @@ -0,0 +1,8 @@ +package com.twitter.util.routing.dynamic + +private[routing] sealed trait Method + +private[routing] object Method { + case object Read extends Method + case object Write extends Method +} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Request.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Request.scala new file mode 100644 index 0000000000..bdb3f2bf90 --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Request.scala @@ -0,0 +1,3 @@ +package com.twitter.util.routing.dynamic + +case class Request(method: Method, query: String) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Result.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Result.scala new file mode 100644 index 0000000000..67f5ef9ff5 --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Result.scala @@ -0,0 +1,8 @@ +package com.twitter.util.routing.dynamic + +private[routing] sealed trait Result + +private[routing] object Result { + case object Success extends Result + case object Failure extends Result +} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD new file mode 100644 index 0000000000..83fcd5c38b --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD @@ -0,0 +1,7 @@ +scala_library( + sources = globs("*.scala"), + compiler_option_sets = {"fatal_warnings"}, + dependencies = [ + "util/util-routing/src/main/scala/com/twitter/util/routing", + ], +) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRoute.scala b/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRoute.scala new file mode 100644 index 0000000000..ca7c10f2df --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRoute.scala @@ -0,0 +1,3 @@ +package com.twitter.util.routing.simple + +private[routing] case class SimpleRoute(in: String, out: Boolean) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRouter.scala b/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRouter.scala new file mode 100644 index 0000000000..232452eb93 --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRouter.scala @@ -0,0 +1,23 @@ +package com.twitter.util.routing.simple + +import com.twitter.util.routing.Router +import com.twitter.util.{Future, Time} +import java.util.concurrent.atomic.AtomicInteger + +private[routing] class SimpleRouter(routeMap: Map[String, SimpleRoute]) + extends Router[String, SimpleRoute] { + + override def label: String = "test-router" + + val closedTimes: AtomicInteger = new AtomicInteger(0) + + def find(req: String): Option[SimpleRoute] = routeMap.get(req) + + override protected def closeOnce(deadline: Time): Future[Unit] = { + closedTimes.incrementAndGet() + Future.Done + } + + override def routes: Iterable[SimpleRoute] = routeMap.values + +} From 1213f629008bcc0c3c9aa0889c7ee2c8ff04a87d Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 8 Apr 2020 17:42:15 +0000 Subject: [PATCH 308/796] util-core: Move PantsWorkaroundCore.scala to fix scoverage Problem In a previous commit the pants build was modifed to reduce noise and as part of that the PantsWorkaroundCore.scala file was moved to a spot that broke the sbt build. Specifically, the scoverage plugin isn't pleased to find a source file outside of the perscribed directory structure. Solution Move it to a source directory. JIRA Issues: CSL-9617 Differential Revision: https://phabricator.twitter.biz/D463341 --- util-core/BUILD | 2 +- util-core/{ => src/main/scala}/PantsWorkaroundCore.scala | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename util-core/{ => src/main/scala}/PantsWorkaroundCore.scala (100%) diff --git a/util-core/BUILD b/util-core/BUILD index 3b228be402..2519534794 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -45,7 +45,7 @@ scala_library( scala_library( # This target exists only to act as an aggregator jar. name = "scala", - sources = ["PantsWorkaroundCore.scala"], + sources = ["src/main/scala/PantsWorkaroundCore.scala"], provides = scala_artifact( org = "com.twitter", name = "util-core", diff --git a/util-core/PantsWorkaroundCore.scala b/util-core/src/main/scala/PantsWorkaroundCore.scala similarity index 100% rename from util-core/PantsWorkaroundCore.scala rename to util-core/src/main/scala/PantsWorkaroundCore.scala From 2ab745acad495fe56ac44bbb9b42dc19e88e6879 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 8 Apr 2020 22:10:22 +0000 Subject: [PATCH 309/796] util-app: Widen visibility of `c.t.app.Flag` constructors Problem Some secondary constructors of `c.t.app.Flag` are `private[app]` because we wanted to minimize the ability for users to set the `failFastUntilParsed` boolean but until the default is updated to the more correct value of 'true' (currently false for legacy reasons) it makes sense to widen the visibility of these constructors to allow for `failFastUntilParsed` to manually toggled to 'true' overridding the default of 'false'. Solution Change `private[app]` to `private[twitter]` on two secondary constructors for `c.t.app.Flag` . Differential Revision: https://phabricator.twitter.biz/D463701 --- util-app/src/main/scala/com/twitter/app/Flag.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/Flag.scala b/util-app/src/main/scala/com/twitter/app/Flag.scala index 1a551c720f..191934d4ae 100644 --- a/util-app/src/main/scala/com/twitter/app/Flag.scala +++ b/util-app/src/main/scala/com/twitter/app/Flag.scala @@ -60,10 +60,20 @@ class Flag[T: Flaggable] private[app] ( import com.twitter.app.Flag._ import java.util.logging._ - private[app] def this(name: String, help: String, default: => T, failFastUntilParsed: Boolean) = + private[twitter] def this( + name: String, + help: String, + default: => T, + failFastUntilParsed: Boolean + ) = this(name, help, Left(() => default), failFastUntilParsed) - private[app] def this(name: String, help: String, usage: String, failFastUntilParsed: Boolean) = + private[twitter] def this( + name: String, + help: String, + usage: String, + failFastUntilParsed: Boolean + ) = this(name, help, Right(usage), failFastUntilParsed) private[twitter] def this(name: String, help: String, default: => T) = From c97037a494460e52789db0809eb131be80746b8c Mon Sep 17 00:00:00 2001 From: Daniel Wagner-Hall Date: Wed, 15 Apr 2020 18:23:56 +0000 Subject: [PATCH 310/796] compiler_option_sets and tags are lists not sets This change is a no-op - nothing should be observably different. Some of our tools are more comfortable with lists than sets, so use them more broadly. This change was generated with: ``` rg -g BUILD -l compiler_option_sets | xargs perl -0 -pi -e 's#compiler_option_sets = {([^}]*)}#compiler_option_sets = [\1]#mg' ``` NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D467400 --- util-routing/src/main/scala/com/twitter/util/routing/BUILD | 2 +- util-routing/src/test/java/com/twitter/util/routing/BUILD | 2 +- util-routing/src/test/scala/com/twitter/util/routing/BUILD | 2 +- .../src/test/scala/com/twitter/util/routing/dynamic/BUILD | 2 +- .../src/test/scala/com/twitter/util/routing/simple/BUILD | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/BUILD b/util-routing/src/main/scala/com/twitter/util/routing/BUILD index 4b6436347d..9640fd5133 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/main/scala/com/twitter/util/routing/BUILD @@ -1,5 +1,5 @@ scala_library( - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", name = "util-routing", diff --git a/util-routing/src/test/java/com/twitter/util/routing/BUILD b/util-routing/src/test/java/com/twitter/util/routing/BUILD index 774806955a..9e7471628b 100644 --- a/util-routing/src/test/java/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/java/com/twitter/util/routing/BUILD @@ -2,7 +2,7 @@ junit_tests( sources = [ "*.java", ], - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index 6d7425ebe9..2560ca3357 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -1,6 +1,6 @@ junit_tests( sources = globs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD index 83fcd5c38b..f9d57206b2 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD @@ -1,6 +1,6 @@ scala_library( sources = globs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "util/util-routing/src/main/scala/com/twitter/util/routing", ], diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD index 83fcd5c38b..f9d57206b2 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD @@ -1,6 +1,6 @@ scala_library( sources = globs("*.scala"), - compiler_option_sets = {"fatal_warnings"}, + compiler_option_sets = ["fatal_warnings"], dependencies = [ "util/util-routing/src/main/scala/com/twitter/util/routing", ], From 71d39d80621fc3ac48d9154e2bd3bc5e049c639b Mon Sep 17 00:00:00 2001 From: Anton Tiurin Date: Fri, 17 Apr 2020 18:20:59 +0000 Subject: [PATCH 311/796] util: tunable-linter accepts a relative path Problem The linter accepts only absolute paths to a file It's not convenient if you want to check a file in the same source root Solution Convert relative path to absolute form Result Relative filepath works, no need for $(pwd) prefix Differential Revision: https://phabricator.twitter.biz/D468528 --- CHANGELOG.rst | 4 ++++ .../twitter/util/tunable/linter/ConfigurationLinter.scala | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9da979e31b..56ed950f2a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ +* util-tunable: ConfigurationLinter accepts a relative path. ``PHAB_ID=D468528`` + 20.4.0 ------ diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/ConfigurationLinter.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/ConfigurationLinter.scala index 13c2eb975c..4818004d85 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/ConfigurationLinter.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/ConfigurationLinter.scala @@ -3,7 +3,8 @@ package com.twitter.util.tunable.linter import com.twitter.app.App import com.twitter.util.{Throw, Return} import com.twitter.util.tunable.JsonTunableMapper -import java.net.URL + +import java.nio.file.Paths object ConfigurationLinter extends App { @@ -15,8 +16,7 @@ object ConfigurationLinter extends App { args.foreach { path => println(s"File: $path\n") - - val url = new URL("file://" + path) + val url = Paths.get(path).toUri().toURL() JsonTunableMapper().parse(url) match { case Return(map) => From 5e326a1109e2cd608515ce87badfb792bd346a3d Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 23 Apr 2020 23:56:11 +0000 Subject: [PATCH 312/796] Twitter-oss: Prepare OSS libraries for release 20.4.1 Problem We want to release the next versions of our Twitter OSS libraries 20.4.1 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D473181 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 56ed950f2a..00ae8e7089 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.4.1 +------ + New Features ~~~~~~~~~~~~ * util-tunable: ConfigurationLinter accepts a relative path. ``PHAB_ID=D468528`` diff --git a/README.md b/README.md index 08f8599dc6..1ea73194ff 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.4.0" +val utilCore = "com.twitter" %% "util-core" % "20.4.1" ``` # Units diff --git a/build.sbt b/build.sbt index 8e35186ea4..fce7f68fe0 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.5.0-SNAPSHOT" +val releaseVersion = "20.4.1" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 9eae50d11014d3f93f1c9c4f7c760b73aa1d638e Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 27 Apr 2020 16:44:33 +0000 Subject: [PATCH 313/796] Twitter-oss: update OSS libraries post-release to 20.5.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D474463 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ea73194ff..1425d370cf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index fce7f68fe0..8e35186ea4 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.4.1" +val releaseVersion = "20.5.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 82270a8152e8dac69606b1e35177d6b9dfdaf019 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 29 Apr 2020 05:14:39 +0000 Subject: [PATCH 314/796] 3rdparty: Bump jackson to 2.11.0 Problem / Solution / Result There's a new version of jackson, let's update to the new version. JIRA Issues: CSL-9562, CSL-9723 Differential Revision: https://phabricator.twitter.biz/D457496 --- CHANGELOG.rst | 5 +++++ build.sbt | 7 +++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 00ae8e7089..a3b673bf8a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,11 @@ New Features ~~~~~~~~~~~~ * util-tunable: ConfigurationLinter accepts a relative path. ``PHAB_ID=D468528`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Bump jackson version to 2.11.0. ``PHAB_ID=D457496`` + 20.4.0 ------ diff --git a/build.sbt b/build.sbt index 8e35186ea4..3965e9819f 100644 --- a/build.sbt +++ b/build.sbt @@ -12,8 +12,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll ( ExclusionRule("javax.jms", "jms") ) val slf4jVersion = "1.7.30" -val jacksonVersion = "2.9.10" -val jacksonDatabindVersion = "2.9.10.1" +val jacksonVersion = "2.11.0" val guavaLib = "com.google.guava" % "guava" % "25.1-jre" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" @@ -481,7 +480,7 @@ lazy val utilThrift = Project( "org.apache.thrift" % "libthrift" % "0.10.0", slf4jApi % "provided", "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion ) ).dependsOn(utilCodec) @@ -494,7 +493,7 @@ lazy val utilTunable = Project( name := "util-tunable", libraryDependencies ++= Seq( "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava") ) ).dependsOn(utilApp, utilCore) From 6ee7b49a7aa22462648dd9be27e2c577b3a9f003 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 1 May 2020 19:14:22 +0000 Subject: [PATCH 315/796] csl: Bump sbt version Problem/Solution Bump the sbt to 1.3.10 to help OSS experiment cross-building our libs with Dotty/Scala3. sbt-dotty requires 1.3.10. JIRA Issues: CSL-9718 Differential Revision: https://phabricator.twitter.biz/D476543 --- project/build.properties | 2 +- sbt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/project/build.properties b/project/build.properties index c0bab04941..797e7ccfdb 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.8 +sbt.version=1.3.10 diff --git a/sbt b/sbt index e7f538ee13..4cb7d05ede 100755 --- a/sbt +++ b/sbt @@ -1,8 +1,8 @@ #!/bin/bash -sbtver=1.2.8 +sbtver=1.3.10 sbtjar=sbt-launch.jar -sbtsha128=073c169c6e1a47b8ae78a7a718b907424dedab30 +sbtsha128=f49861befe36746728182ea7efb30096c724b0d7 sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From 3e939701d61b6678c3e6338853c9a48ceb8ce3c1 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 5 May 2020 19:29:57 +0000 Subject: [PATCH 316/796] util-core: Promote ActivitySource Out of Experimental Problem / Solution `com.twitter.io.exp.ActivitySource` et al. are depended on in production at Twitter, and have remained mostly untouched in the last five years. It no longer makes sense for them to remain as 'experimental', so let's promote them out of that namespace to `com.twitter.io.ActivitySource`, etc. Differential Revision: https://phabricator.twitter.biz/D478498 --- CHANGELOG.rst | 9 +- util-core/BUILD | 2 - .../scala/com/twitter/io/ActivitySource.scala | 63 ++++++ .../twitter/io/CachingActivitySource.scala | 47 ++++ .../io/ClassLoaderActivitySource.scala | 47 ++++ .../io/FilePollingActivitySource.scala | 51 +++++ .../com/twitter/io/exp/ActivitySource.scala | 204 ------------------ .../src/main/scala/com/twitter/io/exp/BUILD | 12 -- util-core/src/test/scala/BUILD | 1 - .../com/twitter/io/ActivitySourceTest.scala | 33 +++ .../io/CachingActivitySourceTest.scala | 18 ++ .../io/ClassLoaderActivitySourceTest.scala | 21 ++ .../io/FilePollingActivitySourceTest.scala | 62 ++++++ .../twitter/io/exp/ActivitySourceTest.scala | 101 --------- .../src/test/scala/com/twitter/io/exp/BUILD | 21 -- 15 files changed, 350 insertions(+), 342 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/io/ActivitySource.scala create mode 100644 util-core/src/main/scala/com/twitter/io/CachingActivitySource.scala create mode 100644 util-core/src/main/scala/com/twitter/io/ClassLoaderActivitySource.scala create mode 100644 util-core/src/main/scala/com/twitter/io/FilePollingActivitySource.scala delete mode 100644 util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala delete mode 100644 util-core/src/main/scala/com/twitter/io/exp/BUILD create mode 100644 util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala create mode 100644 util-core/src/test/scala/com/twitter/io/CachingActivitySourceTest.scala create mode 100644 util-core/src/test/scala/com/twitter/io/ClassLoaderActivitySourceTest.scala create mode 100644 util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala delete mode 100644 util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala delete mode 100644 util-core/src/test/scala/com/twitter/io/exp/BUILD diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a3b673bf8a..e6d5b0b17b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: Change the namespace of `ActivitySource` and its derivatives to + `com.twitter.io` as its no longer considered experimental since the code has + changed minimally in the past 5 years. ``PHAB_ID=D478498`` + 20.4.1 ------ @@ -160,7 +167,7 @@ Breaking API Changes * util-stats: abstract methods of StatsReceiver now take Schemas. The old APIs are now final and cannot be overriden. For custom implementations, define - schema based methods (eg, counter(verbosity, name) is now + schema based methods (eg, counter(verbosity, name) is now counter(CounterSchema)). NB: users can continue to call the old interface; only implementors must migrate.``PHAB_ID=D385068`` diff --git a/util-core/BUILD b/util-core/BUILD index 2519534794..460d03d2e0 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -56,13 +56,11 @@ scala_library( "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/main/scala/com/twitter/io", - "util/util-core/src/main/scala/com/twitter/io/exp", ], exports = [ ":util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/main/scala/com/twitter/io", - "util/util-core/src/main/scala/com/twitter/io/exp", ], ) diff --git a/util-core/src/main/scala/com/twitter/io/ActivitySource.scala b/util-core/src/main/scala/com/twitter/io/ActivitySource.scala new file mode 100644 index 0000000000..4e3aefba8d --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/ActivitySource.scala @@ -0,0 +1,63 @@ +package com.twitter.io + +import com.twitter.util._ + +/** + * An ActivitySource provides access to observerable named variables. + */ +trait ActivitySource[+T] { + + /** + * Returns an [[com.twitter.util.Activity]] for a named T-typed variable. + */ + def get(name: String): Activity[T] + + /** + * Produces an ActivitySource which queries this ActivitySource, falling back to + * a secondary ActivitySource only when the primary result is ActivitySource.Failed. + * + * @param that the secondary ActivitySource + */ + def orElse[U >: T](that: ActivitySource[U]): ActivitySource[U] = + new ActivitySource.OrElse(this, that) +} + +object ActivitySource { + + /** + * A Singleton exception to indicate that an ActivitySource failed to find + * a named variable. + */ + object NotFound extends IllegalStateException + + /** + * An ActivitySource for observing file contents. Once observed, + * each file will be polled once per period. + */ + def forFiles( + period: Duration = Duration.fromSeconds(60) + )( + implicit timer: Timer + ): ActivitySource[Buf] = + new CachingActivitySource(new FilePollingActivitySource(period)(timer)) + + /** + * Create an ActivitySource for ClassLoader resources. + */ + def forClassLoaderResources( + cl: ClassLoader = ClassLoader.getSystemClassLoader + ): ActivitySource[Buf] = + new CachingActivitySource(new ClassLoaderActivitySource(cl)) + + private[ActivitySource] class OrElse[T, U >: T]( + primary: ActivitySource[T], + failover: ActivitySource[U]) + extends ActivitySource[U] { + def get(name: String): Activity[U] = { + primary.get(name) transform { + case Activity.Failed(_) => failover.get(name) + case state => Activity(Var.value(state)) + } + } + } +} diff --git a/util-core/src/main/scala/com/twitter/io/CachingActivitySource.scala b/util-core/src/main/scala/com/twitter/io/CachingActivitySource.scala new file mode 100644 index 0000000000..a49a1abe6c --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/CachingActivitySource.scala @@ -0,0 +1,47 @@ +package com.twitter.io + +import com.twitter.util._ +import java.lang.ref.{ReferenceQueue, WeakReference} +import java.util.HashMap + +/** + * A convenient wrapper for caching the results returned by the + * underlying ActivitySource. + */ +class CachingActivitySource[T](underlying: ActivitySource[T]) extends ActivitySource[T] { + + private[this] val refq = new ReferenceQueue[Activity[T]] + private[this] val forward = new HashMap[String, WeakReference[Activity[T]]] + private[this] val reverse = new HashMap[WeakReference[Activity[T]], String] + + /** + * A caching proxy to the underlying ActivitySource. Vars are cached by + * name, and are tracked with WeakReferences. + */ + def get(name: String): Activity[T] = synchronized { + gc() + Option(forward.get(name)) flatMap { wr => Option(wr.get()) } match { + case Some(v) => v + case None => + val v = underlying.get(name) + val ref = new WeakReference(v, refq) + forward.put(name, ref) + reverse.put(ref, name) + v + } + } + + /** + * Remove garbage collected cache entries. + */ + def gc(): Unit = synchronized { + var ref = refq.poll() + while (ref != null) { + val key = reverse.remove(ref) + if (key != null) + forward.remove(key) + + ref = refq.poll() + } + } +} diff --git a/util-core/src/main/scala/com/twitter/io/ClassLoaderActivitySource.scala b/util-core/src/main/scala/com/twitter/io/ClassLoaderActivitySource.scala new file mode 100644 index 0000000000..d6cc130db1 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/ClassLoaderActivitySource.scala @@ -0,0 +1,47 @@ +package com.twitter.io + +import com.twitter.util._ +import java.util.concurrent.atomic.AtomicBoolean + +/** + * An ActivitySource for ClassLoader resources. + */ +class ClassLoaderActivitySource private[io] (classLoader: ClassLoader, pool: FuturePool) + extends ActivitySource[Buf] { + + private[io] def this(classLoader: ClassLoader) = this(classLoader, FuturePool.unboundedPool) + + def get(name: String): Activity[Buf] = { + // This Var is updated at most once since ClassLoader + // resources don't change (do they?). + val runOnce = new AtomicBoolean(false) + val p = new Promise[Activity.State[Buf]] + + // Defer loading until the first observation + val v = Var.async[Activity.State[Buf]](Activity.Pending) { value => + if (runOnce.compareAndSet(false, true)) { + pool { + classLoader.getResourceAsStream(name) match { + case null => p.setValue(Activity.Failed(ActivitySource.NotFound)) + case stream => + val reader = + new InputStreamReader(stream, InputStreamReader.DefaultMaxBufferSize, pool) + BufReader.readAll(reader) respond { + case Return(buf) => + p.setValue(Activity.Ok(buf)) + case Throw(cause) => + p.setValue(Activity.Failed(cause)) + } ensure { + // InputStreamReader ignores the deadline in close + reader.close(Time.Undefined) + } + } + } + } + p.onSuccess(value() = _) + Closable.nop + } + + Activity(v) + } +} diff --git a/util-core/src/main/scala/com/twitter/io/FilePollingActivitySource.scala b/util-core/src/main/scala/com/twitter/io/FilePollingActivitySource.scala new file mode 100644 index 0000000000..6ebd0979ba --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/FilePollingActivitySource.scala @@ -0,0 +1,51 @@ +package com.twitter.io + +import com.twitter.util._ +import java.io.{File, FileInputStream} + +/** + * An ActivitySource for observing the contents of a file with periodic polling. + */ +class FilePollingActivitySource private[io] ( + period: Duration, + pool: FuturePool +)( + implicit timer: Timer) + extends ActivitySource[Buf] { + + private[io] def this(period: Duration)(implicit timer: Timer) = + this(period, FuturePool.unboundedPool) + + def get(name: String): Activity[Buf] = { + val v = Var.async[Activity.State[Buf]](Activity.Pending) { value => + val timerTask = timer.schedule(Time.now, period) { + val file = new File(name) + + if (file.exists()) { + pool { + val reader = new InputStreamReader( + new FileInputStream(file), + InputStreamReader.DefaultMaxBufferSize, + pool + ) + BufReader.readAll(reader) respond { + case Return(buf) => + value() = Activity.Ok(buf) + case Throw(cause) => + value() = Activity.Failed(cause) + } ensure { + // InputStreamReader ignores the deadline in close + reader.close(Time.Undefined) + } + } + } else { + value() = Activity.Failed(ActivitySource.NotFound) + } + } + + Closable.make { _ => Future { timerTask.cancel() } } + } + + Activity(v) + } +} diff --git a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala b/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala deleted file mode 100644 index 82599622fe..0000000000 --- a/util-core/src/main/scala/com/twitter/io/exp/ActivitySource.scala +++ /dev/null @@ -1,204 +0,0 @@ -package com.twitter.io.exp - -import com.twitter.io.{Buf, BufReader, InputStreamReader} -import com.twitter.util._ -import java.io.{File, FileInputStream} -import java.lang.ref.{ReferenceQueue, WeakReference} -import java.util.concurrent.atomic.AtomicBoolean -import java.util.HashMap - -/** - * An ActivitySource provides access to observerable named variables. - */ -trait ActivitySource[+T] { - - /** - * Returns an [[com.twitter.util.Activity]] for a named T-typed variable. - */ - def get(name: String): Activity[T] - - /** - * Produces an ActivitySource which queries this ActivitySource, falling back to - * a secondary ActivitySource only when the primary result is ActivitySource.Failed. - * - * @param that the secondary ActivitySource - */ - def orElse[U >: T](that: ActivitySource[U]): ActivitySource[U] = - new ActivitySource.OrElse(this, that) -} - -object ActivitySource { - - /** - * A Singleton exception to indicate that an ActivitySource failed to find - * a named variable. - */ - object NotFound extends IllegalStateException - - /** - * An ActivitySource for observing file contents. Once observed, - * each file will be polled once per period. - */ - def forFiles( - period: Duration = Duration.fromSeconds(60) - )( - implicit timer: Timer - ): ActivitySource[Buf] = - new CachingActivitySource(new FilePollingActivitySource(period)(timer)) - - /** - * Create an ActivitySource for ClassLoader resources. - */ - def forClassLoaderResources( - cl: ClassLoader = ClassLoader.getSystemClassLoader - ): ActivitySource[Buf] = - new CachingActivitySource(new ClassLoaderActivitySource(cl)) - - private[ActivitySource] class OrElse[T, U >: T]( - primary: ActivitySource[T], - failover: ActivitySource[U]) - extends ActivitySource[U] { - def get(name: String): Activity[U] = { - primary.get(name) transform { - case Activity.Failed(_) => failover.get(name) - case state => Activity(Var.value(state)) - } - } - } -} - -/** - * A convenient wrapper for caching the results returned by the - * underlying ActivitySource. - */ -class CachingActivitySource[T](underlying: ActivitySource[T]) extends ActivitySource[T] { - - private[this] val refq = new ReferenceQueue[Activity[T]] - private[this] val forward = new HashMap[String, WeakReference[Activity[T]]] - private[this] val reverse = new HashMap[WeakReference[Activity[T]], String] - - /** - * A caching proxy to the underlying ActivitySource. Vars are cached by - * name, and are tracked with WeakReferences. - */ - def get(name: String): Activity[T] = synchronized { - gc() - Option(forward.get(name)) flatMap { wr => Option(wr.get()) } match { - case Some(v) => v - case None => - val v = underlying.get(name) - val ref = new WeakReference(v, refq) - forward.put(name, ref) - reverse.put(ref, name) - v - } - } - - /** - * Remove garbage collected cache entries. - */ - def gc(): Unit = synchronized { - var ref = refq.poll() - while (ref != null) { - val key = reverse.remove(ref) - if (key != null) - forward.remove(key) - - ref = refq.poll() - } - } -} - -/** - * An ActivitySource for observing the contents of a file with periodic polling. - */ -class FilePollingActivitySource private[exp] ( - period: Duration, - pool: FuturePool -)( - implicit timer: Timer) - extends ActivitySource[Buf] { - - private[exp] def this(period: Duration)(implicit timer: Timer) = - this(period, FuturePool.unboundedPool) - - import com.twitter.io.exp.ActivitySource._ - - def get(name: String): Activity[Buf] = { - val v = Var.async[Activity.State[Buf]](Activity.Pending) { value => - val timerTask = timer.schedule(Time.now, period) { - val file = new File(name) - - if (file.exists()) { - pool { - val reader = new InputStreamReader( - new FileInputStream(file), - InputStreamReader.DefaultMaxBufferSize, - pool - ) - BufReader.readAll(reader) respond { - case Return(buf) => - value() = Activity.Ok(buf) - case Throw(cause) => - value() = Activity.Failed(cause) - } ensure { - // InputStreamReader ignores the deadline in close - reader.close(Time.Undefined) - } - } - } else { - value() = Activity.Failed(NotFound) - } - } - - Closable.make { _ => Future { timerTask.cancel() } } - } - - Activity(v) - } -} - -/** - * An ActivitySource for ClassLoader resources. - */ -class ClassLoaderActivitySource private[exp] (classLoader: ClassLoader, pool: FuturePool) - extends ActivitySource[Buf] { - - import com.twitter.io.exp.ActivitySource._ - - private[exp] def this(classLoader: ClassLoader) = this(classLoader, FuturePool.unboundedPool) - - def get(name: String): Activity[Buf] = { - // This Var is updated at most once since ClassLoader - // resources don't change (do they?). - val runOnce = new AtomicBoolean(false) - val p = new Promise[Activity.State[Buf]] - - // Defer loading until the first observation - val v = Var.async[Activity.State[Buf]](Activity.Pending) { value => - if (runOnce.compareAndSet(false, true)) { - pool { - classLoader.getResourceAsStream(name) match { - case null => p.setValue(Activity.Failed(NotFound)) - case stream => - val reader = - new InputStreamReader(stream, InputStreamReader.DefaultMaxBufferSize, pool) - BufReader.readAll(reader) respond { - case Return(buf) => - p.setValue(Activity.Ok(buf)) - case Throw(cause) => - p.setValue(Activity.Failed(cause)) - } ensure { - // InputStreamReader ignores the deadline in close - reader.close(Time.Undefined) - } - } - } - } - p.onSuccess(value() = _) - Closable.nop - } - - Activity(v) - } -} diff --git a/util-core/src/main/scala/com/twitter/io/exp/BUILD b/util-core/src/main/scala/com/twitter/io/exp/BUILD deleted file mode 100644 index 0c58937e7c..0000000000 --- a/util-core/src/main/scala/com/twitter/io/exp/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -scala_library( - compiler_option_sets = ["fatal_warnings"], - provides = scala_artifact( - org = "com.twitter", - name = "util-core-io-exp", - repo = artifactory, - ), - dependencies = [ - "util/util-core/src/main/scala/com/twitter/io", - "util/util-core:util-core-util", - ], -) diff --git a/util-core/src/test/scala/BUILD b/util-core/src/test/scala/BUILD index ca90513fde..4ca1cf0944 100644 --- a/util-core/src/test/scala/BUILD +++ b/util-core/src/test/scala/BUILD @@ -3,7 +3,6 @@ target( "util/util-core/src/test/scala/com/twitter/concurrent", "util/util-core/src/test/scala/com/twitter/conversions", "util/util-core/src/test/scala/com/twitter/io", - "util/util-core/src/test/scala/com/twitter/io/exp", "util/util-core/src/test/scala/com/twitter/util", ], ) diff --git a/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala new file mode 100644 index 0000000000..c8f1e78d6f --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala @@ -0,0 +1,33 @@ +package com.twitter.io + +import com.twitter.util.Activity +import org.scalatest.FunSuite + +class ActivitySourceTest extends FunSuite { + + val ok: ActivitySource[String] = new ActivitySource[String] { + def get(varName: String) = Activity.value(varName) + } + + val failed: ActivitySource[Nothing] = new ActivitySource[Nothing] { + def get(varName: String) = Activity.exception(new Exception(varName)) + } + + test("ActivitySource.orElse") { + val a = failed.orElse(ok) + val b = ok.orElse(failed) + + assert("42" == a.get("42").sample()) + assert("42" == b.get("42").sample()) + } + +} + +object ActivitySourceTest { + + def bufToString(buf: Buf): String = buf match { + case Buf.Utf8(s) => s + case _ => "" + } + +} diff --git a/util-core/src/test/scala/com/twitter/io/CachingActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/CachingActivitySourceTest.scala new file mode 100644 index 0000000000..816f068af1 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/CachingActivitySourceTest.scala @@ -0,0 +1,18 @@ +package com.twitter.io + +import com.twitter.util.Activity +import org.scalatest.FunSuite +import scala.util.Random + +class CachingActivitySourceTest extends FunSuite { + + test("CachingActivitySource") { + val cache = new CachingActivitySource[String](new ActivitySource[String] { + def get(varName: String) = Activity.value(Random.alphanumeric.take(10).mkString) + }) + + val a = cache.get("a") + assert(a == cache.get("a")) + } + +} diff --git a/util-core/src/test/scala/com/twitter/io/ClassLoaderActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/ClassLoaderActivitySourceTest.scala new file mode 100644 index 0000000000..1853714f4c --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/ClassLoaderActivitySourceTest.scala @@ -0,0 +1,21 @@ +package com.twitter.io + +import com.twitter.util.FuturePool +import java.io.ByteArrayInputStream +import org.scalatest.FunSuite + +class ClassLoaderActivitySourceTest extends FunSuite { + + test("ClassLoaderActivitySource") { + val classLoader = new ClassLoader() { + override def getResourceAsStream(name: String) = + new ByteArrayInputStream(name.getBytes("UTF-8")) + } + + val loader = new ClassLoaderActivitySource(classLoader, FuturePool.immediatePool) + val bufAct = loader.get("bar baz") + + assert("bar baz" == ActivitySourceTest.bufToString(bufAct.sample())) + } + +} diff --git a/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala new file mode 100644 index 0000000000..ef18587c0f --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala @@ -0,0 +1,62 @@ +package com.twitter.io + +import com.twitter.util.{Activity, FuturePool, MockTimer, Time} +import java.io.{File, ByteArrayInputStream} +import org.scalatest.{BeforeAndAfter, FunSuite} +import scala.util.Random + +class FilePollingActivitySourceTest extends FunSuite with BeforeAndAfter { + + def writeToTempFile(s: String): Unit = { + val printer = new java.io.PrintWriter(new File(tempFile)) + try { + printer.print(s) + } finally { + printer.close() + } + } + + def deleteTempFile(): Unit = + new File(tempFile).delete() + + val tempFile: String = Random.alphanumeric.take(5).mkString + + before { + writeToTempFile("foo bar") + } + + after { + deleteTempFile() + } + + test("FilePollingActivitySource") { + Time.withCurrentTimeFrozen { timeControl => + import com.twitter.conversions.DurationOps._ + implicit val timer = new MockTimer + val file = new FilePollingActivitySource(1.microsecond, FuturePool.immediatePool) + val buf = file.get(tempFile) + + var content: Option[String] = None + + val listen = buf.run.changes.respond { + case Activity.Ok(b) => + content = Some(ActivitySourceTest.bufToString(b)) + case _ => + content = None + } + + timeControl.advance(2.microsecond) + timer.tick() + assert(content == Some("foo bar")) + + writeToTempFile("foo baz") + + timeControl.advance(2.microsecond) + timer.tick() + assert(content == Some("foo baz")) + + listen.close() + } + } + +} diff --git a/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala deleted file mode 100644 index ae6cd586a6..0000000000 --- a/util-core/src/test/scala/com/twitter/io/exp/ActivitySourceTest.scala +++ /dev/null @@ -1,101 +0,0 @@ -package com.twitter.io.exp - -import com.twitter.io.Buf -import com.twitter.util.{Activity, FuturePool, MockTimer, Time} -import java.io.{File, ByteArrayInputStream} -import org.scalatest.{BeforeAndAfter, FunSuite} -import scala.util.Random - -class ActivitySourceTest extends FunSuite with BeforeAndAfter { - - val ok: ActivitySource[String] = new ActivitySource[String] { - def get(varName: String) = Activity.value(varName) - } - - val failed: ActivitySource[Nothing] = new ActivitySource[Nothing] { - def get(varName: String) = Activity.exception(new Exception(varName)) - } - - val tempFile: String = Random.alphanumeric.take(5).mkString - - def writeToTempFile(s: String): Unit = { - val printer = new java.io.PrintWriter(new File(tempFile)) - try { - printer.print(s) - } finally { - printer.close() - } - } - - def bufToString(buf: Buf): String = buf match { - case Buf.Utf8(s) => s - case _ => "" - } - - before { - writeToTempFile("foo bar") - } - - after { - new File(tempFile).delete() - } - - test("ActivitySource.orElse") { - val a = failed.orElse(ok) - val b = ok.orElse(failed) - - assert("42" == a.get("42").sample()) - assert("42" == b.get("42").sample()) - } - - test("CachingActivitySource") { - val cache = new CachingActivitySource[String](new ActivitySource[String] { - def get(varName: String) = Activity.value(Random.alphanumeric.take(10).mkString) - }) - - val a = cache.get("a") - assert(a == cache.get("a")) - } - - test("FilePollingActivitySource") { - Time.withCurrentTimeFrozen { timeControl => - import com.twitter.conversions.DurationOps._ - implicit val timer = new MockTimer - val file = new FilePollingActivitySource(1.microsecond, FuturePool.immediatePool) - val buf = file.get(tempFile) - - var content: Option[String] = None - - val listen = buf.run.changes.respond { - case Activity.Ok(b) => - content = Some(bufToString(b)) - case _ => - content = None - } - - timeControl.advance(2.microsecond) - timer.tick() - assert(content == Some("foo bar")) - - writeToTempFile("foo baz") - - timeControl.advance(2.microsecond) - timer.tick() - assert(content == Some("foo baz")) - - listen.close() - } - } - - test("ClassLoaderActivitySource") { - val classLoader = new ClassLoader() { - override def getResourceAsStream(name: String) = - new ByteArrayInputStream(name.getBytes("UTF-8")) - } - - val loader = new ClassLoaderActivitySource(classLoader, FuturePool.immediatePool) - val bufAct = loader.get("bar baz") - - assert("bar baz" == bufToString(bufAct.sample())) - } -} diff --git a/util-core/src/test/scala/com/twitter/io/exp/BUILD b/util-core/src/test/scala/com/twitter/io/exp/BUILD deleted file mode 100644 index dae848176c..0000000000 --- a/util-core/src/test/scala/com/twitter/io/exp/BUILD +++ /dev/null @@ -1,21 +0,0 @@ -junit_tests( - sources = ["*.scala"], - # fatal_warnings runs into an issue with scalatest/Future.isDone: - # - # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` - # [warn] assert(future.close().isDone) - # [warn] ^ - # - # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` - # for this target, we could try enabling this again. - compiler_option_sets = [], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-core:util-core-util", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core/src/main/scala/com/twitter/io", - "util/util-core/src/main/scala/com/twitter/io/exp", - ], -) From 95668126f5e3752083df32e08262ddd23c99eb87 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 6 May 2020 19:43:04 +0000 Subject: [PATCH 317/796] util-core: Add Apply Method for Stream and Pool to InputStreamReader Problem / Solution `InputStreamReader` requires 3 parameters for construction, the input stream, a chunk size, and a future pool for executing all I/O. There are multiple scenarios where it makes sense to only provide the input stream and the future pool, and use the `DefaultMaxBufferSize` as the chunk size. `InputStreamReader` does not have an API which allows a user to do that without explicitly specifying the chunk size. Let's add one. Differential Revision: https://phabricator.twitter.biz/D480002 --- .../scala/com/twitter/io/ClassLoaderActivitySource.scala | 3 +-- .../scala/com/twitter/io/FilePollingActivitySource.scala | 6 +----- .../src/main/scala/com/twitter/io/InputStreamReader.scala | 8 ++++++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/ClassLoaderActivitySource.scala b/util-core/src/main/scala/com/twitter/io/ClassLoaderActivitySource.scala index d6cc130db1..b29bf4e412 100644 --- a/util-core/src/main/scala/com/twitter/io/ClassLoaderActivitySource.scala +++ b/util-core/src/main/scala/com/twitter/io/ClassLoaderActivitySource.scala @@ -24,8 +24,7 @@ class ClassLoaderActivitySource private[io] (classLoader: ClassLoader, pool: Fut classLoader.getResourceAsStream(name) match { case null => p.setValue(Activity.Failed(ActivitySource.NotFound)) case stream => - val reader = - new InputStreamReader(stream, InputStreamReader.DefaultMaxBufferSize, pool) + val reader = InputStreamReader(stream, pool) BufReader.readAll(reader) respond { case Return(buf) => p.setValue(Activity.Ok(buf)) diff --git a/util-core/src/main/scala/com/twitter/io/FilePollingActivitySource.scala b/util-core/src/main/scala/com/twitter/io/FilePollingActivitySource.scala index 6ebd0979ba..4ab0402f4a 100644 --- a/util-core/src/main/scala/com/twitter/io/FilePollingActivitySource.scala +++ b/util-core/src/main/scala/com/twitter/io/FilePollingActivitySource.scala @@ -23,11 +23,7 @@ class FilePollingActivitySource private[io] ( if (file.exists()) { pool { - val reader = new InputStreamReader( - new FileInputStream(file), - InputStreamReader.DefaultMaxBufferSize, - pool - ) + val reader = InputStreamReader(new FileInputStream(file), pool) BufReader.readAll(reader) respond { case Return(buf) => value() = Activity.Ok(buf) diff --git a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala index 8530f1e903..ffc9091a5c 100644 --- a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala +++ b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala @@ -97,4 +97,12 @@ object InputStreamReader { def apply(inputStream: InputStream, chunkSize: Int = DefaultMaxBufferSize): InputStreamReader = new InputStreamReader(inputStream, chunkSize) + /** + * Create an [[InputStreamReader]] from the given `InputStream` + * using the provided [[FuturePool]] for executing all I/O. The resulting + * [[InputStreamReader]] emits chunks of at most `DefaultMaxBufferSize`. + */ + def apply(inputStream: InputStream, pool: FuturePool): InputStreamReader = + new InputStreamReader(inputStream, chunkSize = DefaultMaxBufferSize, pool) + } From 24e8ecd62835b6008e9d29a13e226ff7a6e4aeb6 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Fri, 8 May 2020 22:28:17 +0000 Subject: [PATCH 318/796] util-core: Fix Test Warnings Problem / Solution Making changes to util-core can be hard to track due to the number of compiler warnings output when running tests. Let's fix all of them and turn on fatal warnings for Pants. Differential Revision: https://phabricator.twitter.biz/D481334 --- .../twitter/concurrent/AsyncMeterTest.scala | 73 ++++++++------ .../test/scala/com/twitter/concurrent/BUILD | 10 +- .../test/scala/com/twitter/conversions/BUILD | 10 +- .../com/twitter/io/ActivitySourceTest.scala | 1 - util-core/src/test/scala/com/twitter/io/BUILD | 10 +- .../test/scala/com/twitter/io/BufTest.scala | 5 +- .../scala/com/twitter/io/ByteReaderTest.scala | 26 +++-- .../io/FilePollingActivitySourceTest.scala | 2 +- .../test/scala/com/twitter/io/PipeTest.scala | 97 +++++++++++-------- .../scala/com/twitter/util/ActivityTest.scala | 1 - .../src/test/scala/com/twitter/util/BUILD | 12 +-- .../com/twitter/util/CancellableTest.scala | 10 +- .../scala/com/twitter/util/ClosableTest.scala | 35 +++++-- .../scala/com/twitter/util/EventTest.scala | 1 - .../scala/com/twitter/util/FutureTest.scala | 6 +- .../scala/com/twitter/util/LocalTest.scala | 2 +- .../scala/com/twitter/util/PoolTest.scala | 37 +++---- .../test/scala/com/twitter/util/VarTest.scala | 25 ++++- 18 files changed, 206 insertions(+), 157 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala index aeba180642..869ef46bef 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala @@ -8,11 +8,30 @@ import org.scalatest.FunSuite class AsyncMeterTest extends FunSuite { import AsyncMeter._ + // Workaround methods for dealing with Scala compiler warnings: + // + // assert(f.isDone) cannot be called directly because it results in a Scala compiler + // warning: 'possible missing interpolator: detected interpolated identifier `$conforms`' + // + // This is due to the implicit evidence required for `Future.isDone` which checks to see + // whether the Future that is attempting to have `isDone` called on it conforms to the type + // of `Future[Unit]`. This is done using `Predef.$conforms` + // https://www.scala-lang.org/api/2.12.2/scala/Predef$.html#$conforms[A]:A%3C:%3CA + // + // Passing that directly to `assert` causes problems because the `$conforms` is also seen as + // an interpolated string. We get around it by evaluating first and passing the result to + // `assert`. + private[this] def isDone(f: Future[Unit]): Boolean = + f.isDone + + private[this] def assertIsDone(f: Future[Unit]): Unit = + assert(isDone(f)) + test("AsyncMeter shouldn't wait at all when there aren't any waiters.") { val timer = new MockTimer val meter = newMeter(1, 1.second, 100)(timer) val result = meter.await(1) - assert(result.isDone) + assertIsDone(result) } test("AsyncMeter should allow more than one waiter and allow them on the schedule.") { @@ -21,12 +40,12 @@ class AsyncMeterTest extends FunSuite { val meter = newMeter(1, 1.second, 100)(timer) val ready = meter.await(1) val waiter = meter.await(1) - assert(ready.isDone) + assertIsDone(ready) assert(!waiter.isDefined) ctl.advance(1.second) timer.tick() - assert(waiter.isDone) + assertIsDone(waiter) } } @@ -35,7 +54,7 @@ class AsyncMeterTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => val meter = newMeter(1, 1.second, 100)(timer) val ready = meter.await(1) - assert(ready.isDone) + assertIsDone(ready) val waiter = meter.await(1) assert(!waiter.isDefined) @@ -45,7 +64,7 @@ class AsyncMeterTest extends FunSuite { ctl.advance(1.second) timer.tick() - assert(waiter.isDone) + assertIsDone(waiter) } } @@ -54,7 +73,7 @@ class AsyncMeterTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => val meter = newMeter(1, 1.second, 1)(timer) val ready = meter.await(1) - assert(ready.isDone) + assertIsDone(ready) val waiter = meter.await(1) val rejected = meter.await(1) @@ -77,7 +96,7 @@ class AsyncMeterTest extends FunSuite { var nr = 0 val ready = meter.await(1) val waiter = meter.await(1) - assert(ready.isDone) + assertIsDone(ready) assert(!waiter.isDefined) val e = new Exception("boom!") @@ -95,7 +114,7 @@ class AsyncMeterTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => val meter = newMeter(2, 1.second, 100)(timer) val ready = meter.await(2) - assert(ready.isDone) + assertIsDone(ready) val first = meter.await(1) val second = meter.await(1) @@ -104,8 +123,8 @@ class AsyncMeterTest extends FunSuite { ctl.advance(1.second) timer.tick() - assert(first.isDone) - assert(second.isDone) + assertIsDone(first) + assertIsDone(second) } } @@ -114,7 +133,7 @@ class AsyncMeterTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => val meter = newMeter(1, 500.microseconds, 100)(timer) val ready = meter.await(1) - assert(ready.isDone) + assertIsDone(ready) val first = meter.await(1) val second = meter.await(1) @@ -123,8 +142,8 @@ class AsyncMeterTest extends FunSuite { ctl.advance(1.millisecond) timer.tick() - assert(first.isDone) - assert(second.isDone) + assertIsDone(first) + assertIsDone(second) } } @@ -133,7 +152,7 @@ class AsyncMeterTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => val meter = newMeter(2, 500.microseconds, 100)(timer) val ready = meter.await(2) - assert(ready.isDone) + assertIsDone(ready) val first = meter.await(1) val second = meter.await(2) @@ -144,9 +163,9 @@ class AsyncMeterTest extends FunSuite { ctl.advance(1.millisecond) timer.tick() - assert(first.isDone) - assert(second.isDone) - assert(third.isDone) + assertIsDone(first) + assertIsDone(second) + assertIsDone(third) } } @@ -172,7 +191,7 @@ class AsyncMeterTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => val meter = newMeter(2, 1.second, 100)(timer) val ready = meter.await(2) - assert(ready.isDone) + assertIsDone(ready) val waiter = meter.await(2) assert(!waiter.isDefined) @@ -183,7 +202,7 @@ class AsyncMeterTest extends FunSuite { ctl.advance(500.milliseconds) timer.tick() - assert(waiter.isDone) + assertIsDone(waiter) } } @@ -203,7 +222,7 @@ class AsyncMeterTest extends FunSuite { val ready = meter.await(3) val first = meter.await(4) val second = meter.await(4) - assert(ready.isDone) + assertIsDone(ready) assert(!first.isDefined) assert(!second.isDefined) } @@ -216,16 +235,16 @@ class AsyncMeterTest extends FunSuite { val first = meter.await(1) val second = meter.await(1) val third = meter.await(1) - assert(ready.isDone) + assertIsDone(ready) ctl.advance(1.millisecond) timer.tick() - assert(first.isDone) + assertIsDone(first) assert(!second.isDefined) assert(!third.isDefined) ctl.advance(1.millisecond) timer.tick() - assert(second.isDone) - assert(third.isDone) + assertIsDone(second) + assertIsDone(third) } } @@ -237,7 +256,7 @@ class AsyncMeterTest extends FunSuite { assert(!greedy.isDefined) ctl.advance(1.second) timer.tick() - assert(greedy.isDone) + assertIsDone(greedy) } } @@ -254,7 +273,7 @@ class AsyncMeterTest extends FunSuite { } ctl.advance(1.second) timer.tick() - assert(first.isDone) + assertIsDone(first) } } @@ -277,7 +296,7 @@ class AsyncMeterTest extends FunSuite { ctl.advance(1.second) timer.tick() - assert(first.isDone) + assertIsDone(first) } } diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index 335dfbe783..8ca36f4040 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -1,14 +1,6 @@ junit_tests( sources = ["*.scala"], - # fatal_warnings runs into an issue with scalatest/Future.isDone: - # - # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` - # [warn] assert(future.close().isDone) - # [warn] ^ - # - # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` - # for this target, we could try enabling this again. - compiler_option_sets = [], + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD index 647433c77c..4166c9ab67 100644 --- a/util-core/src/test/scala/com/twitter/conversions/BUILD +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -1,14 +1,6 @@ junit_tests( sources = ["*.scala"], - # fatal_warnings runs into an issue with scalatest/Future.isDone: - # - # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` - # [warn] assert(future.close().isDone) - # [warn] ^ - # - # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` - # for this target, we could try enabling this again. - compiler_option_sets = [], + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", diff --git a/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala index c8f1e78d6f..171dffad16 100644 --- a/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala @@ -27,7 +27,6 @@ object ActivitySourceTest { def bufToString(buf: Buf): String = buf match { case Buf.Utf8(s) => s - case _ => "" } } diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index c79cbd83ad..897e3d06d1 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -1,14 +1,6 @@ junit_tests( sources = ["*.scala"], - # fatal_warnings runs into an issue with scalatest/Future.isDone: - # - # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` - # [warn] assert(future.close().isDone) - # [warn] ^ - # - # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` - # for this target, we could try enabling this again. - compiler_option_sets = [], + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index a0c7a73f29..b1b1f03a31 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -254,7 +254,7 @@ class BufTest test("Buf.process from and until returns -1 when fully processed") { def assertAllProcessed(expected: Array[Byte], buf: Buf, from: Int, until: Int): Unit = { - var seen = new mutable.ListBuffer[Byte]() + val seen = new mutable.ListBuffer[Byte]() val processor = new Buf.Processor { def apply(byte: Byte): Boolean = { seen += byte @@ -266,7 +266,6 @@ class BufTest } val arr = Array.range(0, 9).map(_.toByte) - val byteArrayOffset = 3 val byteArray = Buf.ByteArray.Owned(arr, 3, 8) assertAllProcessed(Array[Byte](5, 6, 7), byteArray, 2, 5) @@ -458,7 +457,7 @@ class BufTest val b1 = Buf.ByteArray.Owned(a1) val b2 = Buf.ByteArray.Owned(a2) val b3 = Buf.ByteArray.Owned(a3) - val b4 = Buf.ByteArray.Owned(a4) + val _ = Buf.ByteArray.Owned(a4) val comp2 = b1.concat(b2) val comp3 = comp2.concat(b3) diff --git a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala index 01f0084bef..683a8a5d55 100644 --- a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala @@ -21,9 +21,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test("readString underflow") { val br = readerWith() - intercept[UnderflowException] { - br.readString(1, StandardCharsets.UTF_8) - } + intercept[UnderflowException] { br.readString(1, StandardCharsets.UTF_8) } } test("readString")(forAll { (str1: String, str2: String) => @@ -38,7 +36,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test("readByte")(forAll { byte: Byte => val br = readerWith(byte) assert(br.readByte() == byte) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readShortBE")(forAll { s: Short => @@ -49,7 +47,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { // note, we need to cast here toShort so that the // MSB is interpreted as the sign bit. assert(br.readShortBE() == s) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readShortLE")(forAll { s: Short => @@ -61,7 +59,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { // note, we need to cast here toShort so that the // MSB is interpreted as the sign bit. assert(br.readShortLE() == s) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readUnsignedMediumBE")(forAll { m: Int => @@ -71,7 +69,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { ((m) & 0xff).toByte ) assert(br.readUnsignedMediumBE() == maskMedium(m)) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readUnsignedMediumLE")(forAll { m: Int => @@ -81,7 +79,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { ((m >> 16) & 0xff).toByte ) assert(br.readUnsignedMediumLE() == maskMedium(m)) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readIntBE")(forAll { i: Int => @@ -92,7 +90,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { ((i) & 0xff).toByte ) assert(br.readIntBE() == i) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readIntLE")(forAll { i: Int => @@ -103,7 +101,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { ((i >> 24) & 0xff).toByte ) assert(br.readIntLE() == i) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readLongBE")(forAll { l: Long => @@ -118,7 +116,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { ((l) & 0xff).toByte ) assert(br.readLongBE() == l) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readLongLE")(forAll { l: Long => @@ -133,7 +131,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { ((l >> 56) & 0xff).toByte ) assert(br.readLongLE() == l) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readUnsignedByte")(forAll { b: Byte => @@ -191,7 +189,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { ((bi) & 0xff).toByte ) assert(br.readUnsignedLongBE() == bi) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) test("readUnsignedLongLE")(forAll(uInt64s) { bi1: BigInt => @@ -207,7 +205,7 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { ((bi >> 56) & 0xff).toByte ) assert(br.readUnsignedLongLE() == bi) - val exc = intercept[UnderflowException] { br.readByte() } + intercept[UnderflowException] { br.readByte() } }) // .equals is required to handle NaN diff --git a/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala index ef18587c0f..85a0e6f7bd 100644 --- a/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala +++ b/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala @@ -1,7 +1,7 @@ package com.twitter.io import com.twitter.util.{Activity, FuturePool, MockTimer, Time} -import java.io.{File, ByteArrayInputStream} +import java.io.File import org.scalatest.{BeforeAndAfter, FunSuite} import scala.util.Random diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index dda7b2f6b6..8d1cb8afdc 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -13,8 +13,29 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck private def arr(i: Int, j: Int) = Array.range(i, j).map(_.toByte) private def buf(i: Int, j: Int) = Buf.ByteArray.Owned(arr(i, j)) + // Workaround methods for dealing with Scala compiler warnings: + // + // assert(f.isDone) cannot be called directly because it results in a Scala compiler + // warning: 'possible missing interpolator: detected interpolated identifier `$conforms`' + // + // This is due to the implicit evidence required for `Future.isDone` which checks to see + // whether the Future that is attempting to have `isDone` called on it conforms to the type + // of `Future[Unit]`. This is done using `Predef.$conforms` + // https://www.scala-lang.org/api/2.12.2/scala/Predef$.html#$conforms[A]:A%3C:%3CA + // + // Passing that directly to `assert` causes problems because the `$conforms` is also seen as + // an interpolated string. We get around it by evaluating first and passing the result to + // `assert`. + private def isDone(f: Future[Unit]): Boolean = + f.isDone + + private def assertIsDone(f: Future[Unit]): Unit = + assert(isDone(f)) + + private def assertIsNotDone(f: Future[Unit]): Unit = + assert(!isDone(f)) + private def assertRead(r: Reader[Buf], i: Int, j: Int): Unit = { - val n = j - i val f = r.read() assertRead(f, i, j) } @@ -48,7 +69,7 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck private def assertReadEofAndClosed(rw: Pipe[Buf]): Unit = { assertReadNone(rw) - assert(rw.close().isDone) + assertIsDone(rw.close()) } private def assertReadNone(r: Reader[Buf]): Unit = @@ -143,30 +164,30 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck assert(closed) } - test("write then reads then close") { testWriteReadClose } - def testWriteReadClose = { + test("write then reads then close") { testWriteReadClose() } + def testWriteReadClose() = { val rw = new Pipe[Buf] val wf = rw.write(buf(0, 6)) - assert(!wf.isDone) + assertIsNotDone(wf) assertRead(rw, 0, 6) - assert(wf.isDone) - assert(!rw.close().isDone) + assertIsDone(wf) + assertIsNotDone(rw.close()) assertReadEofAndClosed(rw) } - test("read then write then close") { readWriteClose } - def readWriteClose = { + test("read then write then close") { readWriteClose() } + def readWriteClose() = { val rw = new Pipe[Buf] val rf = rw.read() assert(!rf.isDefined) val wf = rw.write(buf(0, 6)) - assert(wf.isDone) + assertIsDone(wf) assertRead(rf, 0, 6) - assert(!rw.close().isDone) + assertIsNotDone(rw.close()) assertReadEofAndClosed(rw) } @@ -176,7 +197,7 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck assertFailedEx(rw.write(buf(0, 6))) val cf = rw.close() - assert(!cf.isDone) + assertIsNotDone(cf) assertFailedEx(rw.read()) assertFailedEx(cf) @@ -185,9 +206,9 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck test("write after close") { val rw = new Pipe[Buf] val cf = rw.close() - assert(!cf.isDone) + assertIsNotDone(cf) assertReadEofAndClosed(rw) - assert(cf.isDone) + assertIsDone(cf) intercept[IllegalStateException] { await(rw.write(buf(0, 1))) @@ -199,7 +220,7 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck var closed = false rw.onClose.ensure { closed = true } val wf = rw.write(buf(0, 1)) - assert(!wf.isDone) + assertIsNotDone(wf) intercept[IllegalStateException] { await(rw.write(buf(0, 1))) @@ -216,23 +237,23 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck assertFailedEx(rw.read()) } - def readAfterCloseNoPendingReads = { + def readAfterCloseNoPendingReads() = { val rw = new Pipe[Buf] - assert(!rw.close().isDone) + assertIsNotDone(rw.close()) assertReadEofAndClosed(rw) } test("read after close with no pending reads") { - readAfterCloseNoPendingReads + readAfterCloseNoPendingReads() } def readAfterClosePendingData = { val rw = new Pipe[Buf] val wf = rw.write(buf(0, 1)) - assert(!wf.isDone) + assertIsNotDone(wf) // close before the write is satisfied wipes the pending write - assert(!rw.close().isDone) + assertIsNotDone(rw.close()) intercept[IllegalStateException] { await(wf) } @@ -280,52 +301,52 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck test("close not satisfied until writes are read") { val rw = new Pipe[Buf] val cf = rw.write(buf(0, 6)).before(rw.close()) - assert(!cf.isDone) + assertIsNotDone(cf) assertRead(rw, 0, 6) - assert(!cf.isDone) + assertIsNotDone(cf) assertReadEofAndClosed(rw) } - def closeNotSatisfiedUntillAllReadsDone = { + def closeNotSatisfiedUntillAllReadsDone() = { val rw = new Pipe[Buf] val rf = rw.read() val cf = rf.flatMap { _ => rw.close() } assert(!rf.isDefined) - assert(!cf.isDone) + assertIsNotDone(cf) - assert(rw.write(buf(0, 3)).isDone) + assertIsDone(rw.write(buf(0, 3))) assertRead(rf, 0, 3) - assert(!cf.isDone) + assertIsNotDone(cf) assertReadEofAndClosed(rw) } - test("close not satisfied until reads are fulfilled")(closeNotSatisfiedUntillAllReadsDone) + test("close not satisfied until reads are fulfilled")(closeNotSatisfiedUntillAllReadsDone()) def closeWhileReadPending = { val rw = new Pipe[Buf] val rf = rw.read() assert(!rf.isDefined) - assert(rw.close().isDone) + assertIsDone(rw.close()) assert(rf.isDefined) } test("close while read pending")(closeWhileReadPending) - def closeTwice = { + def closeTwice() = { val rw = new Pipe[Buf] - assert(!rw.close().isDone) + assertIsNotDone(rw.close()) assertReadEofAndClosed(rw) - assert(rw.close().isDone) + assertIsDone(rw.close()) assertReadEofAndClosed(rw) } - test("close then close")(closeTwice) + test("close then close")(closeTwice()) test("close after fail") { val rw = new Pipe[Buf] rw.fail(failedEx) val cf = rw.close() - assert(!cf.isDone) + assertIsNotDone(cf) assertFailedEx(rw.read()) assertFailedEx(cf) @@ -336,7 +357,7 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck Time.withCurrentTimeFrozen { ctrl => val rw = new Pipe[Buf](timer) val cf = rw.close(1.second) - assert(!cf.isDone) + assertIsNotDone(cf) ctrl.advance(1.second) timer.tick() @@ -352,10 +373,10 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck Time.withCurrentTimeFrozen { _ => val rw = new Pipe[Buf](timer) val cf = rw.close(1.second) - assert(!cf.isDone) + assertIsNotDone(cf) rw.fail(failedEx) - assert(!cf.isDone) + assertIsNotDone(cf) assertFailedEx(rw.read()) assertFailedEx(cf) @@ -365,9 +386,9 @@ class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyCheck test("close while write pending") { val rw = new Pipe[Buf] val wf = rw.write(buf(0, 1)) - assert(!wf.isDone) + assertIsNotDone(wf) val cf = rw.close() - assert(!cf.isDone) + assertIsNotDone(cf) intercept[IllegalStateException] { await(wf) } diff --git a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala index 80f0987307..7c53e9f9b8 100644 --- a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala @@ -2,7 +2,6 @@ package com.twitter.util import java.util.concurrent.atomic.AtomicReference import org.scalatest.FunSuite -import scala.collection.compat._ class ActivityTest extends FunSuite { test("Activity#flatMap") { diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index f94ee19552..07a33c9023 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -1,21 +1,13 @@ junit_tests( sources = ["*.scala"], - # fatal_warnings runs into an issue with scalatest/Future.isDone: - # - # [warn] ...: possible missing interpolator: detected interpolated identifier `$conforms` - # [warn] assert(future.close().isDone) - # [warn] ^ - # - # At some point later, when we can enable `-Xlint:-warn-missing-interpolator` - # for this target, we could try enabling this again. - compiler_option_sets = [], + compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/test/java/com/twitter/util:object_size_calculator", ], ) diff --git a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala index 8e139d6fa3..299fb9d7a0 100644 --- a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala @@ -20,7 +20,7 @@ class CancellableTest extends FunSuite { test("CancellableSink cancel normal case") { var count = 2 - def square: Unit = count *= count + def square(): Unit = count *= count val s = new CancellableSink(square) assert(count == 2) s.cancel() @@ -29,7 +29,7 @@ class CancellableTest extends FunSuite { test("CancellableSink cancel once") { var count = 0 - def increment: Unit = count += 1 + def increment(): Unit = count += 1 val s = new CancellableSink(increment) s.cancel() assert(count == 1) @@ -39,7 +39,7 @@ class CancellableTest extends FunSuite { test("CancellableSink has not cancelled") { var count = 0 - def increment: Unit = count += 1 + def increment(): Unit = count += 1 val s = new CancellableSink(increment) assert(!s.isCancelled) assert(count == 0) @@ -47,7 +47,7 @@ class CancellableTest extends FunSuite { test("CancellableSink confirm cancelled") { var count = 0 - def increment: Unit = count += 1 + def increment(): Unit = count += 1 val s = new CancellableSink(increment) s.cancel() assert(s.isCancelled) @@ -56,7 +56,7 @@ class CancellableTest extends FunSuite { test("CancellableSink not support linking") { var count = 1 - def multiply: Unit = count *= 2 + def multiply(): Unit = count *= 2 val s1 = new CancellableSink(multiply) val s2 = new CancellableSink(multiply) assertThrows[Exception] { diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index 925eb34959..5c5372958e 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -6,6 +6,29 @@ import org.scalatest.concurrent.{Eventually, IntegrationPatience} import scala.language.reflectiveCalls class ClosableTest extends FunSuite with Eventually with IntegrationPatience { + + // Workaround methods for dealing with Scala compiler warnings: + // + // assert(f.isDone) cannot be called directly because it results in a Scala compiler + // warning: 'possible missing interpolator: detected interpolated identifier `$conforms`' + // + // This is due to the implicit evidence required for `Future.isDone` which checks to see + // whether the Future that is attempting to have `isDone` called on it conforms to the type + // of `Future[Unit]`. This is done using `Predef.$conforms` + // https://www.scala-lang.org/api/2.12.2/scala/Predef$.html#$conforms[A]:A%3C:%3CA + // + // Passing that directly to `assert` causes problems because the `$conforms` is also seen as + // an interpolated string. We get around it by evaluating first and passing the result to + // `assert`. + private[this] def isDone(f: Future[Unit]): Boolean = + f.isDone + + private[this] def assertIsDone(f: Future[Unit]): Unit = + assert(isDone(f)) + + private[this] def assertIsNotDone(f: Future[Unit]): Unit = + assert(!isDone(f)) + test("Closable.close(Duration)") { Time.withCurrentTimeFrozen { _ => var time: Option[Time] = None @@ -46,11 +69,11 @@ class ClosableTest extends FunSuite with Eventually with IntegrationPatience { assert(n1 == 1) assert(n2 == 1) - assert(!f.isDone) + assertIsNotDone(f) p1.setDone() - assert(!f.isDone) + assertIsNotDone(f) p2.setDone() - assert(f.isDone) + assertIsDone(f) } test("Closable.all with exceptions") { @@ -94,17 +117,17 @@ class ClosableTest extends FunSuite with Eventually with IntegrationPatience { val f = c.close() assert(n1 == 1) assert(n2 == 0) - assert(!f.isDone) + assertIsNotDone(f) p1.setDone() assert(n1 == 1) assert(n2 == 1) - assert(!f.isDone) + assertIsNotDone(f) p2.setDone() assert(n1 == 1) assert(n2 == 1) - assert(f.isDone) + assertIsDone(f) } test("Closable.sequence is resilient to failed closes") { diff --git a/util-core/src/test/scala/com/twitter/util/EventTest.scala b/util-core/src/test/scala/com/twitter/util/EventTest.scala index 48d642cbc0..07830a83eb 100644 --- a/util-core/src/test/scala/com/twitter/util/EventTest.scala +++ b/util-core/src/test/scala/com/twitter/util/EventTest.scala @@ -4,7 +4,6 @@ import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import java.util.concurrent.{CountDownLatch, Executors} import org.scalatest.FunSuite import scala.collection.mutable -import scala.collection.compat._ class EventTest extends FunSuite { diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index 61b07a2cf6..034197a171 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -829,7 +829,11 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert val actual = intercept[Exception] { await(f) } assert(actual == exn, explain(left, right, leftFirst)) case (Nvr, Ret(_)) | (Ret(_), Nvr) | (Nvr, Nvr) => assert(!f.isDefined) - case (Ret(a), Ret(b)) => assert(await(f) == (a, b), explain(left, right, leftFirst)) + case (Ret(a), Ret(b)) => + val expected: (Int, Int) = (a, b) + val result = await(f) + val isEqual = result == expected + assert(isEqual, explain(left, right, leftFirst)) } } } diff --git a/util-core/src/test/scala/com/twitter/util/LocalTest.scala b/util-core/src/test/scala/com/twitter/util/LocalTest.scala index 0c2bf8ca3e..803bd0e3b3 100644 --- a/util-core/src/test/scala/com/twitter/util/LocalTest.scala +++ b/util-core/src/test/scala/com/twitter/util/LocalTest.scala @@ -167,7 +167,7 @@ class LocalTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test("Context should have the same behavior with Map") { val context = Local.Context.empty - val map = Map.empty[Local.Key, Option[Int]] + val map = Map.empty[Local.Key, Int] forAll(arbitrary[Int], arbitrary[Local.Key]) { (value, key) => if (value % 2 == 0) { diff --git a/util-core/src/test/scala/com/twitter/util/PoolTest.scala b/util-core/src/test/scala/com/twitter/util/PoolTest.scala index 076d42f47b..da58d91fc3 100644 --- a/util-core/src/test/scala/com/twitter/util/PoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PoolTest.scala @@ -1,23 +1,24 @@ package com.twitter.util -import scala.collection.mutable -import scala.collection.Seq - -import org.scalatest.WordSpec import com.twitter.conversions.DurationOps._ +import org.scalatest.WordSpec +import scala.collection.mutable class PoolTest extends WordSpec { + private[this] def await[T](f: Awaitable[T]): T = + Await.result(f, 2.seconds) + "SimplePool" should { "with a simple queue of items" should { "it reseves items in FIFO order" in { val queue = new mutable.Queue[Int] ++ List(1, 2, 3) val pool = new SimplePool(queue) - assert(Await.result(pool.reserve()) == 1) - assert(Await.result(pool.reserve()) == 2) + assert(await(pool.reserve()) == 1) + assert(await(pool.reserve()) == 2) pool.release(2) - assert(Await.result(pool.reserve()) == 3) - assert(Await.result(pool.reserve()) == 2) + assert(await(pool.reserve()) == 3) + assert(await(pool.reserve()) == 2) pool.release(1) pool.release(2) pool.release(3) @@ -37,18 +38,18 @@ class PoolTest extends WordSpec { val h = new PoolSpecHelper import h._ - assert(Await.result(pool.reserve()) == 2) - assert(Await.result(pool.reserve()) == 4) - assert(Await.result(pool.reserve()) == 6) - assert(Await.result(pool.reserve()) == 8) + assert(await(pool.reserve()) == 2) + assert(await(pool.reserve()) == 4) + assert(await(pool.reserve()) == 6) + assert(await(pool.reserve()) == 8) val promise = pool.reserve() intercept[TimeoutException] { Await.result(promise, 1.millisecond) } pool.release(8) pool.release(6) - assert(Await.result(promise) == 8) - assert(Await.result(pool.reserve()) == 6) + assert(await(promise) == 8) + assert(await(pool.reserve()) == 6) intercept[TimeoutException] { Await.result(pool.reserve(), 1.millisecond) } @@ -58,10 +59,10 @@ class PoolTest extends WordSpec { val h = new PoolSpecHelper import h._ - assert(Await.result(pool.reserve()) == 2) - assert(Await.result(pool.reserve()) == 4) - assert(Await.result(pool.reserve()) == 6) - assert(Await.result(pool.reserve()) == 8) + assert(await(pool.reserve()) == 2) + assert(await(pool.reserve()) == 4) + assert(await(pool.reserve()) == 6) + assert(await(pool.reserve()) == 8) intercept[TimeoutException] { Await.result(pool.reserve(), 1.millisecond) } diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 38acfbb32f..ea66b05817 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -5,9 +5,28 @@ import org.scalacheck.Arbitrary.arbitrary import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable -import scala.collection.compat._ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { + + // Workaround methods for dealing with Scala compiler warnings: + // + // assert(f.isDone) cannot be called directly because it results in a Scala compiler + // warning: 'possible missing interpolator: detected interpolated identifier `$conforms`' + // + // This is due to the implicit evidence required for `Future.isDone` which checks to see + // whether the Future that is attempting to have `isDone` called on it conforms to the type + // of `Future[Unit]`. This is done using `Predef.$conforms` + // https://www.scala-lang.org/api/2.12.2/scala/Predef$.html#$conforms[A]:A%3C:%3CA + // + // Passing that directly to `assert` causes problems because the `$conforms` is also seen as + // an interpolated string. We get around it by evaluating first and passing the result to + // `assert`. + private[this] def isDone(f: Future[Unit]): Boolean = + f.isDone + + private[this] def assertIsDone(f: Future[Unit]): Unit = + assert(isDone(f)) + private case class U[T](init: T) extends UpdatableVar[T](init) { import Var.Observer @@ -188,13 +207,13 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { val f = o.close(t) assert(called == 1) assert(closed == Time.Zero) - assert(f.isDone) + assertIsDone(f) // Closing the Var.async process is asynchronous with closing // the Var itself. val f1 = o1.close(t) assert(closed == t) - assert(f1.isDone) + assertIsDone(f1) } test("Var.collect: empty") { From 6e7095b1b7223376932f4b45524a60776a5462d9 Mon Sep 17 00:00:00 2001 From: Shane Delmore Date: Sun, 10 May 2020 01:37:41 +0000 Subject: [PATCH 319/796] reformat source with scalafmt 2.5.1 part 4 If you have merge conflicts see go/scalafmt NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D481624 --- .../scala/com/twitter/app/Flaggable.scala | 10 +-- .../scala/com/twitter/app/FlagsTest.scala | 7 ++- .../com/twitter/app/LoadServiceTest.scala | 3 +- .../com/twitter/cache/guava/GuavaCache.scala | 8 ++- .../com/twitter/cache/EvictingCache.scala | 8 ++- .../cache/caffeine/CaffeineCache.scala | 8 ++- .../scala/com/twitter/concurrent/Tx.scala | 10 +-- .../src/main/scala/com/twitter/io/Buf.scala | 8 ++- .../scala/com/twitter/io/BufInputStream.scala | 2 +- .../scala/com/twitter/io/ByteReader.scala | 4 +- .../main/scala/com/twitter/util/Diff.scala | 8 ++- .../test/scala/com/twitter/io/BufTest.scala | 3 +- .../scala/com/twitter/io/ByteReaderTest.scala | 4 +- .../scala/com/twitter/io/ReaderTest.scala | 3 +- .../scala/com/twitter/util/FutureTest.scala | 16 +++-- .../scala/com/twitter/util/NetUtilTest.scala | 26 ++++---- .../test/scala/com/twitter/util/VarTest.scala | 20 ++++-- .../ConsistentHashingDistributor.scala | 10 +-- .../scala/com/twitter/hashing/Hashable.scala | 12 ++-- .../twitter/logging/SyslogHandlerTest.scala | 20 ++++-- .../twitter/finagle/stats/CachedRegex.scala | 14 +++-- .../util/testing/ArgumentCapture.scala | 62 +++++++++++-------- .../util/tunable/JsonTunableMapperTest.scala | 3 +- .../main/scala/com/twitter/zk/AuthInfo.scala | 6 +- .../scala/com/twitter/zk/ZkClientTest.scala | 39 +++++++++--- 25 files changed, 197 insertions(+), 117 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index 2912db176e..03fe738118 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -132,10 +132,12 @@ object Flaggable { } override def show(addr: InetSocketAddress): String = - "%s:%d".format(Option(addr.getAddress) match { - case Some(a) if a.isAnyLocalAddress => "" - case _ => addr.getHostName - }, addr.getPort) + "%s:%d".format( + Option(addr.getAddress) match { + case Some(a) if a.isAnyLocalAddress => "" + case _ => addr.getHostName + }, + addr.getPort) } implicit val ofFile: Flaggable[File] = new Flaggable[File] { diff --git a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala index 1788c94cc8..efdc97a460 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala @@ -254,7 +254,12 @@ class FlagsTest extends FunSuite { new Flag[Int]("something.id", "bar", Left(() => 3), failFastUntilParsed = false) assert(somethingIdFlag() == 3) // this logs a message in SEVERE and we get the default value - val testFlags = new Flags("failFastTest", includeGlobal = false, failFastUntilParsed = true) // added flags will inherit this value + val testFlags = + new Flags( + "failFastTest", + includeGlobal = false, + failFastUntilParsed = true + ) // added flags will inherit this value testFlags.add(somethingIdFlag) intercept[IllegalStateException] { testFlags diff --git a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala index d67cfdd7ea..b0c34d393f 100644 --- a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala +++ b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala @@ -72,7 +72,8 @@ class LoadServiceTest extends FunSuite with MockitoSugar { } } - test("LoadService should only load 1 instance of T, even when there's multiple occurrences of T") { + test( + "LoadService should only load 1 instance of T, even when there's multiple occurrences of T") { val randomIfaces = LoadService[LoadServiceRandomInterface]() assert(randomIfaces.size == 1) } diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala b/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala index dacafb1537..ab0eb7f602 100644 --- a/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala @@ -18,9 +18,11 @@ import java.util.concurrent.Callable */ class GuavaCache[K, V](cache: GCache[K, Future[V]]) extends ConcurrentMapCache[K, V](cache.asMap) { override def getOrElseUpdate(k: K)(v: => Future[V]): Future[V] = - cache.get(k, new Callable[Future[V]] { - def call(): Future[V] = v - }) + cache.get( + k, + new Callable[Future[V]] { + def call(): Future[V] = v + }) } /** diff --git a/util-cache/src/main/scala/com/twitter/cache/EvictingCache.scala b/util-cache/src/main/scala/com/twitter/cache/EvictingCache.scala index 3b0d28498e..220b6c0907 100644 --- a/util-cache/src/main/scala/com/twitter/cache/EvictingCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/EvictingCache.scala @@ -15,9 +15,11 @@ private[cache] class EvictingCache[K, V](underlying: FutureCache[K, V]) } override def getOrElseUpdate(k: K)(v: => Future[V]): Future[V] = - evictOnFailure(k, underlying.getOrElseUpdate(k) { - v - }) + evictOnFailure( + k, + underlying.getOrElseUpdate(k) { + v + }) } private[cache] class LazilyEvictingCache[K, V](underlying: FutureCache[K, V]) diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala b/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala index 92e1eb1164..53a51c946e 100644 --- a/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala @@ -19,9 +19,11 @@ import java.util.function.Function class CaffeineCache[K, V](cache: Cache[K, Future[V]]) extends ConcurrentMapCache[K, V](cache.asMap) { override def getOrElseUpdate(k: K)(v: => Future[V]): Future[V] = - cache.get(k, new Function[K, Future[V]] { - def apply(k: K): Future[V] = v - }) + cache.get( + k, + new Function[K, Future[V]] { + def apply(k: K): Future[V] = v + }) } /** diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala index 82ed9cd20d..970b6d0f91 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala @@ -91,10 +91,12 @@ object Tx { state match { case Idle => val p = new Promise[Result[U]] - state = Ackd(this, { - case true => p.setValue(Commit(msg)) - case false => p.setValue(Abort) - }) + state = Ackd( + this, + { + case true => p.setValue(Commit(msg)) + case false => p.setValue(Abort) + }) p case Ackd(who, confirm) if who ne this => diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 1d1774a684..61ef6400e3 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -140,7 +140,9 @@ abstract class Buf { outer => ) case Buf.Composite(right: IndexedThree, rightLength) => Buf.Composite( - (new VectorBuilder[Buf] += left.a += left.b += left.c += right.a += right.b += right.c) + (new VectorBuilder[ + Buf + ] += left.a += left.b += left.c += right.a += right.b += right.c) .result(), leftLength + rightLength ) @@ -993,9 +995,9 @@ object Buf { def hash(buf: Buf): Int = finishHash(hashBuf(buf)) // Adapted from util-hashing. - private[this] val UintMax: Long = 0xFFFFFFFFL + private[this] val UintMax: Long = 0xffffffffL private[this] val Fnv1a32Prime: Int = 16777619 - private[this] val Fnv1a32Init: Long = 0x811C9DC5L + private[this] val Fnv1a32Init: Long = 0x811c9dc5L private[this] def finishHash(hash: Long): Int = (hash & UintMax).toInt private[this] def hashBuf(buf: Buf, init: Long = Fnv1a32Init): Long = buf match { case b if b.isEmpty => init diff --git a/util-core/src/main/scala/com/twitter/io/BufInputStream.scala b/util-core/src/main/scala/com/twitter/io/BufInputStream.scala index 86a6b8e241..0a8941ae6f 100644 --- a/util-core/src/main/scala/com/twitter/io/BufInputStream.scala +++ b/util-core/src/main/scala/com/twitter/io/BufInputStream.scala @@ -36,7 +36,7 @@ class BufInputStream(val buf: Buf) extends InputStream { else { val b = buf.get(index) index += 1 - b & 0xFF + b & 0xff } } diff --git a/util-core/src/main/scala/com/twitter/io/ByteReader.scala b/util-core/src/main/scala/com/twitter/io/ByteReader.scala index fd39b231f5..3d71044d35 100644 --- a/util-core/src/main/scala/com/twitter/io/ByteReader.scala +++ b/util-core/src/main/scala/com/twitter/io/ByteReader.scala @@ -425,9 +425,9 @@ private class ByteReaderImpl(buf: Buf) extends ByteReader { ret } - def readUnsignedIntBE(): Long = readIntBE() & 0xFFFFFFFFL + def readUnsignedIntBE(): Long = readIntBE() & 0xffffffffL - def readUnsignedIntLE(): Long = readIntLE() & 0xFFFFFFFFL + def readUnsignedIntLE(): Long = readIntLE() & 0xffffffffL // - Long - def readLongBE(): Long = { diff --git a/util-core/src/main/scala/com/twitter/util/Diff.scala b/util-core/src/main/scala/com/twitter/util/Diff.scala index 6d1b972709..1a58fdc29e 100644 --- a/util-core/src/main/scala/com/twitter/util/Diff.scala +++ b/util-core/src/main/scala/com/twitter/util/Diff.scala @@ -98,9 +98,11 @@ object Diffable { def diff[T](left: Seq[T], right: Seq[T]): SeqDiff[T] = if (left.length < right.length) { val SeqDiff(_, insert) = diff(left, right.take(left.length)) - SeqDiff(right.length, insert ++ ((left.length until right.length) map { i => - (i -> right(i)) - })) + SeqDiff( + right.length, + insert ++ ((left.length until right.length) map { i => + (i -> right(i)) + })) } else if (left.length > right.length) { diff(left.take(right.length), right) } else { diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index b1b1f03a31..dfa05de28f 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -696,7 +696,8 @@ class BufTest } AllCharsets foreach { charset => - test("Buf.StringCoder: %s charset can encode and decode an English phrase".format(charset.name)) { + test( + "Buf.StringCoder: %s charset can encode and decode an English phrase".format(charset.name)) { val coder = new Buf.StringCoder(charset) {} val phrase = "Hello, world!" val encoded = coder(phrase) diff --git a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala index 683a8a5d55..6785a60473 100644 --- a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala @@ -165,12 +165,12 @@ class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { test("readUnsignedIntBE")(forAll { i: Int => val br = new ByteReaderImpl(Buf.Empty) { override def readIntBE() = i } - assert(br.readUnsignedIntBE() == (i & 0xFFFFFFFFL)) + assert(br.readUnsignedIntBE() == (i & 0xffffffffL)) }) test("readUnsignedIntLE")(forAll { i: Int => val br = new ByteReaderImpl(Buf.Empty) { override def readIntLE() = i } - assert(br.readUnsignedIntLE() == (i & 0xFFFFFFFFL)) + assert(br.readUnsignedIntLE() == (i & 0xffffffffL)) }) val uInt64s: Gen[BigInt] = Gen diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index a34f8b6045..85e3944bcb 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -242,7 +242,8 @@ class ReaderTest assert(exception.getMessage == exceptionMsg) } - test("Reader.flatten - re-register `curReaderClosep` to listen to the next reader in the stream") { + test( + "Reader.flatten - re-register `curReaderClosep` to listen to the next reader in the stream") { val exceptionMsg = "boom" val r1 = Reader.value(1) diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index 034197a171..cf67ca7886 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -1718,12 +1718,16 @@ class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropert } } - test("ConstFuture", new MkConst { - def apply[A](r: Try[A]): Future[A] = Future.const(r) - }) - test("Promise", new MkConst { - def apply[A](r: Try[A]): Future[A] = new Promise(r) - }) + test( + "ConstFuture", + new MkConst { + def apply[A](r: Try[A]): Future[A] = Future.const(r) + }) + test( + "Promise", + new MkConst { + def apply[A](r: Try[A]): Future[A] = new Promise(r) + }) "Future.apply" should { "fail on NLRC" in { diff --git a/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala b/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala index e9d9cb5062..3d5542e0b1 100644 --- a/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala +++ b/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala @@ -51,9 +51,9 @@ class NetUtilTest extends WordSpec { "ipToInt" in { assert(NetUtil.ipToInt("0.0.0.0") == 0) - assert(NetUtil.ipToInt("255.255.255.255") == 0xFFFFFFFF) - assert(NetUtil.ipToInt("255.255.255.0") == 0xFFFFFF00) - assert(NetUtil.ipToInt("255.0.255.0") == 0xFF00FF00) + assert(NetUtil.ipToInt("255.255.255.255") == 0xffffffff) + assert(NetUtil.ipToInt("255.255.255.0") == 0xffffff00) + assert(NetUtil.ipToInt("255.0.255.0") == 0xff00ff00) assert(NetUtil.ipToInt("61.197.253.56") == 0x3dc5fd38) intercept[IllegalArgumentException] { NetUtil.ipToInt("256.0.255.0") @@ -62,9 +62,9 @@ class NetUtilTest extends WordSpec { "inetAddressToInt" in { assert(NetUtil.inetAddressToInt(InetAddress.getByName("0.0.0.0")) == 0) - assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.255.255.255")) == 0xFFFFFFFF) - assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.255.255.0")) == 0xFFFFFF00) - assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.0.255.0")) == 0xFF00FF00) + assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.255.255.255")) == 0xffffffff) + assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.255.255.0")) == 0xffffff00) + assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.0.255.0")) == 0xff00ff00) assert(NetUtil.inetAddressToInt(InetAddress.getByName("61.197.253.56")) == 0x3dc5fd38) intercept[IllegalArgumentException] { NetUtil.inetAddressToInt(InetAddress.getByName("::1")) @@ -72,13 +72,13 @@ class NetUtilTest extends WordSpec { } "cidrToIpBlock" in { - assert(NetUtil.cidrToIpBlock("127") == ((0x7F000000, 0xFF000000))) - assert(NetUtil.cidrToIpBlock("127.0.0") == ((0x7F000000, 0xFFFFFF00))) - assert(NetUtil.cidrToIpBlock("127.0.0.1") == ((0x7F000001, 0xFFFFFFFF))) - assert(NetUtil.cidrToIpBlock("127.0.0.1/1") == ((0x7F000001, 0x80000000))) - assert(NetUtil.cidrToIpBlock("127.0.0.1/4") == ((0x7F000001, 0xF0000000))) - assert(NetUtil.cidrToIpBlock("127.0.0.1/32") == ((0x7F000001, 0xFFFFFFFF))) - assert(NetUtil.cidrToIpBlock("127/24") == ((0x7F000000, 0xFFFFFF00))) + assert(NetUtil.cidrToIpBlock("127") == ((0x7f000000, 0xff000000))) + assert(NetUtil.cidrToIpBlock("127.0.0") == ((0x7f000000, 0xffffff00))) + assert(NetUtil.cidrToIpBlock("127.0.0.1") == ((0x7f000001, 0xffffffff))) + assert(NetUtil.cidrToIpBlock("127.0.0.1/1") == ((0x7f000001, 0x80000000))) + assert(NetUtil.cidrToIpBlock("127.0.0.1/4") == ((0x7f000001, 0xf0000000))) + assert(NetUtil.cidrToIpBlock("127.0.0.1/32") == ((0x7f000001, 0xffffffff))) + assert(NetUtil.cidrToIpBlock("127/24") == ((0x7f000000, 0xffffff00))) } "isInetAddressInBlock" in { diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index ea66b05817..99f08bbfe0 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -36,10 +36,12 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { override def observe(d: Int, obs: Observer[T]) = { accessCount += 1 observerCount += 1 - Closable.all(super.observe(d, obs), Closable.make { deadline => - observerCount -= 1 - Future.Done - }) + Closable.all( + super.observe(d, obs), + Closable.make { deadline => + observerCount -= 1 + Future.Done + }) } } @@ -81,8 +83,14 @@ class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { var result = 0 val o1 = v1.changes.register(Witness({ (i: Int) => result = result + i })) // result = 2 - val o2 = v1.changes.register(Witness({ (i: Int) => result = result * i * i })) // result = 2 * 2 * 2 = 8 - val o3 = v1.changes.register(Witness({ (i: Int) => result = result + result + i })) // result = 8 + 8 + 2 = 18 + val o2 = + v1.changes.register(Witness({ (i: Int) => + result = result * i * i + })) // result = 2 * 2 * 2 = 8 + val o3 = + v1.changes.register(Witness({ (i: Int) => + result = result + result + i + })) // result = 8 + 8 + 2 = 18 assert(result == 18) // ensure those three things happened in sequence diff --git a/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala b/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala index f237bf8114..a7ab923bdf 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala @@ -84,10 +84,10 @@ class ConsistentHashingDistributor[A]( val buffer = md5.digest() - underlying.put(byteArrayToLE(buffer, 0).toLong & 0xFFFFFFFFL, node) - underlying.put(byteArrayToLE(buffer, 4).toLong & 0xFFFFFFFFL, node) - underlying.put(byteArrayToLE(buffer, 8).toLong & 0xFFFFFFFFL, node) - underlying.put(byteArrayToLE(buffer, 12).toLong & 0xFFFFFFFFL, node) + underlying.put(byteArrayToLE(buffer, 0).toLong & 0xffffffffL, node) + underlying.put(byteArrayToLE(buffer, 4).toLong & 0xffffffffL, node) + underlying.put(byteArrayToLE(buffer, 8).toLong & 0xffffffffL, node) + underlying.put(byteArrayToLE(buffer, 12).toLong & 0xffffffffL, node) i += 1 } @@ -106,7 +106,7 @@ class ConsistentHashingDistributor[A]( // hashes are 32-bit because they are 32-bit on the libmemcached and // we need to maintain compatibility with libmemcached - private[this] def truncateHash(hash: Long): Long = hash & 0xFFFFFFFFL + private[this] def truncateHash(hash: Long): Long = hash & 0xffffffffL private def mapEntryForHash(hash: Long): java.util.Map.Entry[Long, HashNode[A]] = { val truncatedHash = truncateHash(hash) diff --git a/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala b/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala index 0b9ca12d2d..233e713fb8 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala @@ -39,7 +39,7 @@ object Hashable extends LowPriorityHashable { // Some standard hashing: def hashCode[T]: Hashable[T, Int] = new Hashable[T, Int] { def apply(t: T) = t.hashCode } - private[this] val MaxUnsignedInt: Long = 0xFFFFFFFFL + private[this] val MaxUnsignedInt: Long = 0xffffffffL /** * FNV fast hashing algorithm in 32 bits. @@ -50,7 +50,7 @@ object Hashable extends LowPriorityHashable { val PRIME: Int = 16777619 var i = 0 val len = key.length - var rv: Long = 0x811C9DC5L + var rv: Long = 0x811c9dc5L while (i < len) { rv = (rv * PRIME) ^ (key(i) & 0xff) i += 1 @@ -70,7 +70,7 @@ object Hashable extends LowPriorityHashable { val PRIME: Int = 16777619 var i = 0 val len = key.length - var rv: Long = 0x811C9DC5L + var rv: Long = 0x811c9dc5L while (i < len) { rv = (rv ^ (key(i) & 0xff)) * PRIME i += 1 @@ -90,7 +90,7 @@ object Hashable extends LowPriorityHashable { val PRIME: Long = 1099511628211L var i = 0 val len = key.length - var rv: Long = 0xCBF29CE484222325L + var rv: Long = 0xcbf29ce484222325L while (i < len) { rv = (rv * PRIME) ^ (key(i) & 0xff) i += 1 @@ -110,7 +110,7 @@ object Hashable extends LowPriorityHashable { val PRIME: Long = 1099511628211L var i = 0 val len = key.length - var rv: Long = 0xCBF29CE484222325L + var rv: Long = 0xcbf29ce484222325L while (i < len) { rv = (rv ^ (key(i) & 0xff)) * PRIME i += 1 @@ -159,7 +159,7 @@ object Hashable extends LowPriorityHashable { var j = 0 while (j < 8) { if ((rv & 1) != 0) { - rv = (rv >> 1) ^ 0xEDB88320L + rv = (rv >> 1) ^ 0xedb88320L } else { rv >>= 1 } diff --git a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala index 1e16ce18fd..7be3894240 100644 --- a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala @@ -49,11 +49,17 @@ class SyslogHandlerTest extends WordSpec { val p = new DatagramPacket(new Array[Byte](1024), 1024) serverSocket.receive(p) assert( - new String(p.getData, 0, p.getLength) == "<9>2008-03-29T05:53:16 raccoon.local whiskey: fatal message!" + new String( + p.getData, + 0, + p.getLength) == "<9>2008-03-29T05:53:16 raccoon.local whiskey: fatal message!" ) serverSocket.receive(p) assert( - new String(p.getData, 0, p.getLength) == "<11>2008-03-29T05:53:16 raccoon.local whiskey: error message!" + new String( + p.getData, + 0, + p.getLength) == "<11>2008-03-29T05:53:16 raccoon.local whiskey: error message!" ) } @@ -76,7 +82,10 @@ class SyslogHandlerTest extends WordSpec { val p = new DatagramPacket(new Array[Byte](1024), 1024) serverSocket.receive(p) assert( - new String(p.getData, 0, p.getLength) == "<9>2008-03-29T05:53:16 raccoon.local [pingd] whiskey: fatal message!" + new String( + p.getData, + 0, + p.getLength) == "<9>2008-03-29T05:53:16 raccoon.local [pingd] whiskey: fatal message!" ) } @@ -99,7 +108,10 @@ class SyslogHandlerTest extends WordSpec { val p = new DatagramPacket(new Array[Byte](1024), 1024) serverSocket.receive(p) assert( - new String(p.getData, 0, p.getLength) == "<9>Mar 29 05:53:16 raccoon.local whiskey: fatal message!" + new String( + p.getData, + 0, + p.getLength) == "<9>Mar 29 05:53:16 raccoon.local whiskey: fatal message!" ) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala index d34b475380..142a3ee997 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala @@ -41,13 +41,15 @@ private[stats] class CachedRegex(regex: Regex) def apply(samples: collection.Map[String, Number]): collection.Map[String, Number] = { // don't really want this to be executed in parallel - regexMatchCache.forEachKey(Long.MaxValue, new Consumer[String] { - def accept(key: String): Unit = { - if (!samples.contains(key)) { - regexMatchCache.remove(key) + regexMatchCache.forEachKey( + Long.MaxValue, + new Consumer[String] { + def accept(key: String): Unit = { + if (!samples.contains(key)) { + regexMatchCache.remove(key) + } } - } - }) + }) samples.filterKeys(filterFn).toMap } } diff --git a/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala b/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala index 4567e358f7..9961cf33ba 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala +++ b/util-test/src/main/scala/com/twitter/util/testing/ArgumentCapture.scala @@ -490,7 +490,9 @@ trait ArgumentCapture { .zip(arg7) .zip(arg8) .zip(arg9) - .map({ case (((((((((a, b), c), d), e), f), g), h), i), j) => (a, b, c, d, e, f, g, h, i, j) }) + .map({ + case (((((((((a, b), c), d), e), f), g), h), i), j) => (a, b, c, d, e, f, g, h, i, j) + }) .toSeq } @@ -1548,8 +1550,8 @@ trait ArgumentCapture { .zip(arg17) .map({ case ( - ((((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), p), q), - r + ((((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), p), q), + r ) => (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r) }) @@ -1726,8 +1728,10 @@ trait ArgumentCapture { .zip(arg18) .map({ case ( - (((((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), p), q), r), - s + ( + ((((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), p), q), + r), + s ) => (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s) }) @@ -1912,14 +1916,14 @@ trait ArgumentCapture { .zip(arg19) .map({ case ( - ( ( - ((((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), p), q), - r + ( + ((((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), p), q), + r + ), + s ), - s - ), - t + t ) => (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t) }) @@ -2112,17 +2116,19 @@ trait ArgumentCapture { .zip(arg20) .map({ case ( - ( ( ( - ((((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), p), q), - r + ( + ( + (((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), p), + q), + r + ), + s ), - s + t ), - t - ), - u + u ) => (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u) }) @@ -2323,23 +2329,25 @@ trait ArgumentCapture { .zip(arg21) .map({ case ( - ( ( ( ( ( - (((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), p), - q + ( + ( + ((((((((((((((a, b), c), d), e), f), g), h), i), j), k), l), m), n), o), + p), + q + ), + r ), - r + s ), - s + t ), - t + u ), - u - ), - v + v ) => (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v) }) diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala index 37c3b0c41e..fc3cbff309 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala @@ -246,7 +246,8 @@ class JsonTunableMapperTest extends FunSuite { assert(map(TunableMap.Key[Duration]("timeoutId4"))() == Some(Duration.Undefined)) } - test("tunableMapForResources throws an Illegal argument exception when there are multiple paths") { + test( + "tunableMapForResources throws an Illegal argument exception when there are multiple paths") { val rsc = getClass.getClassLoader .getResources("com/twitter/tunables/IdForValidJson/instances.json") .asScala diff --git a/util-zk/src/main/scala/com/twitter/zk/AuthInfo.scala b/util-zk/src/main/scala/com/twitter/zk/AuthInfo.scala index 3714f512ce..ab6488f87e 100644 --- a/util-zk/src/main/scala/com/twitter/zk/AuthInfo.scala +++ b/util-zk/src/main/scala/com/twitter/zk/AuthInfo.scala @@ -4,7 +4,11 @@ case class AuthInfo(mode: String, data: Array[Byte]) object AuthInfo { def digest(user: String, secret: String): AuthInfo = { - val authString = "%s:%s".format(user, secret).getBytes("UTF-8") //DigestAuthenticationProvider.generateDigest("%s:%s".format(user, secret)).getBytes("UTF-8") + val authString = + "%s:%s" + .format(user, secret).getBytes( + "UTF-8" + ) //DigestAuthenticationProvider.generateDigest("%s:%s".format(user, secret)).getBytes("UTF-8") AuthInfo("digest", authString) } } diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index 569cf5c5ee..a66547ffbb 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -78,7 +78,12 @@ class ZkClientTest extends WordSpec with MockitoSugar { } def delete(path: String, version: Int)(wait: => Future[Unit]): Unit = { - when(zk.delete(meq(path), meq(version), any[AsyncCallback.VoidCallback], meq(null))) thenAnswer answer[ + when( + zk.delete( + meq(path), + meq(version), + any[AsyncCallback.VoidCallback], + meq(null))) thenAnswer answer[ AsyncCallback.VoidCallback ](2) { cbValue => wait onSuccess { _ => @@ -128,7 +133,12 @@ class ZkClientTest extends WordSpec with MockitoSugar { def getChildren(path: String)(children: => Future[ZNode.Children]): Unit = { val cb = ArgumentCaptor.forClass(classOf[AsyncCallback.Children2Callback]) - when(zk.getChildren(meq(path), meq(false), any[AsyncCallback.Children2Callback], meq(null))) thenAnswer answer[ + when( + zk.getChildren( + meq(path), + meq(false), + any[AsyncCallback.Children2Callback], + meq(null))) thenAnswer answer[ AsyncCallback.Children2Callback ](2) { cbValue => children onSuccess { znode => @@ -172,7 +182,12 @@ class ZkClientTest extends WordSpec with MockitoSugar { } def getData(path: String)(result: => Future[ZNode.Data]): Unit = { - when(zk.getData(meq(path), meq(false), any[AsyncCallback.DataCallback], meq(null))) thenAnswer answer[ + when( + zk.getData( + meq(path), + meq(false), + any[AsyncCallback.DataCallback], + meq(null))) thenAnswer answer[ AsyncCallback.DataCallback ](2) { cbValue => result onSuccess { z => @@ -783,7 +798,9 @@ class ZkClientTest extends WordSpec with MockitoSugar { }.flatten // Initially, we should get a ZNode.TreeUpdate for each node in the tree with only added nodes - val expectedByPath = treeChildren.map { z => z.path -> ZNode.TreeUpdate(z, z.children.toSet) }.toMap + val expectedByPath = treeChildren.map { z => + z.path -> ZNode.TreeUpdate(z, z.children.toSet) + }.toMap val updatesByPath = updateTree.map { z => val prior: Set[ZNode] = expectedByPath.get(z.path).map { _.added }.getOrElse(Set.empty) @@ -885,12 +902,14 @@ class ZkClientTest extends WordSpec with MockitoSugar { sync(znode.path) { Future.exception(new KeeperException.SystemErrorException) } - Await.ready(znode.sync() map { _ => - fail("Unexpected success") - } handle { - case e: KeeperException.SystemErrorException => - assert(e.getPath == znode.path) - }, 1.second) + Await.ready( + znode.sync() map { _ => + fail("Unexpected success") + } handle { + case e: KeeperException.SystemErrorException => + assert(e.getPath == znode.path) + }, + 1.second) } } } From c34cd8ef0a7fc16217035c02202de0ff5a962f79 Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Fri, 15 May 2020 00:00:56 +0000 Subject: [PATCH 320/796] util-security: Move Credentials from util-core to util-security Problem The Credentials class exists in util-core, but a better place for it would be in util-security. It predates the creation of util-security by many years which is why it is where it is. Nothing within util-core depends on it though. Solution move it from util-core to util-security. the namespace should be changed from com.twitter.util to com.twitter.util.security. JIRA Issues: CSL-9730 Differential Revision: https://phabricator.twitter.biz/D477984 --- CHANGELOG.rst | 9 +++++++++ build.sbt | 5 ++++- .../scala/com/twitter/util/security}/Credentials.scala | 2 +- .../src/test/scala/com/twitter/util/security/BUILD | 3 ++- .../com/twitter/util/security}/CredentialsTest.scala | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) rename {util-core/src/main/scala/com/twitter/util => util-security/src/main/scala/com/twitter/util/security}/Credentials.scala (98%) rename {util-core/src/test/scala/com/twitter/util => util-security/src/test/scala/com/twitter/util/security}/CredentialsTest.scala (98%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e6d5b0b17b..9536bccf4e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,8 +7,17 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ +* util-security: Moved Credentials from util-core + `c.t.util.Credentials` => `c.t.util.security.Credentials`. + ``PHAB_ID=D477984`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ +* util-core: Move Credentials to util-security + `c.t.util.Credentials` => `c.t.util.security.Credentials`. + ``PHAB_ID=D477984`` * util-core: Change the namespace of `ActivitySource` and its derivatives to `com.twitter.io` as its no longer considered experimental since the code has diff --git a/build.sbt b/build.sbt index 3965e9819f..8b5ee1b4ed 100644 --- a/build.sbt +++ b/build.sbt @@ -427,7 +427,10 @@ lazy val utilSecurity = Project( ).settings( sharedSettings ).settings( - name := "util-security" + name := "util-security", + libraryDependencies ++= Seq( + scalacheckLib + ) ).dependsOn(utilCore, utilLogging) lazy val utilStats = Project( diff --git a/util-core/src/main/scala/com/twitter/util/Credentials.scala b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala similarity index 98% rename from util-core/src/main/scala/com/twitter/util/Credentials.scala rename to util-security/src/main/scala/com/twitter/util/security/Credentials.scala index 4e125dbe45..0623a238e5 100644 --- a/util-core/src/main/scala/com/twitter/util/Credentials.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.twitter.util +package com.twitter.util.security import java.io.{File, IOException} diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index 6bf546126a..e3022418cd 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -3,9 +3,10 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/io", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/io", "util/util-security/src/main/scala/com/twitter/util/security", "util/util-security/src/test/resources", ], diff --git a/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala b/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala similarity index 98% rename from util-core/src/test/scala/com/twitter/util/CredentialsTest.scala rename to util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala index afa72565ee..0476866c23 100644 --- a/util-core/src/test/scala/com/twitter/util/CredentialsTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.twitter.util +package com.twitter.util.security import org.scalatest.FunSuite import org.scalatestplus.scalacheck.Checkers From 8453ba2dd1df9633902fdae7991c5fa103387511 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 19 May 2020 00:27:27 +0000 Subject: [PATCH 321/796] Twitter-oss: Prepare OSS libraries for release 20.5.0 Problem We want to release the next versions of our Twitter OSS libraries 20.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9788 Differential Revision: https://phabricator.twitter.biz/D486551 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9536bccf4e..3b75d48ad6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.5.0 +------ + New Features ~~~~~~~~~~~~ * util-security: Moved Credentials from util-core diff --git a/README.md b/README.md index 1425d370cf..666110422f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.4.1" +val utilCore = "com.twitter" %% "util-core" % "20.5.0" ``` # Units diff --git a/build.sbt b/build.sbt index 8b5ee1b4ed..158f7f898d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.5.0-SNAPSHOT" +val releaseVersion = "20.5.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 35581901dcbbb792a60fdde8c31600c11b716344 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 19 May 2020 01:17:18 +0000 Subject: [PATCH 322/796] Twitter-oss: update OSS libraries post-release to 20.6.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.6.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9788 Differential Revision: https://phabricator.twitter.biz/D486635 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 666110422f..123cdee665 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=master)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 158f7f898d..57508b5dbb 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.5.0" +val releaseVersion = "20.6.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 66bffd13932605321e7899badfadb23c801f9dbb Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 27 May 2020 17:40:56 +0000 Subject: [PATCH 323/796] Adapt scala rules scalaopts to pants compiler_option_sets This change adapts bazel's translation to pants' compiler option sets JIRA Issues: DPB-12420 Differential Revision: https://phabricator.twitter.biz/D490107 --- util-core/src/test/java/com/twitter/util/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index 1176d163f7..7732db3b19 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -4,6 +4,7 @@ java_library( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", + "3rdparty/jvm/com/google/errorprone:error_prone_annotations" ], ) From eaec32f4079fc5fb966e7c2af41f758d4c0029c4 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 28 May 2020 21:42:10 +0000 Subject: [PATCH 324/796] util|scrooge|finagle|twitter-server|finatra: Use `skip in publish` Problem sbt introduced `skip in publish` here: https://github.com/sbt/sbt/issues/3136, which replaces the need to configure multiple different settings in attempt to skip publishing of a sub-project within a multi-module project. The current workarounds to skip publishing of a sub-project appear to lack some efficacy in practice. Solution Switch to using `skip in publish` and also configure local overwrite to true to aid in release publishing. Differential Revision: https://phabricator.twitter.biz/D491635 --- build.sbt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/build.sbt b/build.sbt index 57508b5dbb..fcfdb02843 100644 --- a/build.sbt +++ b/build.sbt @@ -135,6 +135,7 @@ val baseSettings = Seq( pomIncludeRepository := { _ => false }, publishMavenStyle := true, publishConfiguration := publishConfiguration.value.withOverwrite(true), + publishLocalConfiguration := publishLocalConfiguration.value.withOverwrite(true), autoAPIMappings := true, apiURL := Some(url("https://twitter.github.io/util/docs/")), pomExtra := @@ -168,11 +169,7 @@ val baseSettings = Seq( val sharedSettings = defaultProjectSettings ++ baseSettings lazy val noPublishSettings = Seq( - publish := {}, - publishLocal := {}, - publishArtifact := false, - // sbt-pgp's publishSigned task needs this defined even though it is not publishing. - publishTo := Some(Resolver.file("Unused transient repository", file("target/unusedrepo"))) + skip in publish := true ) lazy val util = Project( From 77c76e418bea0b1ef43168cde73ae30a69fa0cc5 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Wed, 10 Jun 2020 22:02:40 +0000 Subject: [PATCH 325/796] util-stats: Add Java-friendly gauge methods to StatsReceiver Problem Most of the `StatsReceiver` methods are perfectly accessible from Java. The exception is `addGauge` method that not var-arg enabled due to the wrong order of arguments. Solution Add new Java-friendly gauge methods to the `StatsReceiver`, allowing Java users to use var-arg variants w/o any additional tooling (i.e., StatsReceivers). JIRA Issues: CSL-9545 Differential Revision: https://phabricator.twitter.biz/D497885 --- CHANGELOG.rst | 6 ++++ .../twitter/finagle/stats/StatsReceiver.scala | 33 +++++++++++++++++-- .../stats/StatsReceiverCompilationTest.java | 13 +++----- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b75d48ad6..1e3de66133 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ +* util-stats: Add two new Java-friendly methods to `StatsReceiver` (`addGauge` and `provideGauge`) + that take java.util.function.Supplier as well as list vararg argument last to enable better + developers' experience. ``PHAB_ID=D497885`` + 20.5.0 ------ diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 9c3459fcc3..071940439e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -2,6 +2,7 @@ package com.twitter.finagle.stats import java.lang.{Float => JFloat} import java.util.concurrent.Callable +import java.util.function.Supplier import scala.annotation.varargs /** @@ -29,6 +30,7 @@ object StatsReceiver { /** * [[StatsReceiver]] utility methods for ease of use from java. */ +@deprecated("Use StatsReceiver addGauge and provideGauge methods directly", "2020-06-10") object StatsReceivers { /** @@ -64,8 +66,6 @@ object StatsReceivers { * * Metrics created w/o an explicitly specified [[Verbosity]] level, will use [[Verbosity.Default]]. * Use [[VerbosityAdjustingStatsReceiver]] to adjust this behaviour. - * - * @see [[StatsReceivers]] for a Java-friendly API. */ trait StatsReceiver { @@ -134,6 +134,9 @@ trait StatsReceiver { * * @see [[StatsReceiver.addGauge]] if you can properly control the lifecycle * of the returned [[Gauge gauge]]. + * + * @see [[StatsReceiver.provideGauge(java.util.function.Supplier, String*)]] for a Java-friendly + * version. */ def provideGauge(name: String*)(f: => Float): Unit = { val gauge = addGauge(name: _*)(f) @@ -142,6 +145,13 @@ trait StatsReceiver { } } + /** + * Just like [[provideGauge()]] but optimized for better Java experience. + */ + @varargs + def provideGauge(f: Supplier[Float], name: String*): Unit = + provideGauge(name: _*)(f.get()) + /** * Add the function `f` as a [[Gauge gauge]] with the given name. * @@ -157,6 +167,9 @@ trait StatsReceiver { * @see [[StatsReceiver.provideGauge]] when there is not a good location * to store the returned [[Gauge gauge]] that can give the desired lifecycle. * + * @see [[StatsReceiver.addGauge(java.util.function.Supplier,String*)]] for a Java-friendly + * version. + * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ def addGauge(name: String*)(f: => Float): Gauge = addGauge(Verbosity.Default, name: _*)(f) @@ -176,11 +189,27 @@ trait StatsReceiver { * @see [[StatsReceiver.provideGauge]] when there is not a good location * to store the returned [[Gauge gauge]] that can give the desired lifecycle. * + * @see [[StatsReceiver.addGauge(java.util.function.Supplier,Verbosity,String*)]] for a + * Java-friendly version. + * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name)))(f) + /** + * Just like [[addGauge(String*,=>Float)]] but optimized for better Java experience. + */ + @varargs + def addGauge(f: Supplier[Float], name: String*): Gauge = addGauge(f, Verbosity.Default, name: _*) + + /** + * Just like [[addGauge(Verbosity,String*,=>Float)]] but optimized for better Java experience. + */ + @varargs + def addGauge(f: Supplier[Float], verbosity: Verbosity, name: String*): Gauge = + addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name)))(f.get()) + /** * Add the function `f` as a [[Gauge gauge]] with the given name. * diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java index 4a059be386..092ca8ec76 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java @@ -47,14 +47,11 @@ public Future call() { @Test public void testGaugeMethods() { InMemoryStatsReceiver sr = new InMemoryStatsReceiver(); - Callable callable = new Callable() { - public Float call() { - return 3.0f; - } - }; - Gauge gauge = StatsReceivers.addGauge(sr, callable, "hello", "world"); - StatsReceivers.provideGauge(sr, callable, "hello", "world"); - gauge.remove(); + Gauge a = sr.addGauge(() -> 3.0f, "hello", "world"); + Gauge b = sr.addGauge(() -> 4.0f, Verbosity.Debug(), "foo", "bar"); + sr.provideGauge(() -> 3.0f, "hello", "world"); + a.remove(); + b.remove(); } @Test From 4b6619b9797959be3324a5d10f7d6f813b406350 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Thu, 11 Jun 2020 01:55:41 +0000 Subject: [PATCH 326/796] util-app: simplify App close logic Problem App implements Closable and CloseAwaitably in order to deal with some lifecycle edge cases - primarily not related to App, but Twitter Server. The close relationship is not easy to understand. Solution We can simplify this relationship and keep the existing behavior by instead implementing `ClosableOnce`. We introduce a new `CloseOnceAwaitably` that can wrap a `ClosableOnce` and expose the same underlying behavior as the previous implementation without the need to track as much state within App. JIRA Issues: CSL-9864 Differential Revision: https://phabricator.twitter.biz/D497561 --- .../src/main/scala/com/twitter/app/App.scala | 73 ++++++++----------- .../test/scala/com/twitter/app/AppTest.scala | 3 - .../scala/com/twitter/util/Awaitable.scala | 29 ++++++++ .../scala/com/twitter/util/CloseOnce.scala | 28 +++++-- 4 files changed, 79 insertions(+), 54 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index ccc4466aa8..7383af8fbe 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -35,7 +35,7 @@ import scala.util.control.NonFatal * Note that a missing `main` is OK: mixins may provide behavior that * does not require defining a custom `main` method. */ -trait App extends Closable with CloseAwaitably { +trait App extends ClosableOnce with CloseOnceAwaitably { /** The name of the application, based on the classname */ val name: String = getClass.getName.stripSuffix("$") @@ -212,23 +212,16 @@ trait App extends Closable with CloseAwaitably { */ @volatile private[this] var closeDeadline = Time.Top - /** - * This is satisfied when all members of `exits` and `lastExits` have closed. - * - * @note Access needs to be mediated via the intrinsic lock. - */ - private[this] var closing: Future[Unit] = Future.never - /** * Close `closable` when shutdown is requested. Closables are closed in parallel. */ final def closeOnExit(closable: Closable): Unit = synchronized { - if (closing == Future.never) { + if (isClosed) { + // `close()` has already been called, we close eagerly + closable.close(closeDeadline) + } else { // `close()` not yet called, safe to add it exits.add(closable) - } else { - // `close()` already called, we need to close this here, immediately. - closable.close(closeDeadline) } } @@ -244,15 +237,15 @@ trait App extends Closable with CloseAwaitably { * In all cases, the close deadline is enforced. */ final def closeOnExitLast(closable: Closable): Unit = synchronized { - if (closing == Future.never) { - // `close()` not yet called, safe to add it - lastExits.add(closable) - } else { + if (isClosed) { // `close()` already called, we need to close this here, but only // after `close()` completes and `closing` is satisfied - closing + close(closeDeadline) .transform { _ => closable.close(closeDeadline) } .by(shutdownTimer, closeDeadline) + } else { + // `close()` not yet called, safe to add it + lastExits.add(closable) } } @@ -289,26 +282,19 @@ trait App extends Closable with CloseAwaitably { postmains.add(() => f) } - /** - * Notify the application that it may stop running. - * Returns a Future that is satisfied when the App has been torn down or errors at the deadline. - */ - final def close(deadline: Time): Future[Unit] = synchronized { - closing = closeAwaitably { - closeDeadline = deadline.max(Time.now + MinGrace) - Future - .collectToTry(exits.asScala.toSeq.map(_.close(closeDeadline))) - .by(shutdownTimer, closeDeadline) - .transform { - case Return(results) => - closeLastExits(results, closeDeadline) - case Throw(t) => - // this would be triggered by a timeout on the collectToTry of exits, - // still try to close last exits - closeLastExits(Seq(Throw(t)), closeDeadline) - } - } - closing + override protected def closeOnce(deadline: Time): Future[Unit] = synchronized { + closeDeadline = deadline.max(Time.now + MinGrace) + Future + .collectToTry(exits.asScala.toSeq.map(_.close(closeDeadline))) + .by(shutdownTimer, closeDeadline) + .transform { + case Return(results) => + closeLastExits(results, closeDeadline) + case Throw(t) => + // this would be triggered by a timeout on the collectToTry of exits, + // still try to close last exits + closeLastExits(Seq(Throw(t)), closeDeadline) + } } private[this] def newCloseException(errors: Seq[Throwable]): CloseException = { @@ -379,16 +365,17 @@ trait App extends Closable with CloseAwaitably { for (f <- postmains.asScala) f() - // We discard this future but we `Await.result` on `this` which is a - // `CloseAwaitable`, and this means the thread waits for the future to - // complete. - close(defaultCloseGracePeriod) + // We get a reference to the `close()` Future in order to Await upon it, to ensure the thread + // waits for `close()` to complete. + // Note that the Future returned here is the same as `promise` exposed via ClosableOnce, + // which is the same result that would be returned by an `Await.result` on `this`. + val closeFuture = close(defaultCloseGracePeriod) // The deadline to 'close' is advisory; we enforce it here. - if (!suppressGracefulShutdownErrors) Await.result(this, closeDeadline - Time.now) + if (!suppressGracefulShutdownErrors) Await.result(closeFuture, closeDeadline - Time.now) else { try { // even if we suppress shutdown errors, we give the resources time to close - Await.ready(this, closeDeadline - Time.now) + Await.ready(closeFuture, closeDeadline - Time.now) } catch { case e: TimeoutException => throw e // we want TimeoutExceptions to propagate diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index d8949dd832..6aef900f2d 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -42,9 +42,6 @@ class WeNeverClose extends App { closeOnExit(unClosable) - // and we're never ready - override def isReady(implicit permit: Awaitable.CanAwait): Boolean = false - def main(): Unit = {} } diff --git a/util-core/src/main/scala/com/twitter/util/Awaitable.scala b/util-core/src/main/scala/com/twitter/util/Awaitable.scala index ea0942c7f4..1cec24401e 100644 --- a/util-core/src/main/scala/com/twitter/util/Awaitable.scala +++ b/util-core/src/main/scala/com/twitter/util/Awaitable.scala @@ -222,6 +222,35 @@ private[util] trait CloseAwaitably0[U <: Unit] extends Awaitable[U] { onClose.isReady } +// See https://stackoverflow.com/questions/26643045/java-interoperability-woes-with-scala-generics-and-boxing +private[util] trait CloseOnceAwaitably0[U <: Unit] extends Awaitable[U] { self: CloseOnce => + + def ready(timeout: Duration)(implicit permit: Awaitable.CanAwait): this.type = { + closeFuture.ready(timeout) + this + } + + def result(timeout: Duration)(implicit permit: Awaitable.CanAwait): U = + closeFuture.result(timeout).asInstanceOf[U] + + def isReady(implicit permit: Awaitable.CanAwait): Boolean = + closeFuture.isReady +} + +/** + * A mixin to make an [[com.twitter.util.Awaitable]] out + * of a [[com.twitter.util.ClosableOnce]]. + * + * {{{ + * class MyClosable extends ClosableOnce with CloseOnceAwaitably { + * def closeOnce(deadline: Time) = { + * // close the resource + * } + * } + * }}} + */ +trait CloseOnceAwaitably extends CloseOnceAwaitably0[Unit] { self: CloseOnce => } + /** * A mixin to make an [[com.twitter.util.Awaitable]] out * of a [[com.twitter.util.Closable]]. diff --git a/util-core/src/main/scala/com/twitter/util/CloseOnce.scala b/util-core/src/main/scala/com/twitter/util/CloseOnce.scala index 2813480a97..8efa1466be 100644 --- a/util-core/src/main/scala/com/twitter/util/CloseOnce.scala +++ b/util-core/src/main/scala/com/twitter/util/CloseOnce.scala @@ -34,10 +34,20 @@ trait CloseOnce { self: Closable => * finish up other tasks. * * @note if this method throws a synchronous exception, that exception will - * be wrapped in a failed future. + * be wrapped in a failed future. If a fatal error is thrown synchronously, + * the error will be wrapped in a failed Future AND thrown directly. */ protected def closeOnce(deadline: Time): Future[Unit] + /** + * The [[Future]] satisfied upon completion of [[close()]]. + * + * @note we do not expose direct access to the underlying [[closePromise Promise]], because + * the [[Promise]] state is mutable - we only want mutation to occur within this + * [[CloseOnce]] trait. + */ + protected final def closeFuture: Future[Unit] = closePromise + /** * Signals whether or not this [[Closable]] has been closed. * @@ -48,13 +58,15 @@ trait CloseOnce { self: Closable => override final def close(deadline: Time): Future[Unit] = { // only call `closeOnce()` and assign `closePromise` if this is the first `close()` invocation if (firstCloseInvoked()) { - val closeF = - try { - closeOnce(deadline) - } catch { - case NonFatal(ex) => Future.exception(ex) - } - closePromise.become(closeF) + try { + closePromise.become(closeOnce(deadline)) + } catch { + case NonFatal(ex) => + closePromise.setException(ex) + case t: Throwable => + closePromise.setException(t) + throw t + } } closePromise From 19f13a075adeeffd6ef1449e7050eff5e3b070e3 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 12 Jun 2020 01:27:06 +0000 Subject: [PATCH 327/796] util-app: Add Flaggable[LocalTime] Problem There is no `Flaggable` instance for `java.time.LocalTme`. Solution Add a `Flaggable[LocalTime]` so Finatra and others can consume it. JIRA Issues: CSL-9552 Differential Revision: https://phabricator.twitter.biz/D499606 --- CHANGELOG.rst | 3 +++ .../src/main/scala/com/twitter/app/Flaggable.scala | 3 +++ .../test/scala/com/twitter/app/FlaggableTest.scala | 12 ++++++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1e3de66133..55a4073bb9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,10 +9,13 @@ Unreleased New Features ~~~~~~~~~~~~ + * util-stats: Add two new Java-friendly methods to `StatsReceiver` (`addGauge` and `provideGauge`) that take java.util.function.Supplier as well as list vararg argument last to enable better developers' experience. ``PHAB_ID=D497885`` +* util-app: Add a `Flaggable` instance for `java.time.LocalTime`. ``PHAB_ID=D499606`` + 20.5.0 ------ diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index 03fe738118..f210e06181 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -10,6 +10,7 @@ import java.lang.{ Long => JLong } import java.net.InetSocketAddress +import java.time.LocalTime import java.util.{List => JList, Map => JMap, Set => JSet} import scala.jdk.CollectionConverters._ @@ -120,6 +121,8 @@ object Flaggable { private val defaultTimeFormat = new TimeFormat("yyyy-MM-dd HH:mm:ss Z") implicit val ofTime: Flaggable[Time] = mandatory(defaultTimeFormat.parse(_)) + implicit val ofJavaLocalTime: Flaggable[LocalTime] = mandatory(LocalTime.parse) + implicit val ofInetSocketAddress: Flaggable[InetSocketAddress] = new Flaggable[InetSocketAddress] { def parse(v: String): InetSocketAddress = v.split(":") match { diff --git a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala index bf34c7b067..3d166b3b1e 100644 --- a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala @@ -1,6 +1,8 @@ package com.twitter.app import org.scalatest.FunSuite +import com.twitter.util.{Duration, Time} +import java.time.LocalTime class FlaggableTest extends FunSuite { @@ -16,6 +18,16 @@ class FlaggableTest extends FunSuite { assert(Flaggable.ofString.parse("blah") == "blah") } + test("Flaggagle: parse dates and times") { + val t = Time.fromSeconds(123) + assert(Flaggable.ofTime.parse(t.toString) == t) + + assert(Flaggable.ofDuration.parse("1.second") == Duration.fromSeconds(1)) + + val tt = LocalTime.ofSecondOfDay(123) + assert(Flaggable.ofJavaLocalTime.parse(tt.toString) == tt) + } + test("Flaggable: parse/show inet addresses") { // we aren't binding to this port, so any one will do. val port = 1589 From 4ef3c9b99a6421cb1b28ebf5355582efb4392665 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 12 Jun 2020 03:05:19 +0000 Subject: [PATCH 328/796] util-app: Allow retrieving unparsed flag values Problem There is no way to retrieve an unparsed flag value. Finatra, for example, can use that within its flag binding machinery. Solution Allow retrieving unparsed flag values via new APIs: `getUnparsed` and `defWithDefaultUnparsed`. JIRA Issues: CSL-9552 Differential Revision: https://phabricator.twitter.biz/D499628 --- CHANGELOG.rst | 4 +++- .../src/main/scala/com/twitter/app/Flag.scala | 18 ++++++++++++++++++ .../test/scala/com/twitter/app/FlagTest.scala | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 55a4073bb9..699488612c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,13 +9,15 @@ Unreleased New Features ~~~~~~~~~~~~ - * util-stats: Add two new Java-friendly methods to `StatsReceiver` (`addGauge` and `provideGauge`) that take java.util.function.Supplier as well as list vararg argument last to enable better developers' experience. ``PHAB_ID=D497885`` * util-app: Add a `Flaggable` instance for `java.time.LocalTime`. ``PHAB_ID=D499606`` +* util-app: Add two new methods to retrieve flag's unparsed value (as string): `Flag.getUnparsed` + and `Flag.getWithDefaultUnparsed`. ``PHAB_ID=D499628`` + 20.5.0 ------ diff --git a/util-app/src/main/scala/com/twitter/app/Flag.scala b/util-app/src/main/scala/com/twitter/app/Flag.scala index 191934d4ae..2f61c94a86 100644 --- a/util-app/src/main/scala/com/twitter/app/Flag.scala +++ b/util-app/src/main/scala/com/twitter/app/Flag.scala @@ -232,6 +232,16 @@ class Flag[T: Flaggable] private[app] ( */ def get: Option[T] = getValue + /** + * Get the *unparsed* (as string) value if it has been set. + * + * @note if no user-defined value has been set, `None` will be returned even when a default value + * is supplied. + * + * @see [[Flag.getWithDefaultUnparsed]] + */ + def getUnparsed: Option[String] = getValue.map(flaggable.show) + /** * Get the value if it has been set or if there is a default value supplied. * @@ -240,6 +250,14 @@ class Flag[T: Flaggable] private[app] ( */ def getWithDefault: Option[T] = valueOrDefault + /** + * Get the *unparsed* (as string) value if it has been set or if there is a default value supplied. + * + * @see [[Flag.get]] and [[Flag.isDefined]] if you are interested in + * determining if there is a supplied value. + */ + def getWithDefaultUnparsed: Option[String] = valueOrDefault.map(flaggable.show) + /** String representation of this flag's default value */ def defaultString(): String = { try { diff --git a/util-app/src/test/scala/com/twitter/app/FlagTest.scala b/util-app/src/test/scala/com/twitter/app/FlagTest.scala index 4dcc3149c1..c714ec3117 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagTest.scala @@ -175,6 +175,15 @@ class FlagTest extends FunSuite { assert(noDefaultAndSupplied.get.contains(3)) } + test("Flag.getUnparsed") { + val ctx = new GetCtx() + import ctx._ + + assert(withDefault.getUnparsed.isEmpty) + assert(noDefault.getUnparsed.isEmpty) + assert(noDefaultAndSupplied.getUnparsed.contains("3")) + } + test("Flag.getWithDefault") { val ctx = new GetCtx() import ctx._ @@ -184,6 +193,15 @@ class FlagTest extends FunSuite { assert(noDefaultAndSupplied.getWithDefault.contains(3)) } + test("Flag.getWithDefaultUnparsed") { + val ctx = new GetCtx() + import ctx._ + + assert(withDefault.getWithDefaultUnparsed.contains("1")) + assert(noDefault.getWithDefaultUnparsed.isEmpty) + assert(noDefaultAndSupplied.getWithDefaultUnparsed.contains("3")) + } + test("Flag.usage - new flag value") { val marsColony = new ColonizeTestApp() try { From 331f04aa3901359b0d53a21a0fdae3488f62aef2 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 16 Jun 2020 01:21:52 +0000 Subject: [PATCH 329/796] util|scrooge|finagle|twitter-server|finatra: Update TravisCI badge links Problem/Solution We recently moved the projects from `travis-ci.org` to `travis-ci.com` and need to now update the badge links. Differential Revision: https://phabricator.twitter.biz/D501490 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 123cdee665..18232b8d7b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.org/twitter/util.svg?branch=develop)](https://travis-ci.org/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) From 0b3c85bb5804f0db20b9b27975d79cf8c5861c43 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 22 Jun 2020 19:55:59 +0000 Subject: [PATCH 330/796] util-core: Fix a race in Var.flatMap Problem `Var.flatMap` has a race between the close of observations and the rebuilding and registering of new observations based on updates to the Var being flatmapped. Solution Be more careful with respect to closing and updating the internal state. JIRA Issues: CSL-9883 Differential Revision: https://phabricator.twitter.biz/D502882 --- .../src/main/scala/com/twitter/util/Var.scala | 70 +++++++++++++------ 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 1cbbd17018..16ba95b88d 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -95,28 +95,52 @@ trait Var[+T] { self => * unobserved Var returned by flatMap will not invoke `f` */ def flatMap[U](f: T => Var[U]): Var[U] = new Var[U] { - def observe(depth: Int, obs: Observer[U]) = { - val inner = new AtomicReference(Closable.nop) - val outer = self.observe( - depth, - Observer(t => { - // TODO: Right now we rely on synchronous propagation; and - // thus also synchronous closes. We should instead perform - // asynchronous propagation so that it is is safe & - // predictable to have asynchronously closing Vars, for - // example. Currently the only source of potentially - // asynchronous closing is Var.async; here we have modified - // the external process to close asynchronously with the Var - // itself. Thus we know the code path here is synchronous: - // we control all Var implementations, and also all Closable - // combinators have been modified to evaluate their respective - // Futures eagerly. - val done = inner.getAndSet(f(t).observe(depth + 1, obs)).close() - assert(done.isDone) - }) - ) - - Closable.sequence(outer, Closable.ref(inner)) + private class ObserverManager(depth: Int, obs: Observer[U]) extends Closable with (T => Unit) { + private[this] var closable: Closable = Closable.nop + private[this] var closed: Boolean = false + + def apply(t: T): Unit = { + // We have to synchronize and make sure we're not already closed or else + // we may generate a new observation that has raced with the a `close` + // call and lost, therefore making an orphan and a resource leak. + val toClose = this.synchronized { + if (closed) Closable.nop + else { + val old = closable + closable = f(t).observe(depth + 1, obs) + old + } + } + // TODO: Right now we rely on synchronous propagation; and + // thus also synchronous closes. We should instead perform + // asynchronous propagation so that it is is safe & + // predictable to have asynchronously closing Vars, for + // example. Currently the only source of potentially + // asynchronous closing is Var.async; here we have modified + // the external process to close asynchronously with the Var + // itself. Thus we know the code path here is synchronous: + // we control all Var implementations, and also all Closable + // combinators have been modified to evaluate their respective + // Futures eagerly. + val done = toClose.close() + assert(done.isDone) + } + + def close(deadline: Time): Future[Unit] = { + val toClose = this.synchronized { + closed = true + val old = closable + closable = Closable.nop + old + } + toClose.close(deadline) + } + } + + def observe(depth: Int, obs: Observer[U]): Closable = { + val manager = new ObserverManager(depth, obs) + val outer = self.observe(depth, Observer(manager)) + Closable.sequence(outer, manager) } } @@ -515,7 +539,7 @@ private[util] class UpdatableVar[T](init: T) extends Var[T] with Updatable[T] wi obs.publish(this, value, version) new Closable { - def close(deadline: Time) = { + def close(deadline: Time): Future[Unit] = { party.active = false cas(_ - party) Future.Done From c69f07e44852b88091ff7c581bb043788133034b Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 23 Jun 2020 16:43:45 +0000 Subject: [PATCH 331/796] util|scrooge|finagle|twitter-server|finatra - Update READMEs Problem/Solution Ensure all the README files have a license with copyright date section. Update the date to 2020 (finally). Fix up Finatra Thrift example to the latest code. Differential Revision: https://phabricator.twitter.biz/D505486 --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 18232b8d7b..598f44b69e 100644 --- a/README.md +++ b/README.md @@ -195,3 +195,9 @@ which is read by applying `elapsed`: ```scala val duration: Duration = elapsed() ``` + +## License + +Copyright 2010-2020 Twitter, Inc. + +Licensed under the Apache License, Version 2.0: https://www.apache.org/licenses/LICENSE-2.0 From e0d420e55e0e861af06e8a116a7ad016aa13bd19 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 23 Jun 2020 23:58:54 +0000 Subject: [PATCH 332/796] twitter-oss: Prepare OSS libraries for release 20.6.0 Problem We want to release the next versions of our Twitter OSS libraries 20.6.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D506048 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 699488612c..a80ab77290 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.6.0 +------ + New Features ~~~~~~~~~~~~ * util-stats: Add two new Java-friendly methods to `StatsReceiver` (`addGauge` and `provideGauge`) diff --git a/README.md b/README.md index 598f44b69e..121afd2bb0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=master)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.5.0" +val utilCore = "com.twitter" %% "util-core" % "20.6.0" ``` # Units diff --git a/build.sbt b/build.sbt index fcfdb02843..2d7fd3bdbb 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.6.0-SNAPSHOT" +val releaseVersion = "20.6.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From c074a5fcb1ac3430a272253b243170a0561ab151 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 24 Jun 2020 01:05:30 +0000 Subject: [PATCH 333/796] twitter-oss: Update OSS Libraries Post-Release to 20.7.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.7.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D506108 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 121afd2bb0..02bcc38a0d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=master)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 2d7fd3bdbb..fe941c8fdf 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.6.0" +val releaseVersion = "20.7.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 2d2e680310c21052f3879f30c568515585b3148b Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Wed, 1 Jul 2020 00:05:34 +0000 Subject: [PATCH 334/796] util-app: add ability to observe App lifecycle events Problem We don't have the ability to observe when specific lifecycle events of App occur. We primarily have a need for improving our testing utilities throughout the Twitter stack, but this could have further utility for exposing behavior in our stack (ex: timing lifecycle phase execution). Solution Introduce the `AppLifecycle` trait, which specifies the different lifecycle events within App and the ability to introduce new events in the Twitter stack. `App` is modified to emit lifecycle state. Observers can be added to `App` to observe the emitted state. JIRA Issues: CSL-9908 Differential Revision: https://phabricator.twitter.biz/D508145 --- CHANGELOG.rst | 4 + build.sbt | 12 ++- .../scala/com/twitter/app/lifecycle/BUILD | 15 +++ .../com/twitter/app/lifecycle/Event.scala | 46 +++++++++ .../com/twitter/app/lifecycle/Notifier.scala | 64 ++++++++++++ .../com/twitter/app/lifecycle/Observer.scala | 16 +++ .../scala/com/twitter/app/lifecycle/BUILD | 10 ++ .../twitter/app/lifecycle/NotifierTest.scala | 94 ++++++++++++++++++ util-app/src/main/scala/BUILD | 6 +- .../src/main/scala/com/twitter/app/App.scala | 97 +++++++++++-------- .../scala/com/twitter/app/Lifecycle.scala | 32 ++++++ .../test/scala/com/twitter/app/AppTest.scala | 47 ++++++++- 12 files changed, 399 insertions(+), 44 deletions(-) create mode 100644 util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD create mode 100644 util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Event.scala create mode 100644 util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Notifier.scala create mode 100644 util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Observer.scala create mode 100644 util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD create mode 100644 util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/NotifierTest.scala create mode 100644 util-app/src/main/scala/com/twitter/app/Lifecycle.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a80ab77290..0057da1b9b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ +* util-app: Add ability to observe App lifecycle events. ``PHAB_ID=D508145`` + 20.6.0 ------ diff --git a/build.sbt b/build.sbt index fe941c8fdf..bf222539e8 100644 --- a/build.sbt +++ b/build.sbt @@ -185,6 +185,7 @@ lazy val util = Project( ) ).aggregate( utilApp, + utilAppLifecycle, utilBenchmark, utilCache, utilCacheGuava, @@ -218,7 +219,16 @@ lazy val utilApp = Project( sharedSettings ).settings( name := "util-app" - ).dependsOn(utilCore, utilRegistry) + ).dependsOn(utilAppLifecycle, utilCore, utilRegistry) + +lazy val utilAppLifecycle = Project( + id = "util-app-lifecycle", + base = file("util-app-lifecycle") +).settings( + sharedSettings +).settings( + name := "util-app-lifecycle" +).dependsOn(utilCore) lazy val utilBenchmark = Project( id = "util-benchmark", diff --git a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD new file mode 100644 index 0000000000..1ec3f09863 --- /dev/null +++ b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-app-lifecycle", + repo = artifactory, + ), + dependencies = [ + "util/util-core:util-core-util", + ], + exports = [ + "util/util-core:util-core-util", + ], +) diff --git a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Event.scala b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Event.scala new file mode 100644 index 0000000000..bf10e145fa --- /dev/null +++ b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Event.scala @@ -0,0 +1,46 @@ +package com.twitter.app.lifecycle + +/** + * Represents the phase in the application lifecycle that an event occurs in. + * + * App's Lifecycle comprises the following phases + * + * [ Register ] -> [ Load Bindings ] -> [ Init ] -> [ Parse Args ] -> [ Pre Main ] -> + * [ Main ] -> [ Post Main ] -> [ Close ] -> [ CloseExitLast ] + * + * @see [[https://twitter.github.io/twitter-server/Features.html#lifecycle-management TwitterServer]] + * @see [[https://twitter.github.io/finatra/user-guide/getting-started/lifecycle.html Finatra]] + */ +private[twitter] sealed trait Event + +private[twitter] object Event { + + /* com.twitter.app.App + com.twitter.server.Server events */ + case object Register extends Event + case object LoadBindings extends Event + case object Init extends Event + case object ParseArgs extends Event + case object PreMain extends Event + case object Main extends Event + case object PostMain extends Event + case object Close extends Event + case object CloseExitLast extends Event + + /* com.twitter.server.Lifecycle.Warmup events */ + case object PrebindWarmup extends Event + case object WarmupComplete extends Event + + /* com.twitter.inject.App + com.twitter.inject.TwitterServer events */ + /* + * [ Register ] -> [ Load Bindings] -> [ Init ] -> [ Parse Args ] -> [ Pre Main ] -> + * [ Main // the following events take place within main() + * ( Post Injector Startup ) -> ( Warmup ) -> ( Before Post Warmup ) -> ( Post Warmup ) -> + * ( After Post Warmup ) + * ] -> [ Post Main ] -> [ Close ] -> [ CloseExitLast ] + */ + case object PostInjectorStartup extends Event + case object Warmup extends Event + case object BeforePostWarmup extends Event + case object PostWarmup extends Event + case object AfterPostWarmup extends Event +} diff --git a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Notifier.scala b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Notifier.scala new file mode 100644 index 0000000000..5424961aa4 --- /dev/null +++ b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Notifier.scala @@ -0,0 +1,64 @@ +package com.twitter.app.lifecycle + +import com.twitter.util.{Future, Return, Throw} + +/** + * The [[Notifier]] emits signals to [[Observer observers]] when lifecycle [[Event events]] are + * executed. + * + * @param observers the observers who will be notified of lifecycle event activities + * @note exposed for testing + */ +private[twitter] class Notifier private[app] (observers: Iterable[Observer]) { + + /** + * Record and execute a lifecycle event and emit the result to any [[Observer observers]]. + * + * @param event The [[Event lifecycle event]] to execute. + * @param f The function to execute as part of this [[Event lifecycle event]]. + */ + final def apply(event: Event)(f: => Unit): Unit = execAndSignal(event, f) + + /** + * Record and execute a lifecycle event [[Future]] and emit the result to + * any [[Observer observers]] on completion of the [[Future]]. + * + * @param event The [[Event lifecycle event]] to execute. + * @param f The Future that will be responded to on completion for this + * [[Event lifecycle event]]. + */ + final def future(event: Event)(f: Future[Unit]): Future[Unit] = { + onEntry(event) + f.respond { + case Return(_) => + onSuccess(event) + case Throw(t) => + onFailure(event, t) + } + } + + private[this] def execAndSignal(event: Event, f: => Unit): Unit = { + onEntry(event) + try { + f + onSuccess(event) + } catch { + case t: Throwable => + onFailure(event, t) + throw t + } + } + + // notify all observers we are entering the event + private[this] def onEntry(stage: Event): Unit = + observers.foreach(_.onEntry(stage)) + + // notify all observers on success + private[this] def onSuccess(stage: Event): Unit = + observers.foreach(_.onSuccess(stage)) + + // notify all observers on failures + private[this] def onFailure(stage: Event, throwable: Throwable): Unit = + observers.foreach(_.onFailure(stage, throwable)) + +} diff --git a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Observer.scala b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Observer.scala new file mode 100644 index 0000000000..f29788a812 --- /dev/null +++ b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/Observer.scala @@ -0,0 +1,16 @@ +package com.twitter.app.lifecycle + +/** + * A trait for observing lifecycle event notifications + */ +private[twitter] trait Observer { + + /** notification that the [[event]] lifecycle phase has begun */ + def onEntry(event: Event): Unit = () + + /** notification that the [[event]] lifecycle phase has completed successfully */ + def onSuccess(event: Event): Unit + + /** notification that the [[event]] lifecycle phase has encountered [[throwable]] error */ + def onFailure(stage: Event, throwable: Throwable): Unit +} diff --git a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD new file mode 100644 index 0000000000..adb6046d63 --- /dev/null +++ b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD @@ -0,0 +1,10 @@ +junit_tests( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "util/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle", + "util/util-core:util-core-util", + ], +) diff --git a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/NotifierTest.scala b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/NotifierTest.scala new file mode 100644 index 0000000000..eef5439bfb --- /dev/null +++ b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/NotifierTest.scala @@ -0,0 +1,94 @@ +package com.twitter.app.lifecycle + +import org.scalatest.FunSuite +import Event._ + +class NotifierTest extends FunSuite { + + test("emits success") { + var success = false + var observedEvent: Event = null + val observer = new Observer { + def onSuccess(event: Event): Unit = { + observedEvent = event + success = true + } + + def onFailure(event: Event, throwable: Throwable): Unit = { + observedEvent = event + success = false + } + } + val notifier = new Notifier(Seq(observer)) + notifier(PreMain) { + // success + () + } + assert(observedEvent == PreMain, "Observer did not see PreMain event") + assert(success, "Observer did not see onSuccess") + } + + test("emits failure and throws") { + var failed = false + var observedEvent: Event = null + var thrown: Throwable = null + + val observer = new Observer { + def onSuccess(event: Event): Unit = { + observedEvent = event + failed = false + } + + def onFailure(event: Event, throwable: Throwable): Unit = { + observedEvent = event + failed = true + thrown = throwable + } + } + val notifier = new Notifier(Seq(observer)) + + intercept[IllegalStateException] { + notifier(PostMain) { + throw new IllegalStateException("BOOM!") + } + } + } + + test("continues to emit after failure") { + var failed = false + var observedEvent: Event = null + var thrown: Throwable = null + + val observer = new Observer { + def onSuccess(event: Event): Unit = { + observedEvent = event + failed = false + } + + def onFailure(event: Event, throwable: Throwable): Unit = { + observedEvent = event + failed = true + thrown = throwable + } + } + val notifier = new Notifier(Seq(observer)) + + intercept[IllegalStateException] { + notifier(PostMain) { + throw new IllegalStateException("BOOM!") + } + } + assert(observedEvent == PostMain, "Observer did not see PostMain event") + assert(failed, "Observer did not see onFailure") + assert(thrown.isInstanceOf[IllegalStateException] && thrown.getMessage == "BOOM!") + + // our next lifecycle event will succeed + notifier(Close) { + () + } + + assert(observedEvent == Close) + assert(failed == false) + } + +} diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index b46efc9d04..dec16f5047 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -7,14 +7,16 @@ scala_library( repo = artifactory, ), dependencies = [ + "util/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle", "util/util-app/src/main/java/com/twitter/app", - "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", "util/util-registry/src/main/scala/com/twitter/util/registry", ], exports = [ + "util/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle", "util/util-app/src/main/java/com/twitter/app", - "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", ], ) diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index 7383af8fbe..6eefbb74ec 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -1,5 +1,6 @@ package com.twitter.app +import com.twitter.app.lifecycle.Event._ import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.lang.reflect.InvocationTargetException @@ -35,7 +36,7 @@ import scala.util.control.NonFatal * Note that a missing `main` is OK: mixins may provide behavior that * does not require defining a custom `main` method. */ -trait App extends ClosableOnce with CloseOnceAwaitably { +trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { /** The name of the application, based on the classname */ val name: String = getClass.getName.stripSuffix("$") @@ -289,11 +290,15 @@ trait App extends ClosableOnce with CloseOnceAwaitably { .by(shutdownTimer, closeDeadline) .transform { case Return(results) => - closeLastExits(results, closeDeadline) + observeFuture(CloseExitLast) { + closeLastExits(results, closeDeadline) + } case Throw(t) => // this would be triggered by a timeout on the collectToTry of exits, // still try to close last exits - closeLastExits(Seq(Throw(t)), closeDeadline) + observeFuture(CloseExitLast) { + closeLastExits(Seq(Throw(t)), closeDeadline) + } } } @@ -342,44 +347,58 @@ trait App extends ClosableOnce with CloseOnceAwaitably { } final def nonExitingMain(args: Array[String]): Unit = { - App.register(this) - - loadServiceBindings.foreach { binding => LoadService.bind(binding) } - - for (f <- inits) f() - - parseArgs(args) - - for (f <- premains) f() - - // Get a main() if it's defined. It's possible to define traits that only use pre/post mains. - val mainMethod = - try Some(getClass.getMethod("main")) - catch { case _: NoSuchMethodException => None } - - // Invoke main() if it exists. - mainMethod.foreach { method => - try method.invoke(this) - catch { case e: InvocationTargetException => throw e.getCause } + observe(Register) { + App.register(this) + } + observe(LoadBindings) { + loadServiceBindings.foreach { binding => LoadService.bind(binding) } + } + observe(Init) { + for (f <- inits) f() + } + observe(ParseArgs) { + parseArgs(args) + } + observe(PreMain) { + for (f <- premains) f() + } + observe(Main) { + // Get a main() if it's defined. It's possible to define traits that only use pre/post mains. + val mainMethod = + try Some(getClass.getMethod("main")) + catch { + case _: NoSuchMethodException => None + } + + // Invoke main() if it exists. + mainMethod.foreach { method => + try method.invoke(this) + catch { + case e: InvocationTargetException => throw e.getCause + } + } + } + observe(PostMain) { + for (f <- postmains.asScala) f() } - for (f <- postmains.asScala) f() - - // We get a reference to the `close()` Future in order to Await upon it, to ensure the thread - // waits for `close()` to complete. - // Note that the Future returned here is the same as `promise` exposed via ClosableOnce, - // which is the same result that would be returned by an `Await.result` on `this`. - val closeFuture = close(defaultCloseGracePeriod) - - // The deadline to 'close' is advisory; we enforce it here. - if (!suppressGracefulShutdownErrors) Await.result(closeFuture, closeDeadline - Time.now) - else { - try { // even if we suppress shutdown errors, we give the resources time to close - Await.ready(closeFuture, closeDeadline - Time.now) - } catch { - case e: TimeoutException => - throw e // we want TimeoutExceptions to propagate - case NonFatal(_) => () + observe(Close) { + // We get a reference to the `close()` Future in order to Await upon it, to ensure the thread + // waits for `close()` to complete. + // Note that the Future returned here is the same as `promise` exposed via ClosableOnce, + // which is the same result that would be returned by an `Await.result` on `this`. + val closeFuture = close(defaultCloseGracePeriod) + + // The deadline to 'close' is advisory; we enforce it here. + if (!suppressGracefulShutdownErrors) Await.result(closeFuture, closeDeadline - Time.now) + else { + try { // even if we suppress shutdown errors, we give the resources time to close + Await.ready(closeFuture, closeDeadline - Time.now) + } catch { + case e: TimeoutException => + throw e // we want TimeoutExceptions to propagate + case NonFatal(_) => () + } } } } diff --git a/util-app/src/main/scala/com/twitter/app/Lifecycle.scala b/util-app/src/main/scala/com/twitter/app/Lifecycle.scala new file mode 100644 index 0000000000..454e55d1bc --- /dev/null +++ b/util-app/src/main/scala/com/twitter/app/Lifecycle.scala @@ -0,0 +1,32 @@ +package com.twitter.app + +import com.twitter.app.lifecycle.{Event, Notifier, Observer} +import com.twitter.util.Future +import java.util.concurrent.ConcurrentLinkedQueue +import scala.jdk.CollectionConverters._ + +/** + * Mixin to an App that allows for observing lifecycle events. + * To emit an event + * {{{ observe(PreMain) { // logic for premain } }}} + */ +private[app] trait Lifecycle { self: App => + private[this] val observers = new ConcurrentLinkedQueue[Observer]() + private[this] val notifier = new Notifier(observers.asScala) + + /** + * Notifies [[Observer observers]] of [[com.twitter.app.lifecycle.Event events]] + */ + protected final def observe(event: Event)(f: => Unit): Unit = + notifier(event)(f) + + /** + * Notifies [[Observer observers]] of [[Future]] [[com.twitter.app.lifecycle.Event events]] + */ + protected final def observeFuture(event: Event)(f: Future[Unit]): Future[Unit] = + notifier.future(event)(f) + + /** Add a lifecycle [[com.twitter.app.lifecycle.Event]] [[Observer]] */ + private[twitter] final def withObserver(observer: Observer): Unit = + observers.add(observer) +} diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 6aef900f2d..4d2d541486 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -1,10 +1,13 @@ package com.twitter.app +import com.twitter.app.lifecycle.Event._ +import com.twitter.app.lifecycle.{Event, Observer} import com.twitter.app.LoadService.Binding import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.util.concurrent.ConcurrentLinkedQueue import org.scalatest.FunSuite +import scala.jdk.CollectionConverters._ import scala.language.reflectiveCalls class TestApp(f: () => Unit) extends App { @@ -98,8 +101,6 @@ class AppTest extends FunSuite { } test("App: order of hooks") { - import scala.jdk.CollectionConverters._ - val q = new ConcurrentLinkedQueue[Int] class Test1 extends App { onExitLast(q.add(5)) @@ -113,6 +114,48 @@ class AppTest extends FunSuite { assert(q.asScala.toSeq == Seq(0, 1, 2, 3, 4, 5)) } + test("App: order of hooks via observer") { + val enterQueue = new ConcurrentLinkedQueue[Event] + val completeQueue = new ConcurrentLinkedQueue[Event] + val observer = new Observer { + override def onEntry(event: Event): Unit = enterQueue.add(event) + def onSuccess(event: Event): Unit = completeQueue.add(event) + def onFailure(event: Event, throwable: Throwable): Unit = + completeQueue.add(event) + } + + class ObservedApp extends App + val test = new ObservedApp + test.withObserver(observer) + test.main(Array.empty) + + assert( + enterQueue.asScala.toSeq == Seq( + Register, + LoadBindings, + Init, + ParseArgs, + PreMain, + Main, + PostMain, + Close, + CloseExitLast + )) + + assert( + completeQueue.asScala.toSeq == Seq( + Register, + LoadBindings, + Init, + ParseArgs, + PreMain, + Main, + PostMain, + CloseExitLast, // CloseExitLast COMPLETES within the Close phase + Close + )) + } + test("loadServiceBinds") { trait Iface1 trait Iface2 From 228660acb0af076ff3330ce11228d5f7d79d32a4 Mon Sep 17 00:00:00 2001 From: Vijay Rajaram Date: Thu, 2 Jul 2020 23:02:09 +0000 Subject: [PATCH 335/796] [tiny] Bump caffeine version to 2.8.5. Bump caffeine version to 2.8.5 (latest) JIRA Issues: SD-14992 Differential Revision: https://phabricator.twitter.biz/D511414 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index bf222539e8..eab87f4f06 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,7 @@ val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.0" val guavaLib = "com.google.guava" % "guava" % "25.1-jre" -val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.0" +val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.5" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion From b92578c7117f9dfd90fcd168d7cb27b51fa7aa84 Mon Sep 17 00:00:00 2001 From: Borja Lorente Date: Thu, 9 Jul 2020 18:52:27 +0000 Subject: [PATCH 336/796] [Bazel Migration] Minor fixes in BUILD files Problem/Solution There are incompatibilities in BUILD files between Pants and Bazel. Let's fix the following: - `rglobs` and `globs` are not supported in Bazel and deprecated in Pants. - `tags` attributes should be a list, since Bazel doesn't support sets. - correct wonky values for `sources` fields. Differential Revision: https://phabricator.twitter.biz/D510399 --- util-routing/src/test/scala/com/twitter/util/routing/BUILD | 2 +- .../src/test/scala/com/twitter/util/routing/dynamic/BUILD | 2 +- .../src/test/scala/com/twitter/util/routing/simple/BUILD | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index 2560ca3357..b263d81012 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -1,5 +1,5 @@ junit_tests( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD index f9d57206b2..f56cc417db 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], dependencies = [ "util/util-routing/src/main/scala/com/twitter/util/routing", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD index f9d57206b2..f56cc417db 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD @@ -1,5 +1,5 @@ scala_library( - sources = globs("*.scala"), + sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], dependencies = [ "util/util-routing/src/main/scala/com/twitter/util/routing", From b78ba1fb2542b8c6c1dacd973cb8821727dd8cc9 Mon Sep 17 00:00:00 2001 From: Borja Lorente Date: Tue, 14 Jul 2020 17:03:38 +0000 Subject: [PATCH 337/796] [Bazel Migration] Add dependency on util-core-util to util/util-thrift/src/main/scala/com/twitter/util/ **Problem** util/util-thrift/src/main/scala/com/twitter/util/ uses class `Codec` from `util-core-util`. In pants, this was (mistakingly) pulled via transitive dependency, which was not correct. **Solution** To add a direct dependency on `util-core-util`. *Why direct?*: Since it uses the `Codec` class directly, I figured the target really wanted to depend on `util-core-util` directly, and not doing so previously was because pants not being strict enough with the exports. *Why do we export it?*: Because tests that depend on the changed target need `Codec` to be in the classpath when they compile, but they don't use it directly. Exporting seemed like the right thing to do. **Result** Compilation passes in Bazel and Pants. JIRA Issues: DPB-12610 Differential Revision: https://phabricator.twitter.biz/D516475 --- util-thrift/src/main/scala/com/twitter/util/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util-thrift/src/main/scala/com/twitter/util/BUILD b/util-thrift/src/main/scala/com/twitter/util/BUILD index 30467df3bc..ddb11f1ffe 100644 --- a/util-thrift/src/main/scala/com/twitter/util/BUILD +++ b/util-thrift/src/main/scala/com/twitter/util/BUILD @@ -9,9 +9,11 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/org/apache/thrift:libthrift", "util/util-codec/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], exports = [ "3rdparty/jvm/org/apache/thrift:libthrift", "util/util-codec/src/main/scala/com/twitter/util", + "util/util-core:util-core-util", ], ) From 23e255b3fb96716a24795b76b4a984ab961fe554 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Tue, 14 Jul 2020 22:11:08 +0000 Subject: [PATCH 338/796] [util] Simplify BUILD structures **Problem** Bazel treats `target` as `test_suite` if it points to tests, but `test_suite` must depend on other tests or test_suite, so another `alias` or `target` isn't allowed. However, some tests under `util/` are set up like ``` target( dependencies = [ "util/util-app/src/test/java/com/twitter/app", ], ) ``` but `util/util-app/src/test/java/com/twitter/app` itself is another alias (not a test). **Solution** Convert the alias in the middle to a real test target. **Result** No visible change from developer point of view. JIRA Issues: DPB-12604 Differential Revision: https://phabricator.twitter.biz/D516441 --- util-app/src/test/java/BUILD | 11 +++++- util-app/src/test/java/com/twitter/app/BUILD | 12 ------ util-app/src/test/scala/BUILD | 12 +++++- util-app/src/test/scala/com/twitter/app/BUILD | 13 ------- util-cache/src/test/java/BUILD | 10 ++++- .../src/test/java/com/twitter/cache/BUILD | 11 ------ util-jvm/src/test/java/BUILD | 7 +++- util-jvm/src/test/java/com/twitter/jvm/BUILD | 8 ---- util-jvm/src/test/scala/BUILD | 39 +++++++++++++++++-- util-jvm/src/test/scala/com/twitter/jvm/BUILD | 34 ---------------- util-logging/src/test/java/BUILD | 7 +++- .../src/test/java/com/twitter/logging/BUILD | 8 ---- util-logging/src/test/scala/BUILD | 13 ++++++- .../src/test/scala/com/twitter/logging/BUILD | 14 ------- util-registry/src/test/java/BUILD | 8 +++- .../test/java/com/twitter/util/registry/BUILD | 9 ----- util-registry/src/test/scala/BUILD | 9 ++++- .../scala/com/twitter/util/registry/BUILD | 10 ----- util-stats/src/test/java/BUILD | 8 +++- .../test/java/com/twitter/finagle/stats/BUILD | 9 ----- util-stats/src/test/scala/BUILD | 12 +++++- .../scala/com/twitter/finagle/stats/BUILD | 13 ------- util-zk-test/src/test/java/BUILD | 7 +++- .../src/test/java/com/twitter/zk/BUILD | 8 ---- util-zk-test/src/test/scala/BUILD | 10 ++++- .../src/test/scala/com/twitter/zk/BUILD | 11 ------ 26 files changed, 126 insertions(+), 187 deletions(-) delete mode 100644 util-app/src/test/java/com/twitter/app/BUILD delete mode 100644 util-app/src/test/scala/com/twitter/app/BUILD delete mode 100644 util-cache/src/test/java/com/twitter/cache/BUILD delete mode 100644 util-jvm/src/test/java/com/twitter/jvm/BUILD delete mode 100644 util-jvm/src/test/scala/com/twitter/jvm/BUILD delete mode 100644 util-logging/src/test/java/com/twitter/logging/BUILD delete mode 100644 util-logging/src/test/scala/com/twitter/logging/BUILD delete mode 100644 util-registry/src/test/java/com/twitter/util/registry/BUILD delete mode 100644 util-registry/src/test/scala/com/twitter/util/registry/BUILD delete mode 100644 util-stats/src/test/java/com/twitter/finagle/stats/BUILD delete mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/BUILD delete mode 100644 util-zk-test/src/test/java/com/twitter/zk/BUILD delete mode 100644 util-zk-test/src/test/scala/com/twitter/zk/BUILD diff --git a/util-app/src/test/java/BUILD b/util-app/src/test/java/BUILD index 6edff9df68..38de4db476 100644 --- a/util-app/src/test/java/BUILD +++ b/util-app/src/test/java/BUILD @@ -1,5 +1,12 @@ -target( +junit_tests( + sources = ["**/*.java"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-app/src/test/java/com/twitter/app", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-app/src/main/java/com/twitter/app", + "util/util-app/src/main/scala", + "util/util-core:util-core-util", + "util/util-function/src/main/java/com/twitter/function", ], ) diff --git a/util-app/src/test/java/com/twitter/app/BUILD b/util-app/src/test/java/com/twitter/app/BUILD deleted file mode 100644 index caa454b38c..0000000000 --- a/util-app/src/test/java/com/twitter/app/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -junit_tests( - sources = ["*.java"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-app/src/main/java/com/twitter/app", - "util/util-app/src/main/scala", - "util/util-core:util-core-util", - "util/util-function/src/main/java/com/twitter/function", - ], -) diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index 6e3fe3ea41..c9b7e5a5c2 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -1,5 +1,13 @@ -target( +junit_tests( + sources = ["**/*.scala"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-app/src/test/scala/com/twitter/app", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-app/src/main/scala", + "util/util-app/src/test/resources", + "util/util-core:util-core-util", + "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-app/src/test/scala/com/twitter/app/BUILD b/util-app/src/test/scala/com/twitter/app/BUILD deleted file mode 100644 index 625b5e5f3c..0000000000 --- a/util-app/src/test/scala/com/twitter/app/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -junit_tests( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-app/src/main/scala", - "util/util-app/src/test/resources", - "util/util-core:util-core-util", - "util/util-registry/src/main/scala/com/twitter/util/registry", - ], -) diff --git a/util-cache/src/test/java/BUILD b/util-cache/src/test/java/BUILD index 97107dac55..c763885764 100644 --- a/util-cache/src/test/java/BUILD +++ b/util-cache/src/test/java/BUILD @@ -1,5 +1,11 @@ -target( +junit_tests( + sources = ["**/*.java"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-cache/src/test/java/com/twitter/cache", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-cache/src/main/scala/com/twitter/cache", + "util/util-core:util-core-util", ], ) diff --git a/util-cache/src/test/java/com/twitter/cache/BUILD b/util-cache/src/test/java/com/twitter/cache/BUILD deleted file mode 100644 index 84243db2a9..0000000000 --- a/util-cache/src/test/java/com/twitter/cache/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -junit_tests( - sources = ["*.java"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-cache/src/main/scala/com/twitter/cache", - "util/util-core:util-core-util", - ], -) diff --git a/util-jvm/src/test/java/BUILD b/util-jvm/src/test/java/BUILD index 33ad7812e9..6a956db251 100644 --- a/util-jvm/src/test/java/BUILD +++ b/util-jvm/src/test/java/BUILD @@ -1,5 +1,8 @@ -target( +junit_tests( + sources = ["**/*.java"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-jvm/src/test/java/com/twitter/jvm", + "3rdparty/jvm/junit", + "util/util-jvm/src/main/scala/com/twitter/jvm", ], ) diff --git a/util-jvm/src/test/java/com/twitter/jvm/BUILD b/util-jvm/src/test/java/com/twitter/jvm/BUILD deleted file mode 100644 index 8f76149ddd..0000000000 --- a/util-jvm/src/test/java/com/twitter/jvm/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -junit_tests( - sources = ["*.java"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "util/util-jvm/src/main/scala/com/twitter/jvm", - ], -) diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index 6f1f6ef0ad..c095e7f0de 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -1,5 +1,38 @@ -target( - dependencies = [ - "util/util-jvm/src/test/scala/com/twitter/jvm", +COMMON_DEPS = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-app/src/main/scala", + "util/util-core:util-core-util", + "util/util-jvm/src/main/scala/com/twitter/jvm", + "util/util-logging/src/main/scala", + "util/util-test/src/main/scala", +] + +junit_tests( + sources = [ + "!**/EstimatorApp.scala", + "**/*.scala", ], + compiler_option_sets = ["fatal_warnings"], + dependencies = COMMON_DEPS, +) + +scala_library( + name = "estimator_app", + # Overlap the sources (except EstimatorApp.scala) with the above target for the following reasons: + # 1) We cannot extract a common `scala_library` because Pants does not like `junit_tests` + # depending on a `scala_library` containing the test sources while the `junit_test` + # itself has empty sources. + # 2) Bazel strictly forbids `scala_library` to depend back onto a test target, whereas Pants + # does not strictly enforce it. + sources = ["**/*.scala"], + compiler_option_sets = ["fatal_warnings"], + dependencies = COMMON_DEPS, +) + +jvm_binary( + name = "bin", + main = "com.twitter.jvm.EstimatorApp", + dependencies = [":estimator_app"], ) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/BUILD b/util-jvm/src/test/scala/com/twitter/jvm/BUILD deleted file mode 100644 index 87df917b93..0000000000 --- a/util-jvm/src/test/scala/com/twitter/jvm/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -junit_tests( - sources = [ - "!EstimatorApp.scala", - "*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-app/src/main/scala", - "util/util-core:util-core-util", - "util/util-jvm/src/main/scala/com/twitter/jvm", - "util/util-logging/src/main/scala", - "util/util-test/src/main/scala", - ], -) - -scala_library( - name = "estimator_app", - sources = ["EstimatorApp.scala"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "util/util-jvm/src/main/scala/com/twitter/jvm", - ], -) - -jvm_binary( - name = "bin", - main = "com.twitter.jvm.EstimatorApp", - dependencies = [ - ":estimator_app", - ], -) diff --git a/util-logging/src/test/java/BUILD b/util-logging/src/test/java/BUILD index 076e509f5c..576ae96b09 100644 --- a/util-logging/src/test/java/BUILD +++ b/util-logging/src/test/java/BUILD @@ -1,5 +1,8 @@ -target( +junit_tests( + sources = ["**/*.java"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-logging/src/test/java/com/twitter/logging", + "3rdparty/jvm/junit", + "util/util-logging/src/main/scala/com/twitter/logging", ], ) diff --git a/util-logging/src/test/java/com/twitter/logging/BUILD b/util-logging/src/test/java/com/twitter/logging/BUILD deleted file mode 100644 index 7c8d12286a..0000000000 --- a/util-logging/src/test/java/com/twitter/logging/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -junit_tests( - sources = ["*.java"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "util/util-logging/src/main/scala/com/twitter/logging", - ], -) diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 822a12ed7f..0cada470f1 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -1,5 +1,14 @@ -target( +junit_tests( + sources = ["**/*.scala"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-logging/src/test/scala/com/twitter/logging", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-app/src/main/scala", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-core:util-core-util", + "util/util-logging/src/main/scala/com/twitter/logging", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) diff --git a/util-logging/src/test/scala/com/twitter/logging/BUILD b/util-logging/src/test/scala/com/twitter/logging/BUILD deleted file mode 100644 index 20e0ad38c4..0000000000 --- a/util-logging/src/test/scala/com/twitter/logging/BUILD +++ /dev/null @@ -1,14 +0,0 @@ -junit_tests( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-app/src/main/scala", - "util/util-core/src/main/scala/com/twitter/io", - "util/util-core:util-core-util", - "util/util-logging/src/main/scala/com/twitter/logging", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/util-registry/src/test/java/BUILD b/util-registry/src/test/java/BUILD index 92f3a26ae8..ff4ef5597c 100644 --- a/util-registry/src/test/java/BUILD +++ b/util-registry/src/test/java/BUILD @@ -1,5 +1,9 @@ -target( +junit_tests( + sources = ["**/*.java"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-registry/src/test/java/com/twitter/util/registry", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-registry/src/test/java/com/twitter/util/registry/BUILD b/util-registry/src/test/java/com/twitter/util/registry/BUILD deleted file mode 100644 index a9142d0144..0000000000 --- a/util-registry/src/test/java/com/twitter/util/registry/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -junit_tests( - sources = ["*.java"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scala-lang:scala-library", - "util/util-registry/src/main/scala/com/twitter/util/registry", - ], -) diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index 03e8040f41..6ed8c18ee6 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -1,5 +1,10 @@ -target( +junit_tests( + sources = ["**/*.scala"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-registry/src/test/scala/com/twitter/util/registry", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalatest", + "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-registry/src/test/scala/com/twitter/util/registry/BUILD b/util-registry/src/test/scala/com/twitter/util/registry/BUILD deleted file mode 100644 index df58be17b1..0000000000 --- a/util-registry/src/test/scala/com/twitter/util/registry/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -junit_tests( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalatest", - "util/util-registry/src/main/scala/com/twitter/util/registry", - ], -) diff --git a/util-stats/src/test/java/BUILD b/util-stats/src/test/java/BUILD index ee3927bab5..d2052ffd1c 100644 --- a/util-stats/src/test/java/BUILD +++ b/util-stats/src/test/java/BUILD @@ -1,5 +1,9 @@ -target( +junit_tests( + sources = ["**/*.java"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-stats/src/test/java/com/twitter/finagle/stats", + "3rdparty/jvm/junit", + "util/util-core:util-core-util", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/BUILD b/util-stats/src/test/java/com/twitter/finagle/stats/BUILD deleted file mode 100644 index e01b10ddd7..0000000000 --- a/util-stats/src/test/java/com/twitter/finagle/stats/BUILD +++ /dev/null @@ -1,9 +0,0 @@ -junit_tests( - sources = ["*.java"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "util/util-core:util-core-util", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/util-stats/src/test/scala/BUILD b/util-stats/src/test/scala/BUILD index 9071fa8e03..9a2d862cbf 100644 --- a/util-stats/src/test/scala/BUILD +++ b/util-stats/src/test/scala/BUILD @@ -1,5 +1,13 @@ -target( +junit_tests( + sources = ["**/*.scala"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-stats/src/test/scala/com/twitter/finagle/stats", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core:util-core-util", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD deleted file mode 100644 index 2116031fbb..0000000000 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -junit_tests( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", - "3rdparty/jvm/org/scalacheck", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core:util-core-util", - "util/util-stats/src/main/scala/com/twitter/finagle/stats", - ], -) diff --git a/util-zk-test/src/test/java/BUILD b/util-zk-test/src/test/java/BUILD index ceef82a3ff..6e41b20b16 100644 --- a/util-zk-test/src/test/java/BUILD +++ b/util-zk-test/src/test/java/BUILD @@ -1,5 +1,8 @@ -target( +junit_tests( + sources = ["**/*.java"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-zk-test/src/test/java/com/twitter/zk", + "3rdparty/jvm/junit", + "util/util-zk-test/src/main/scala/com/twitter/zk", ], ) diff --git a/util-zk-test/src/test/java/com/twitter/zk/BUILD b/util-zk-test/src/test/java/com/twitter/zk/BUILD deleted file mode 100644 index ac7f94a3bf..0000000000 --- a/util-zk-test/src/test/java/com/twitter/zk/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -junit_tests( - sources = ["*.java"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "util/util-zk-test/src/main/scala/com/twitter/zk", - ], -) diff --git a/util-zk-test/src/test/scala/BUILD b/util-zk-test/src/test/scala/BUILD index 77c3ee4906..6c9149bfa3 100644 --- a/util-zk-test/src/test/scala/BUILD +++ b/util-zk-test/src/test/scala/BUILD @@ -1,5 +1,11 @@ -target( +junit_tests( + sources = ["**/*.scala"], + compiler_option_sets = ["fatal_warnings"], dependencies = [ - "util/util-zk-test/src/test/scala/com/twitter/zk", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", + "3rdparty/jvm/org/scalatest", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-zk-test/src/main/scala/com/twitter/zk", ], ) diff --git a/util-zk-test/src/test/scala/com/twitter/zk/BUILD b/util-zk-test/src/test/scala/com/twitter/zk/BUILD deleted file mode 100644 index b3b809ca20..0000000000 --- a/util-zk-test/src/test/scala/com/twitter/zk/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -junit_tests( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", - "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/io", - "util/util-zk-test/src/main/scala/com/twitter/zk", - ], -) From d5f30938eb47a89bdb63ba24303f36776f171272 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Tue, 14 Jul 2020 23:57:04 +0000 Subject: [PATCH 339/796] [util] Fix a missed test Differential Revision: https://phabricator.twitter.biz/D517171 --- .../com/twitter/finagle/stats/StatsReceiverCompilationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java index 092ca8ec76..f06579c87b 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java @@ -63,6 +63,7 @@ public void testCounterMethods() { counter.incr(100); } + @Test public void testScope() { InMemoryStatsReceiver sr = new InMemoryStatsReceiver(); StatsReceiver sr2 = sr.scope(); From 336a878c31a7692f0132a4009187e1adbe864c17 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 15 Jul 2020 01:36:21 +0000 Subject: [PATCH 340/796] [util] add scala_xml dep explicitly JIRA Issues: DPB-12589 Differential Revision: https://phabricator.twitter.biz/D517238 --- .../src/test/scala/com/twitter/util/capturepoints/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD index 6c621f2dca..833f28ea0b 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD @@ -3,6 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], strict_deps = True, dependencies = [ + "3rdparty/jvm/org/scala-lang/modules:scala-xml", "3rdparty/jvm/org/scalatest", "util/util-core:util-core-util", "util/util-intellij/src/main/resources", From 359473b765c794ead2e874b8965c84b243c45aee Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Thu, 16 Jul 2020 21:42:07 +0000 Subject: [PATCH 341/796] dodo et al.: Update Projects to use Scala 2.12.11 Problem The Scala 2.12 version used for CSL's open source projects has diverged from that used internally at Twitter via pants. Solution Update all CSL open source projects to use Scala 2.12.11 to match the recently updated internal version. JIRA Issues: CSL-9990 Differential Revision: https://phabricator.twitter.biz/D518430 --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index eab87f4f06..445c5b551f 100644 --- a/build.sbt +++ b/build.sbt @@ -75,8 +75,8 @@ def jdk11GcJavaOptions: Seq[String] = { } val defaultProjectSettings = Seq( - scalaVersion := "2.12.8", - crossScalaVersions := Seq("2.11.12", "2.12.8", "2.13.1") + scalaVersion := "2.12.11", + crossScalaVersions := Seq("2.11.12", "2.12.11", "2.13.1") ) val baseSettings = Seq( From 0976b23edcfeb18a5a4c496e71243930338573a9 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 17 Jul 2020 01:20:53 +0000 Subject: [PATCH 342/796] 3rdparty: Upgrade scalatest to 3.1.2 Problem / Solution We need to upgrade scalatest, so that we can stay up to date, take advantage of new features, and test our libraries against dotty. This is a more involved upgrade than normal. A lot of old names have been removed, and a bunch of namespaces were moved into new modules, and so we've had to move a lot of things around. TBR=true NO-QUEUE=true NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D518553 --- build.sbt | 30 ++++++++++++++----- .../scala/com/twitter/app/lifecycle/BUILD | 1 + util-app/src/test/java/BUILD | 1 + util-app/src/test/scala/BUILD | 2 ++ .../test/java/com/twitter/cache/guava/BUILD | 1 + .../test/scala/com/twitter/cache/guava/BUILD | 1 + util-cache/src/test/java/BUILD | 1 + .../src/test/scala/com/twitter/cache/BUILD | 4 ++- .../scala/com/twitter/cache/caffeine/BUILD | 2 ++ .../src/test/scala/com/twitter/util/BUILD | 1 + .../test/java/com/twitter/concurrent/BUILD | 1 + util-core/src/test/java/com/twitter/io/BUILD | 1 + .../src/test/java/com/twitter/service/BUILD | 1 + .../src/test/java/com/twitter/util/BUILD | 3 +- .../java/com/twitter/util/javainterop/BUILD | 1 + .../test/scala/com/twitter/concurrent/BUILD | 3 ++ .../test/scala/com/twitter/conversions/BUILD | 2 ++ util-core/src/test/scala/com/twitter/io/BUILD | 3 ++ .../src/test/scala/com/twitter/util/BUILD | 3 ++ .../src/test/scala/com/twitter/hashing/BUILD | 2 ++ .../com/twitter/util/capturepoints/BUILD | 1 + util-jvm/src/test/java/BUILD | 1 + util-jvm/src/test/scala/BUILD | 2 +- .../test/scala/com/twitter/util/lint/BUILD | 2 ++ util-logging/src/test/java/BUILD | 1 + util-logging/src/test/scala/BUILD | 3 +- .../test/scala/com/twitter/util/reflect/BUILD | 1 + util-registry/src/test/java/BUILD | 1 + util-registry/src/test/scala/BUILD | 2 ++ .../test/java/com/twitter/util/routing/BUILD | 1 + .../test/scala/com/twitter/util/routing/BUILD | 2 ++ .../test/java/com/twitter/util/security/BUILD | 1 + .../scala/com/twitter/util/security/BUILD | 2 ++ .../test/java/com/twitter/util/logging/BUILD | 1 + .../test/scala/com/twitter/util/logging/BUILD | 2 ++ util-stats/src/test/java/BUILD | 1 + util-stats/src/test/scala/BUILD | 4 ++- .../main/scala/com/twitter/util/testing/BUILD | 1 + .../test/scala/com/twitter/util/testing/BUILD | 2 ++ .../src/test/scala/com/twitter/util/BUILD | 1 + .../test/java/com/twitter/util/tunable/BUILD | 1 + .../test/scala/com/twitter/util/tunable/BUILD | 1 + util-zk-test/src/test/java/BUILD | 1 + util-zk-test/src/test/scala/BUILD | 1 + util-zk/src/test/scala/com/twitter/zk/BUILD | 4 ++- .../scala/com/twitter/zk/coordination/BUILD | 4 ++- .../coordination/ZkAsyncSemaphoreTest.scala | 4 +-- 47 files changed, 95 insertions(+), 17 deletions(-) diff --git a/build.sbt b/build.sbt index 445c5b551f..9cde60fc9e 100644 --- a/build.sbt +++ b/build.sbt @@ -17,7 +17,7 @@ val jacksonVersion = "2.11.0" val guavaLib = "com.google.guava" % "guava" % "25.1-jre" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.5" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" -val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.0" % "test" +val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.3" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion def travisTestJavaOptions: Seq[String] = { @@ -89,7 +89,9 @@ val baseSettings = Seq( // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.10.19" % "test", - "org.scalatest" %% "scalatest" % "3.0.8" % "test" + "org.scalatest" %% "scalatest" % "3.1.1" % "test", + "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test", + "org.scalatestplus" %% "mockito_1-10" % "3.1.0.0" % "test" ), fork in Test := true, // We have to fork to get the JavaOptions // Workaround for cross building HealthyQueue.scala, which is not compatible between @@ -293,7 +295,8 @@ lazy val utilCore = Project( caffeineLib % "test", scalacheckLib, "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2" + "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2", + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ), resourceGenerators in Compile += Def.task { val projectName = name.value @@ -356,7 +359,10 @@ lazy val utilHashing = Project( sharedSettings ).settings( name := "util-hashing", - libraryDependencies += scalacheckLib + libraryDependencies ++= Seq( + scalacheckLib, + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" + ) ).dependsOn(utilCore % "test") lazy val utilIntellij = Project( @@ -436,7 +442,8 @@ lazy val utilSecurity = Project( ).settings( name := "util-security", libraryDependencies ++= Seq( - scalacheckLib + scalacheckLib, + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ) ).dependsOn(utilCore, utilLogging) @@ -447,7 +454,12 @@ lazy val utilStats = Project( sharedSettings ).settings( name := "util-stats", - libraryDependencies ++= Seq(caffeineLib, jsr305Lib, scalacheckLib) ++ { + libraryDependencies ++= Seq( + caffeineLib, + jsr305Lib, + scalacheckLib, + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" + ) ++ { CrossVersion.partialVersion(scalaVersion.value) match { case Some((2, major)) if major >= 13 => Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "0.2.0" % "test") @@ -474,8 +486,10 @@ lazy val utilTest = Project( ).settings( name := "util-test", libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % "3.0.0", - "org.mockito" % "mockito-all" % "1.10.19" + "org.mockito" % "mockito-all" % "1.10.19", + "org.scalatest" %% "scalatest" % "3.1.1", + "org.scalatestplus" %% "junit-4-12" % "3.1.2.0", + "org.scalatestplus" %% "mockito_1-10" % "3.1.1.0" ) ).dependsOn(utilCore, utilLogging) diff --git a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD index adb6046d63..f1ca84a0c1 100644 --- a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD +++ b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle", "util/util-core:util-core-util", ], diff --git a/util-app/src/test/java/BUILD b/util-app/src/test/java/BUILD index 38de4db476..73cb1a6444 100644 --- a/util-app/src/test/java/BUILD +++ b/util-app/src/test/java/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-app/src/main/java/com/twitter/app", "util/util-app/src/main/scala", "util/util-core:util-core-util", diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index c9b7e5a5c2..cfd289bc65 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -5,6 +5,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-app/src/main/scala", "util/util-app/src/test/resources", "util/util-core:util-core-util", diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD index 508db84ad5..204ec7fa8b 100644 --- a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD @@ -5,6 +5,7 @@ junit_tests( "3rdparty/jvm/com/google/guava", "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-cache-guava/src/main/scala/com/twitter/cache/guava", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-core:util-core-util", diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD index 49a911f633..8c27750fe4 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD @@ -6,6 +6,7 @@ junit_tests( "3rdparty/jvm/com/google/guava", "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-cache-guava/src/main/scala/com/twitter/cache/guava", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", diff --git a/util-cache/src/test/java/BUILD b/util-cache/src/test/java/BUILD index c763885764..5a0ea76ebc 100644 --- a/util-cache/src/test/java/BUILD +++ b/util-cache/src/test/java/BUILD @@ -5,6 +5,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-core:util-core-util", ], diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index 3fec9f1e3d..34fdee3ea2 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -11,6 +11,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-core:util-core-util", @@ -28,7 +30,7 @@ scala_library( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", "util/util-cache/src/main/scala/com/twitter/cache", - "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", ], ) diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index c34ffeecc7..851da83281 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -7,6 +7,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", diff --git a/util-codec/src/test/scala/com/twitter/util/BUILD b/util-codec/src/test/scala/com/twitter/util/BUILD index 6eebe20b6a..9537a62c5a 100644 --- a/util-codec/src/test/scala/com/twitter/util/BUILD +++ b/util-codec/src/test/scala/com/twitter/util/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-codec/src/main/scala/com/twitter/util", ], ) diff --git a/util-core/src/test/java/com/twitter/concurrent/BUILD b/util-core/src/test/java/com/twitter/concurrent/BUILD index f561361353..9ff36924bc 100644 --- a/util-core/src/test/java/com/twitter/concurrent/BUILD +++ b/util-core/src/test/java/com/twitter/concurrent/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core/src/main/java/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/concurrent", ], diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD index ca525eb773..e14e5536d5 100644 --- a/util-core/src/test/java/com/twitter/io/BUILD +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:scala", "util/util-core/src/main/java/com/twitter/io", ], diff --git a/util-core/src/test/java/com/twitter/service/BUILD b/util-core/src/test/java/com/twitter/service/BUILD index d7702ba8a7..03c081ebc3 100644 --- a/util-core/src/test/java/com/twitter/service/BUILD +++ b/util-core/src/test/java/com/twitter/service/BUILD @@ -3,6 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core/src/main/java/com/twitter/service", ], ) diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index 7732db3b19..632c62cb01 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -4,7 +4,7 @@ java_library( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", - "3rdparty/jvm/com/google/errorprone:error_prone_annotations" + "3rdparty/jvm/com/google/errorprone:error_prone_annotations", ], ) @@ -19,6 +19,7 @@ junit_tests( "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", "util/util-core/src/main/java/com/twitter/util", ], diff --git a/util-core/src/test/java/com/twitter/util/javainterop/BUILD b/util-core/src/test/java/com/twitter/util/javainterop/BUILD index 06308b494c..7c178fa76d 100644 --- a/util-core/src/test/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/test/java/com/twitter/util/javainterop/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core/src/main/java/com/twitter/util/javainterop", ], ) diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index 8ca36f4040..b60d3d986d 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -6,6 +6,9 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD index 4166c9ab67..591c088df5 100644 --- a/util-core/src/test/scala/com/twitter/conversions/BUILD +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -5,6 +5,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", ], diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 897e3d06d1..6d1367e4f4 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -6,6 +6,9 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index 07a33c9023..56199e4890 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -6,6 +6,9 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/test/java/com/twitter/util:object_size_calculator", diff --git a/util-hashing/src/test/scala/com/twitter/hashing/BUILD b/util-hashing/src/test/scala/com/twitter/hashing/BUILD index 3c75b82c7b..45f47eb8b6 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/test/scala/com/twitter/hashing/BUILD @@ -5,6 +5,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core/src/main/scala/com/twitter/io", "util/util-hashing/src/main/scala/com/twitter/hashing", "util/util-hashing/src/test/resources", diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD index 833f28ea0b..2632ef1de6 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD @@ -5,6 +5,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/org/scala-lang/modules:scala-xml", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", "util/util-intellij/src/main/resources", ], diff --git a/util-jvm/src/test/java/BUILD b/util-jvm/src/test/java/BUILD index 6a956db251..9ecd1b3da6 100644 --- a/util-jvm/src/test/java/BUILD +++ b/util-jvm/src/test/java/BUILD @@ -3,6 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-jvm/src/main/scala/com/twitter/jvm", ], ) diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index c095e7f0de..3b5682f28b 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -15,7 +15,7 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], - dependencies = COMMON_DEPS, + dependencies = COMMON_DEPS + ["3rdparty/jvm/org/scalatestplus:junit"], ) scala_library( diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index d0829af8a8..6dae7fa1ff 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -5,6 +5,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-lint/src/main/scala/com/twitter/util/lint", ], ) diff --git a/util-logging/src/test/java/BUILD b/util-logging/src/test/java/BUILD index 576ae96b09..460923374e 100644 --- a/util-logging/src/test/java/BUILD +++ b/util-logging/src/test/java/BUILD @@ -3,6 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-logging/src/main/scala/com/twitter/logging", ], ) diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 0cada470f1..479b5b021f 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -5,9 +5,10 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-app/src/main/scala", - "util/util-core/src/main/scala/com/twitter/io", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/io", "util/util-logging/src/main/scala/com/twitter/logging", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD index bdf3e30aca..d9a7039540 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", "util/util-reflect/src/main/scala/com/twitter/util/reflect", ], diff --git a/util-registry/src/test/java/BUILD b/util-registry/src/test/java/BUILD index ff4ef5597c..441bf008aa 100644 --- a/util-registry/src/test/java/BUILD +++ b/util-registry/src/test/java/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index 6ed8c18ee6..d012dfc789 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -5,6 +5,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-routing/src/test/java/com/twitter/util/routing/BUILD b/util-routing/src/test/java/com/twitter/util/routing/BUILD index 9e7471628b..2bd7dc18a3 100644 --- a/util-routing/src/test/java/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/java/com/twitter/util/routing/BUILD @@ -6,6 +6,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", "util/util-logging/src/main/scala", "util/util-routing/src/main/scala/com/twitter/util/routing", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index b263d81012..58314f4931 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -5,6 +5,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-routing/src/main/scala/com/twitter/util/routing", "util/util-routing/src/test/scala/com/twitter/util/routing/dynamic", "util/util-routing/src/test/scala/com/twitter/util/routing/simple", diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD index 55596161b8..4e3f011d4d 100644 --- a/util-security/src/test/java/com/twitter/util/security/BUILD +++ b/util-security/src/test/java/com/twitter/util/security/BUILD @@ -3,6 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", "util/util-security/src/main/scala/com/twitter/util/security", ], diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index e3022418cd..3606cd0060 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -5,6 +5,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/io", "util/util-security/src/main/scala/com/twitter/util/security", diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD index 07b1519143..72acca2533 100644 --- a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD @@ -3,6 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-simple", "util/util-slf4j-api/src/main/scala", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index 565d7b52df..e2c66a901f 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -7,6 +7,8 @@ junit_tests( dependencies = [ "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-simple", "util/util-core/src/main/scala/com/twitter/io", diff --git a/util-stats/src/test/java/BUILD b/util-stats/src/test/java/BUILD index d2052ffd1c..13a4fba0c9 100644 --- a/util-stats/src/test/java/BUILD +++ b/util-stats/src/test/java/BUILD @@ -3,6 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], diff --git a/util-stats/src/test/scala/BUILD b/util-stats/src/test/scala/BUILD index 9a2d862cbf..d7261205f1 100644 --- a/util-stats/src/test/scala/BUILD +++ b/util-stats/src/test/scala/BUILD @@ -6,8 +6,10 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/conversions", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) diff --git a/util-test/src/main/scala/com/twitter/util/testing/BUILD b/util-test/src/main/scala/com/twitter/util/testing/BUILD index eef5aca25d..06e0d253b5 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/main/scala/com/twitter/util/testing/BUILD @@ -8,6 +8,7 @@ scala_library( dependencies = [ "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", ], exports = [ "3rdparty/jvm/org/scalatest", diff --git a/util-test/src/test/scala/com/twitter/util/testing/BUILD b/util-test/src/test/scala/com/twitter/util/testing/BUILD index f73116776c..4846f80967 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/test/scala/com/twitter/util/testing/BUILD @@ -5,6 +5,8 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-test/src/main/scala/com/twitter/util/testing", ], ) diff --git a/util-thrift/src/test/scala/com/twitter/util/BUILD b/util-thrift/src/test/scala/com/twitter/util/BUILD index 7824a410fe..c5543e819f 100644 --- a/util-thrift/src/test/scala/com/twitter/util/BUILD +++ b/util-thrift/src/test/scala/com/twitter/util/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-thrift/src/main/scala/com/twitter/util", "util/util-thrift/src/test/java/com/twitter/util", ], diff --git a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD index 7ed727c998..e22239aaf7 100644 --- a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD @@ -4,6 +4,7 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-tunable/src/main/scala/com/twitter/util/tunable", ], ) diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD index f6d67c888e..7e3c81ecad 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -8,6 +8,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", "util/util-tunable/src/main/scala/com/twitter/util/tunable", "util/util-tunable/src/test/resources", diff --git a/util-zk-test/src/test/java/BUILD b/util-zk-test/src/test/java/BUILD index 6e41b20b16..dc52d1bd40 100644 --- a/util-zk-test/src/test/java/BUILD +++ b/util-zk-test/src/test/java/BUILD @@ -3,6 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-zk-test/src/main/scala/com/twitter/zk", ], ) diff --git a/util-zk-test/src/test/scala/BUILD b/util-zk-test/src/test/scala/BUILD index 6c9149bfa3..e0111f0e8e 100644 --- a/util-zk-test/src/test/scala/BUILD +++ b/util-zk-test/src/test/scala/BUILD @@ -5,6 +5,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", "util/util-core/src/main/scala/com/twitter/io", "util/util-zk-test/src/main/scala/com/twitter/zk", ], diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index b233b1d006..5314eebd64 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -6,8 +6,10 @@ junit_tests( "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/conversions", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", "util/util-logging/src/main/scala/com/twitter/logging", "util/util-zk/src/main/scala/com/twitter/zk", ], diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index ebe6b6fc15..35c70bac58 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -6,8 +6,10 @@ junit_tests( "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", - "util/util-core/src/main/scala/com/twitter/conversions", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", "util/util-zk/src/main/scala/com/twitter/zk", "util/util-zk/src/main/scala/com/twitter/zk/coordination", ], diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala index 54b207c643..591dccbbaf 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala @@ -8,13 +8,13 @@ import java.util.concurrent.ConcurrentLinkedQueue import org.apache.zookeeper.KeeperException.NoNodeException import org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE import org.scalatest.WordSpec -import org.scalatest.concurrent.AsyncAssertions +import org.scalatest.concurrent.Waiters import org.scalatest.concurrent.Eventually._ import org.scalatest.time.{Millis, Seconds, Span} import org.scalatestplus.mockito.MockitoSugar import scala.jdk.CollectionConverters._ -class ZkAsyncSemaphoreTest extends WordSpec with MockitoSugar with AsyncAssertions { +class ZkAsyncSemaphoreTest extends WordSpec with MockitoSugar with Waiters { "ZkAsyncSemaphore" should { From db63a6dcf2113606f27e04d6860ad2d0ca72ab5e Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 17 Jul 2020 18:04:22 +0000 Subject: [PATCH 343/796] 3rdparty: Actually upgrade scalatest to 3.1.2 Problem In the previous commit where I claimed to have upgraded to 3.1.2, I actually upgraded to 3.1.1. I also broke the sbt build :grimacing: Solution Let's actually move to 3.1.2. We aren't moving to 3.2.0 yet because of the breaking API changes related to suite names. JIRA Issues: CSL-9722 Differential Revision: https://phabricator.twitter.biz/D518794 --- build.sbt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 9cde60fc9e..14636a1457 100644 --- a/build.sbt +++ b/build.sbt @@ -89,9 +89,9 @@ val baseSettings = Seq( // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", "org.mockito" % "mockito-all" % "1.10.19" % "test", - "org.scalatest" %% "scalatest" % "3.1.1" % "test", + "org.scalatest" %% "scalatest" % "3.1.2" % "test", "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test", - "org.scalatestplus" %% "mockito_1-10" % "3.1.0.0" % "test" + "org.scalatestplus" %% "mockito-1-10" % "3.1.0.0" % "test" ), fork in Test := true, // We have to fork to get the JavaOptions // Workaround for cross building HealthyQueue.scala, which is not compatible between @@ -487,9 +487,9 @@ lazy val utilTest = Project( name := "util-test", libraryDependencies ++= Seq( "org.mockito" % "mockito-all" % "1.10.19", - "org.scalatest" %% "scalatest" % "3.1.1", + "org.scalatest" %% "scalatest" % "3.1.2", "org.scalatestplus" %% "junit-4-12" % "3.1.2.0", - "org.scalatestplus" %% "mockito_1-10" % "3.1.1.0" + "org.scalatestplus" %% "mockito-1-10" % "3.1.0.0" ) ).dependsOn(utilCore, utilLogging) From 12cb9c2619c09f6238c3f251497bd5552c6865e2 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 21 Jul 2020 18:07:20 +0000 Subject: [PATCH 344/796] twitter-oss: Prepare OSS libraries for release 20.7.0 Problem We want to release the next versions of our Twitter OSS libraries 20.7.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-9977 Differential Revision: https://phabricator.twitter.biz/D519631 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0057da1b9b..17c0cf39f9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.7.0 +------ + New Features ~~~~~~~~~~~~ * util-app: Add ability to observe App lifecycle events. ``PHAB_ID=D508145`` diff --git a/README.md b/README.md index 02bcc38a0d..d66b6476e0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=master)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.6.0" +val utilCore = "com.twitter" %% "util-core" % "20.7.0" ``` # Units diff --git a/build.sbt b/build.sbt index 14636a1457..4fde31015a 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.7.0-SNAPSHOT" +val releaseVersion = "20.7.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 13b4f16bc7d844a3f94715ae76acd5bd373dcb7e Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Tue, 21 Jul 2020 19:14:43 +0000 Subject: [PATCH 345/796] twitter-oss: Update OSS Libraries Post-Release to 20.8.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.8.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-9977 Differential Revision: https://phabricator.twitter.biz/D519672 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d66b6476e0..6c50472674 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=master)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 4fde31015a..8d6f87e5be 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.7.0" +val releaseVersion = "20.8.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 43f83d2250af2f47425825175bd49e563a79fa18 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 21 Jul 2020 20:28:13 +0000 Subject: [PATCH 346/796] util-app: Comma-separated Flaggables now parse empty strings as empty collections Problem Comma-separated Flaggables (for `Seq`, `Set`, `java.util.List`, and `java.util.Set`) parse empty strings as collections of a single element - an empty string. This breaks Finatra's flag-converters, making it impossible to inject (as in type-convert) an empty collection as a default flag value (very popular use-case). Solution Treat empty strings as empty collections in comma-separated Flaggables. JIRA Issues: CSL-9983 Differential Revision: https://phabricator.twitter.biz/D516724 --- CHANGELOG.rst | 6 ++++++ util-app/src/main/scala/com/twitter/app/Flaggable.scala | 8 ++++++-- .../src/test/scala/com/twitter/app/FlaggableTest.scala | 6 ++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 17c0cf39f9..530f87732b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ +* util-app: Treat empty strings as empty collections in `Flag[Seq[_]]`, `Flag[Set[_]]`, + `Flag[java.util.List[_]]`, and `Flag[java.util.Set[_]]`. They were treated as collections + with single element, empty string, before. ``PHAB_ID=D516724`` + 20.7.0 ------ diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index f210e06181..7eb566fe36 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -169,14 +169,18 @@ object Flaggable { private[app] class SetFlaggable[T: Flaggable] extends Flaggable[Set[T]] { private val flag = implicitly[Flaggable[T]] assert(flag.default.isEmpty) - override def parse(v: String): Set[T] = v.split(",").iterator.map(flag.parse(_)).toSet + override def parse(v: String): Set[T] = + if (v.isEmpty) Set.empty[T] + else v.split(",").iterator.map(flag.parse(_)).toSet override def show(set: Set[T]): String = set.map(flag.show).mkString(",") } private[app] class SeqFlaggable[T: Flaggable] extends Flaggable[Seq[T]] { private val flag = implicitly[Flaggable[T]] assert(flag.default.isEmpty) - def parse(v: String): Seq[T] = v.split(",").toSeq.map(flag.parse) + def parse(v: String): Seq[T] = + if (v.isEmpty) Seq.empty[T] + else v.split(",").toSeq.map(flag.parse) override def show(seq: Seq[T]): String = seq.map(flag.show).mkString(",") } diff --git a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala index 3d166b3b1e..43b6616828 100644 --- a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala @@ -47,6 +47,12 @@ class FlaggableTest extends FunSuite { test("Flaggable: parse seqs") { assert(Flaggable.ofSeq[Int].parse("1,2,3,4") == Seq(1, 2, 3, 4)) + assert(Flaggable.ofSeq[Int].parse("") == Seq.empty[Int]) + } + + test("Flaggable: parse sets") { + assert(Flaggable.ofSet[Int].parse("1,2,3,4") == Set(1, 2, 3, 4)) + assert(Flaggable.ofSet[Int].parse("") == Set.empty[Int]) } test("Flaggable: parse maps") { From 2430cbc43db33fdcb01a14dee1076da68ecb9d2e Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 22 Jul 2020 01:47:20 +0000 Subject: [PATCH 347/796] util-core: Rework Monitors Internals for Unidoc Generation Problem / Solution Trying to generate `unidoc` documentation for util fails when using Scala 2.12.11. This is due to it not liking the `it` value of `Monitors` which has type `Monitor$`. To work around the problem, we stepped the version of Scala that we use go generate documentation down to 2.12.8. However, it's not necessary for `Monitors` to have this value at all, and so removing it is in our best interests moving forward. JIRA Issues: CSL-9977 Differential Revision: https://phabricator.twitter.biz/D520046 --- .../main/java/com/twitter/util/Monitors.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/util-core/src/main/java/com/twitter/util/Monitors.java b/util-core/src/main/java/com/twitter/util/Monitors.java index d8887d75c2..8022586fdc 100644 --- a/util-core/src/main/java/com/twitter/util/Monitors.java +++ b/util-core/src/main/java/com/twitter/util/Monitors.java @@ -14,56 +14,53 @@ public final class Monitors { private Monitors() { } - private static final Monitor$ it = - Monitor$.MODULE$; - /** * The singleton `Monitor` instance that is the Monitor companion object. */ public static Monitor instance() { - return it; + return Monitor$.MODULE$; } /** * Equivalent to calling `Monitor.get`. */ public static Monitor get() { - return it.get(); + return Monitor$.MODULE$.get(); } /** * Equivalent to calling `Monitor.set`. */ public static void set(Monitor monitor) { - it.set(monitor); + Monitor$.MODULE$.set(monitor); } /** * Equivalent to calling `Monitor.using`. */ public static T using(Monitor monitor, Supplier supplier) { - return it.using(monitor, func0(supplier)); + return Monitor$.MODULE$.using(monitor, func0(supplier)); } /** * Equivalent to calling `Monitor.restoring`. */ public static T restoring(Supplier supplier) { - return it.restoring(func0(supplier)); + return Monitor$.MODULE$.restoring(func0(supplier)); } /** * Equivalent to `Monitor.catcher`. */ public static PartialFunction catcher() { - return it.catcher(); + return Monitor$.MODULE$.catcher(); } /** * Equivalent to calling `Monitor.isActive`. */ public static boolean isActive() { - return it.isActive(); + return Monitor$.MODULE$.isActive(); } } From ee1cebcf44897cdd4df87c11ebc752fd9d583e07 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 22 Jul 2020 17:33:53 +0000 Subject: [PATCH 348/796] util-core: Make Var.Observer public Problem The Var.Observer class is private[util], but a part of the abstract `Var.observe` method which is intended to be public. This can be seen through the `AbstractVar` class which is supposed to be an API for Java users to make their own Var definitions. Solution Make it public (and final) so that it can be used. Differential Revision: https://phabricator.twitter.biz/D520237 --- CHANGELOG.rst | 5 +++++ util-core/src/main/scala/com/twitter/util/Var.scala | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 530f87732b..e731f47098 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ +* util-core: c.t.u.Var.Observer is now public. This allows scala users to extend the Var trait + as has been the case for Java users. ``PHAB_ID=D520237`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ * util-app: Treat empty strings as empty collections in `Flag[Seq[_]]`, `Flag[Set[_]]`, diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 16ba95b88d..3968ef43ab 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -183,7 +183,7 @@ object Var { * A Var observer. Observers are owned by exactly one producer, * enforced by a leasing mechanism. */ - private[util] class Observer[-T](observe: T => Unit) { + final class Observer[-T](observe: T => Unit) { private[this] var thisOwner: AnyRef = null private[this] var thisVersion = Long.MinValue @@ -214,7 +214,7 @@ object Var { } } - private[util] object Observer { + object Observer { def apply[T](k: T => Unit): Observer[T] = new Observer(k) } From 2aa8179f5e568ad063d183c3c04daec8726b2e11 Mon Sep 17 00:00:00 2001 From: Kevin Oliver Date: Thu, 23 Jul 2020 15:49:10 +0000 Subject: [PATCH 349/796] Have koliver own fewer projects Problem I am an owner for a bunch of projects, some of which no longer spark joy. Solution Kondo myself from them. Differential Revision: https://phabricator.twitter.biz/D520454 --- util-codec/OWNERS | 1 - util-intellij/OWNERS | 1 - util-logging/OWNERS | 1 - util-security/OWNERS | 1 - util-slf4j-api/OWNERS | 1 - util-slf4j-jul-bridge/OWNERS | 1 - util-zk/OWNERS | 1 - 7 files changed, 7 deletions(-) diff --git a/util-codec/OWNERS b/util-codec/OWNERS index b749f10796..10222fe988 100644 --- a/util-codec/OWNERS +++ b/util-codec/OWNERS @@ -1,5 +1,4 @@ banderson -koliver mnakamura roanta ryano diff --git a/util-intellij/OWNERS b/util-intellij/OWNERS index ddd4683efe..450160e1be 100644 --- a/util-intellij/OWNERS +++ b/util-intellij/OWNERS @@ -1,7 +1,6 @@ banderson ccoco ibennett -koliver mnakamura roanta ryano diff --git a/util-logging/OWNERS b/util-logging/OWNERS index ddd4683efe..450160e1be 100644 --- a/util-logging/OWNERS +++ b/util-logging/OWNERS @@ -1,7 +1,6 @@ banderson ccoco ibennett -koliver mnakamura roanta ryano diff --git a/util-security/OWNERS b/util-security/OWNERS index f5d09e0dd3..5b1829a750 100644 --- a/util-security/OWNERS +++ b/util-security/OWNERS @@ -1,6 +1,5 @@ banderson bdimmick -koliver mnakamura roanta ryano diff --git a/util-slf4j-api/OWNERS b/util-slf4j-api/OWNERS index 995bde0b3c..9a1030a1b3 100644 --- a/util-slf4j-api/OWNERS +++ b/util-slf4j-api/OWNERS @@ -1,7 +1,6 @@ banderson ccoco ibennett -koliver mnakamura roanta ryano diff --git a/util-slf4j-jul-bridge/OWNERS b/util-slf4j-jul-bridge/OWNERS index 995bde0b3c..9a1030a1b3 100644 --- a/util-slf4j-jul-bridge/OWNERS +++ b/util-slf4j-jul-bridge/OWNERS @@ -1,7 +1,6 @@ banderson ccoco ibennett -koliver mnakamura roanta ryano diff --git a/util-zk/OWNERS b/util-zk/OWNERS index 40598c8896..5b39b3b086 100644 --- a/util-zk/OWNERS +++ b/util-zk/OWNERS @@ -1,5 +1,4 @@ banderson -koliver mnakamura roanta ryano From 96d170cb2a962e166908a10cde5d188edde50e7d Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Thu, 23 Jul 2020 17:34:45 +0000 Subject: [PATCH 350/796] csl-libs: move master refs to release Problem / Solution We'd like to remove references to master in our OSS repos and replace them with release as it's more inclusive (and more accurately represents what this branch tracks). Differential Revision: https://phabricator.twitter.biz/D521022 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6c50472674..64557283bf 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,10 @@ public API breaks and changes in runtime behavior. ## Contributing -The `master` branch of this repository contains the latest stable release of +The `release` branch of this repository contains the latest stable release of Util, and weekly snapshots are published to the `develop` branch. In general pull requests should be submitted against `develop`. See -[CONTRIBUTING.md](https://github.com/twitter/util/blob/master/CONTRIBUTING.md) +[CONTRIBUTING.md](https://github.com/twitter/util/blob/release/CONTRIBUTING.md) for more details about how to contribute. # Using in your project From 93b2bdf6be0e716ff3ec761af5d9cf29b8b02db0 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Fri, 24 Jul 2020 18:34:05 +0000 Subject: [PATCH 351/796] [util] declare dependencies explicitly **Problem** Some transitive dependencies are used during compilation but are not explicitly declared or exported at the dependency level. **Solution** Declare the dependencies and exports accordingly. **Result** `//util/... - //util/util-zk-test/...` is now working JIRA Issues: DPB-12589 Differential Revision: https://phabricator.twitter.biz/D521600 --- util-cache/src/test/scala/com/twitter/cache/BUILD | 1 + util-jvm/src/test/scala/BUILD | 1 + util-routing/src/main/scala/com/twitter/util/routing/BUILD | 1 + .../src/test/scala/com/twitter/util/routing/dynamic/BUILD | 1 + .../src/test/scala/com/twitter/util/routing/simple/BUILD | 5 +++++ util-tunable/src/test/scala/com/twitter/util/tunable/BUILD | 1 + 6 files changed, 10 insertions(+) diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index 34fdee3ea2..41ce69098d 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -16,6 +16,7 @@ junit_tests( "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", ], ) diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index 3b5682f28b..d7258d2dfb 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -2,6 +2,7 @@ COMMON_DEPS = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-app/src/main/scala", "util/util-core:util-core-util", "util/util-jvm/src/main/scala/com/twitter/jvm", diff --git a/util-routing/src/main/scala/com/twitter/util/routing/BUILD b/util-routing/src/main/scala/com/twitter/util/routing/BUILD index 9640fd5133..1334037b77 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/main/scala/com/twitter/util/routing/BUILD @@ -9,4 +9,5 @@ scala_library( "util/util-core:util-core-util", "util/util-logging/src/main/scala", ], + exports = ["util/util-logging/src/main/scala"], ) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD index f56cc417db..0135c0eb05 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD @@ -2,6 +2,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], dependencies = [ + "util/util-core:util-core-util", "util/util-routing/src/main/scala/com/twitter/util/routing", ], ) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD index f56cc417db..92fbcb59a9 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD @@ -2,6 +2,11 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], dependencies = [ + "util/util-core:util-core-util", + "util/util-logging/src/main/scala/com/twitter/logging", "util/util-routing/src/main/scala/com/twitter/util/routing", ], + exports = [ + "util/util-logging/src/main/scala/com/twitter/logging", + ], ) diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD index 7e3c81ecad..cf8ea3414a 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -10,6 +10,7 @@ junit_tests( "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", "util/util-tunable/src/main/scala/com/twitter/util/tunable", "util/util-tunable/src/test/resources", ], From 389ca03d4e510d8aa998591871040bf360c0c754 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Fri, 24 Jul 2020 18:36:23 +0000 Subject: [PATCH 352/796] util-zk: Verify Current Owners Differential Revision: https://phabricator.twitter.biz/D520695 --- util-zk/OWNERS | 1 - 1 file changed, 1 deletion(-) diff --git a/util-zk/OWNERS b/util-zk/OWNERS index 5b39b3b086..10222fe988 100644 --- a/util-zk/OWNERS +++ b/util-zk/OWNERS @@ -2,5 +2,4 @@ banderson mnakamura roanta ryano -stuhood vkostyukov From 5dcafb4688728eeea99b44e8deb8b21605148d29 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 28 Jul 2020 01:20:19 +0000 Subject: [PATCH 353/796] finagle/util: Add yufang to owners of some subdirectories Differential Revision: https://phabricator.twitter.biz/D522729 --- util-benchmark/OWNERS | 1 + util-codec/OWNERS | 1 + util-core/OWNERS | 1 + 3 files changed, 3 insertions(+) diff --git a/util-benchmark/OWNERS b/util-benchmark/OWNERS index b749f10796..24a8b106dd 100644 --- a/util-benchmark/OWNERS +++ b/util-benchmark/OWNERS @@ -4,3 +4,4 @@ mnakamura roanta ryano vkostyukov +yufang diff --git a/util-codec/OWNERS b/util-codec/OWNERS index 10222fe988..95b52649d9 100644 --- a/util-codec/OWNERS +++ b/util-codec/OWNERS @@ -3,3 +3,4 @@ mnakamura roanta ryano vkostyukov +yufang diff --git a/util-core/OWNERS b/util-core/OWNERS index b749f10796..24a8b106dd 100644 --- a/util-core/OWNERS +++ b/util-core/OWNERS @@ -4,3 +4,4 @@ mnakamura roanta ryano vkostyukov +yufang From c305712e6fc1fc85d598256b7850393a97cd5c38 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Tue, 28 Jul 2020 18:15:58 +0000 Subject: [PATCH 354/796] util-stats: store schemas in InMemoryStatsReceiver Problem ------- Testing and verifying MetricSchema generation involves using the cumbersome MetricsStatsReceiver. Solution -------- Store MetricSchema data in the standard testing StatsReceiver, InMemoryStatsReceiver. JIRA Issues: CSL-7954 Differential Revision: https://phabricator.twitter.biz/D518195 --- CHANGELOG.rst | 2 + .../finagle/stats/InMemoryStatsReceiver.scala | 21 +++++++++- .../stats/InMemoryStatsReceiverTest.scala | 42 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e731f47098..3b8e0a0db5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,8 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-stats: Store MetricSchemas in InMemoryStatsReceiver to enable further testing. ``PHAB_ID=D518195`` + * util-core: c.t.u.Var.Observer is now public. This allows scala users to extend the Var trait as has been the case for Java users. ``PHAB_ID=D520237`` diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index e6549897e7..f086918004 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -65,6 +65,9 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { val gauges: mutable.Map[Seq[String], () => Float] = new ConcurrentHashMap[Seq[String], () => Float]().asScala + val schemas: mutable.Map[Seq[String], MetricSchema] = + new ConcurrentHashMap[Seq[String], MetricSchema]().asScala + override def counter(name: String*): ReadableCounter = counter(CounterSchema(this.metricBuilder().withName(name))) @@ -80,6 +83,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { counters.synchronized { if (!counters.contains(schema.metricBuilder.name)) { counters(schema.metricBuilder.name) = 0 + schemas(schema.metricBuilder.name) = schema } } @@ -109,6 +113,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { stats.synchronized { if (!stats.contains(schema.metricBuilder.name)) { stats(schema.metricBuilder.name) = Nil + schemas(schema.metricBuilder.name) = schema } } @@ -131,10 +136,12 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { new Gauge { gauges += schema.metricBuilder.name -> (() => f) + schemas += schema.metricBuilder.name -> schema verbosity += schema.metricBuilder.name -> schema.metricBuilder.verbosity def remove(): Unit = { gauges -= schema.metricBuilder.name + schemas -= schema.metricBuilder.name } override def toString: String = { @@ -190,13 +197,25 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } /** - * Clears all registered counters, gauges and stats. + * Dumps this in-memory stats receiver's MetricMetadata to the given [[PrintStream]]. + * @param p the [[PrintStream]] to which to write in-memory metadata. + */ + def printSchemas(p: PrintStream): Unit = { + val sortedSchemas = schemas.mapKeys(_.mkString("/")).toSortedMap + for ((k, schema) <- sortedSchemas) { + p.println(s"$k $schema") + } + } + + /** + * Clears all registered counters, gauges, stats, and their metadata. * @note this is not atomic. If new metrics are added while this method is executing, those metrics may remain. */ def clear(): Unit = { counters.clear() stats.clear() gauges.clear() + schemas.clear() } private[this] def toHistogramDetail(addedValues: Seq[Float]): HistogramDetail = { diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 25b87a9fc5..2984fd618f 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -313,4 +313,46 @@ class InMemoryStatsReceiverTest extends FunSuite with Eventually with Integratio } } // scalafix:on StoreGaugesAsMemberVariables + + test("creating metrics stores their metadata and clear()ing should remove that metadata") { + val stats = new InMemoryStatsReceiver() + stats.addGauge("coolGauge") { 3 } + assert(stats.schemas(Seq("coolGauge")).isInstanceOf[MetricSchema]) + stats.counter("sweetCounter") + assert(stats.schemas(Seq("sweetCounter")).isInstanceOf[MetricSchema]) + stats.stat("radHisto") + assert(stats.schemas(Seq("radHisto")).isInstanceOf[MetricSchema]) + + stats.clear() + assert(stats.schemas.isEmpty) + } + + test("printSchemas should print schemas") { + val stats = new InMemoryStatsReceiver() + stats.addGauge("coolGauge") { 3 } + stats.counter("sweetCounter") + stats.stat("radHisto") + + val baos = new ByteArrayOutputStream() + val ps = new PrintStream(baos, true, "utf-8") + try { + stats.printSchemas(ps) + val content = new String(baos.toByteArray, StandardCharsets.UTF_8) + println(content) + val parts = content.split('\n') + + assert(parts.length == 3) + assert( + parts( + 0) == "coolGauge GaugeSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, WrappedArray(coolGauge), None, Vector(), InMemoryStatsReceiver))") + assert( + parts( + 1) == "radHisto HistogramSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, WrappedArray(radHisto), None, Vector(), InMemoryStatsReceiver))") + assert( + parts( + 2) == "sweetCounter CounterSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, WrappedArray(sweetCounter), None, Vector(), InMemoryStatsReceiver))") + } finally { + ps.close() + } + } } From 6929bf7070d5c9a563581d3ce1bd464e60ea58bb Mon Sep 17 00:00:00 2001 From: Shane Delmore Date: Tue, 28 Jul 2020 21:56:03 +0000 Subject: [PATCH 355/796] Upgrade junit3 tests to junit4 Upgrade java tests still using jUnit3 to jUnit4 Differential Revision: https://phabricator.twitter.biz/D523474 --- .../service/TestVirtualMachineErrorTerminator.java | 7 +++++-- .../java/com/twitter/logging/LoggerFactoryTest.java | 10 ++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/util-core/src/test/java/com/twitter/service/TestVirtualMachineErrorTerminator.java b/util-core/src/test/java/com/twitter/service/TestVirtualMachineErrorTerminator.java index 15e5cb1369..508fd6cd08 100644 --- a/util-core/src/test/java/com/twitter/service/TestVirtualMachineErrorTerminator.java +++ b/util-core/src/test/java/com/twitter/service/TestVirtualMachineErrorTerminator.java @@ -1,11 +1,14 @@ package com.twitter.service; -import junit.framework.TestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; /** * @author Attila Szegedi */ -public class TestVirtualMachineErrorTerminator extends TestCase +public class TestVirtualMachineErrorTerminator { private static boolean exitInvoked; private static final Object lock = new Object(); diff --git a/util-logging/src/test/java/com/twitter/logging/LoggerFactoryTest.java b/util-logging/src/test/java/com/twitter/logging/LoggerFactoryTest.java index 8d39469eb6..345e483764 100644 --- a/util-logging/src/test/java/com/twitter/logging/LoggerFactoryTest.java +++ b/util-logging/src/test/java/com/twitter/logging/LoggerFactoryTest.java @@ -1,15 +1,20 @@ package com.twitter.logging; -import junit.framework.TestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.*; import scala.Function0; // Just make sure this compiles -public class LoggerFactoryTest extends TestCase { +public class LoggerFactoryTest { + @Test public void testBuilder() { LoggerFactoryBuilder builder = LoggerFactory.newBuilder(); } + @Test public void testFactory() { LoggerFactoryBuilder builder = LoggerFactory.newBuilder(); @@ -24,6 +29,7 @@ public void testFactory() { LoggerFactory factory = intermediate.build(); } + @Test public void testAddHandler() { LoggerFactoryBuilder builder = LoggerFactory.newBuilder(); From 3ade368483a129dbabb6a96504c871ebbd5871c7 Mon Sep 17 00:00:00 2001 From: Borja Lorente Date: Wed, 29 Jul 2020 17:05:34 +0000 Subject: [PATCH 356/796] Extract non-test sources to library for util/util-slf4j-api/src/test/scala/com/twitter/util/logging ### Problem The bazel test runner has issues when (1) a test target has mixed scala and java sources, and (2) none of the java sources are tests. ### Solution I've opted to split all the non-test sources into a library, and have the tests depend on that. ### Result Nothing should change externally. JIRA Issues: DPB-12609 Differential Revision: https://phabricator.twitter.biz/D523355 --- .../test/scala/com/twitter/util/logging/BUILD | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index e2c66a901f..12d8fa0184 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -1,18 +1,32 @@ -junit_tests( +scala_library( + name = "testutil", sources = [ + "!*Test.scala", "*.java", "*.scala", ], + dependencies = [ + "3rdparty/jvm/org/slf4j:slf4j-api", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + "util/util-slf4j-api/src/test/resources", + ], + exports = [ + "3rdparty/jvm/org/slf4j:slf4j-api", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + "util/util-slf4j-api/src/test/resources", + ], +) + +junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ + ":testutil", "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/scalatestplus:mockito-1-10", - "3rdparty/jvm/org/slf4j:slf4j-api", "3rdparty/jvm/org/slf4j:slf4j-simple", - "util/util-core/src/main/scala/com/twitter/io", - "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - "util/util-slf4j-api/src/test/resources", ], ) From 955f2ce9458c8a60af78b919275d934e51d7f623 Mon Sep 17 00:00:00 2001 From: Bonnie Eisenman Date: Wed, 29 Jul 2020 18:40:56 +0000 Subject: [PATCH 357/796] [util] Add Duration.fromHours and fromDays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: Java has TimeUnits for HOURS and DAYS, but ctu.Duration does not have the convenience methods to in it's fromXYZ suite. Workarounds exist — `Duration.fromTimeUnit` or `DurationOps`, but it's still awkward to have the discrepancy in our API. Solution: I added `fromHours` and `fromDays` to the `Time` class, mimicking the `fromMinutes`, `fromSeconds`, etc methods. JIRA Issues: CSL-7618 Differential Revision: https://phabricator.twitter.biz/D522734 --- CHANGELOG.rst | 2 ++ .../scala/com/twitter/util/Duration.scala | 2 ++ .../main/scala/com/twitter/util/Time.scala | 4 ++++ .../scala/com/twitter/util/TimeTest.scala | 22 +++++++++++++++++++ 4 files changed, 30 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b8e0a0db5..853284dcb6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,8 @@ New Features * util-core: c.t.u.Var.Observer is now public. This allows scala users to extend the Var trait as has been the case for Java users. ``PHAB_ID=D520237`` +* util-core: Added two new methods to c.t.u.Duration and c.t.u.Time: `.fromHours` and `.fromDays`. ``PHAB_ID=D522734`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ * util-app: Treat empty strings as empty collections in `Flag[Seq[_]]`, `Flag[Set[_]]`, diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index 55525c15ef..aa3dd18c2a 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -15,6 +15,8 @@ object Duration extends TimeLikeOps[Duration] { override def fromMinutes(minutes: Int): Duration = super.fromMinutes(minutes) override def fromMilliseconds(millis: Long): Duration = super.fromMilliseconds(millis) override def fromMicroseconds(micros: Long): Duration = super.fromMicroseconds(micros) + override def fromHours(hours: Int): Duration = super.fromHours(hours) + override def fromDays(days: Int): Duration = super.fromDays(days) val NanosPerMicrosecond: Long = 1000L val NanosPerMillisecond: Long = NanosPerMicrosecond * 1000L diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index 87cc7b8ffc..da179c4ce6 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -99,6 +99,10 @@ trait TimeLikeOps[This <: TimeLike[This]] { def fromMinutes(minutes: Int): This = fromMilliseconds(60L * 1000L * minutes) + def fromHours(hours: Int): This = fromMilliseconds(60L * 60L * 1000L * hours) + + def fromDays(days: Int): This = fromMilliseconds(24L * 3600L * 1000L * days) + /** * Make a new `This` from the given number of seconds. * Because this method takes a Double, it can represent values less than a second. diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index b159783d14..fba5f60652 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -667,6 +667,28 @@ with IntegrationPatience { assert(Time.fromMinutes(Int.MinValue) == Time.Bottom) } + "fromHours" in { + assert(Time.fromHours(1).inNanoseconds == Time.fromMinutes(60).inNanoseconds) + + assert(Time.fromHours(0).inNanoseconds == 0L) + assert(Time.fromHours(-1).inNanoseconds == -3600L * 1000000000L) + + assert(Time.fromHours(Int.MaxValue).inNanoseconds == Long.MaxValue) + assert(Time.fromHours(Int.MaxValue) == Time.Top) + assert(Time.fromHours(Int.MinValue) == Time.Bottom) + } + + "fromDays" in { + assert(Time.fromDays(1).inNanoseconds == Time.fromHours(24).inNanoseconds) + + assert(Time.fromDays(0).inNanoseconds == 0L) + assert(Time.fromDays(-1).inNanoseconds == -3600L * 24L * 1000000000L) + + assert(Time.fromDays(Int.MaxValue).inNanoseconds == Long.MaxValue) + assert(Time.fromDays(Int.MaxValue) == Time.Top) + assert(Time.fromDays(Int.MinValue) == Time.Bottom) + } + "until" in { val t0 = Time.now val t1 = t0 + 10.seconds From 493a8c030781e689b7924565c96fb041b138aac8 Mon Sep 17 00:00:00 2001 From: Borja Lorente Date: Thu, 30 Jul 2020 15:32:32 +0000 Subject: [PATCH 358/796] util-core: Add @Test and @After annotations to //util/util-core/src/test/java/com/twitter/service:service Problem A previous change migrated this target to Junit4. However, it didn't add the annotations necessary to make the tests discoverable. Solution Add the necessary annotations! JIRA Issues: DPB-12609 Differential Revision: https://phabricator.twitter.biz/D524196 --- .../service/TestVirtualMachineErrorTerminator.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/util-core/src/test/java/com/twitter/service/TestVirtualMachineErrorTerminator.java b/util-core/src/test/java/com/twitter/service/TestVirtualMachineErrorTerminator.java index 508fd6cd08..819daafd68 100644 --- a/util-core/src/test/java/com/twitter/service/TestVirtualMachineErrorTerminator.java +++ b/util-core/src/test/java/com/twitter/service/TestVirtualMachineErrorTerminator.java @@ -24,10 +24,12 @@ public void run() { }); } - protected void tearDown() { + @After + public void tearDown() { VirtualMachineErrorTerminator.reset(); } + @Test public void testNotTerminatingWhenNew() { assertNotTerminating(); } @@ -41,16 +43,19 @@ private static void assertNotTerminating() { } } + @Test public void testNotTerminatingOnNull() { VirtualMachineErrorTerminator.terminateVMIfMust(null); assertNotTerminating(); } + @Test public void testNotTerminatingOnOtherException() { VirtualMachineErrorTerminator.terminateVMIfMust(new Exception()); assertNotTerminating(); } + @Test public void testTerminatingOnNestedVirtualMachineError() throws Exception { Exception e = new Exception(); OutOfMemoryError oome = new OutOfMemoryError(); @@ -59,11 +64,13 @@ public void testTerminatingOnNestedVirtualMachineError() throws Exception { assertTerminated(); } + @Test public void testTerminatingOnVirtualMachineError() throws Exception { VirtualMachineErrorTerminator.terminateVMIfMust(new OutOfMemoryError()); assertTerminated(); } + @Test public void testTerminateDoesntAcceptNull() throws Exception { try { VirtualMachineErrorTerminator.terminateVM(null); @@ -73,6 +80,7 @@ public void testTerminateDoesntAcceptNull() throws Exception { } } + @Test public void testSecondTerminateIneffective() throws Exception { OutOfMemoryError oome1 = new OutOfMemoryError(); OutOfMemoryError oome2 = new OutOfMemoryError(); From 26d129d41e6cf3df9adef281fe932906553335b7 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Tue, 4 Aug 2020 21:54:57 +0000 Subject: [PATCH 359/796] util/util-routing: improve ergnomics around building and extending Router's Problem There is too much boilerplate involved with extending and creating a new `RouterBuilder`. Additionally, the generic types accepted for `Router` and `RouterBuilder` are not flexible enough. Solution Let's make `RouterBuilder` a composable, concrete implementation, instead of something that needs to be extended. We'll also fix our generic typing. Result Creating a `Router` requires less boilerplate and generics better support polymorphic types. JIRA Issues: CSL-10047 Differential Revision: https://phabricator.twitter.biz/D523975 --- build.sbt | 2 +- .../twitter/util/routing/AbstractRouter.scala | 6 - .../main/scala/com/twitter/util/routing/BUILD | 4 +- .../com/twitter/util/routing/Generator.scala | 24 ++++ .../routing/RouteValidationException.scala | 10 -- .../com/twitter/util/routing/Router.scala | 18 +-- .../twitter/util/routing/RouterBuilder.scala | 89 +++++------- .../util/routing/ValidationError.scala | 9 ++ .../util/routing/ValidationException.scala | 17 +++ .../com/twitter/util/routing/Validator.scala | 27 ++++ .../util/routing/RouterCompilationTest.java | 57 +++----- .../PolymorphicRouterCompilationTest.scala | 134 ++++++++++++++++++ .../util/routing/RouterBuilderTest.scala | 71 +++++++--- .../com/twitter/util/routing/RouterTest.scala | 2 +- .../util/routing/dynamic/DynamicRouter.scala | 13 +- 15 files changed, 337 insertions(+), 146 deletions(-) create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/Generator.scala delete mode 100644 util-routing/src/main/scala/com/twitter/util/routing/RouteValidationException.scala create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/ValidationError.scala create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/ValidationException.scala create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/Validator.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala diff --git a/build.sbt b/build.sbt index 8d6f87e5be..2b85fd52c5 100644 --- a/build.sbt +++ b/build.sbt @@ -476,7 +476,7 @@ lazy val utilRouting = Project( sharedSettings ).settings( name := "util-routing" - ).dependsOn(utilCore, utilLogging) + ).dependsOn(utilCore, utilSlf4jApi) lazy val utilTest = Project( id = "util-test", diff --git a/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala b/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala index f4307dea69..7e0409793b 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala @@ -1,6 +1,5 @@ package com.twitter.util.routing -import com.twitter.logging.Logger import com.twitter.util.Try import java.lang.{Iterable => JIterable} import java.util.Optional @@ -19,11 +18,6 @@ import scala.jdk.CollectionConverters._ abstract class AbstractRouter[Input, Route](val label: String, jRoutes: JIterable[Route]) extends Router[Input, Route] { - /** The [[Logger]] to use for this [[Router]]. Can be overridden to customize. */ - def getLogger(): Logger = Router.Log - - override final def logger: Logger = getLogger() - /** Java-friendly version of [[apply]] */ final def route(input: Input): Optional[Route] = apply(input) match { case Some(route) => Optional.of(route) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/BUILD b/util-routing/src/main/scala/com/twitter/util/routing/BUILD index 1334037b77..2c7510a5f5 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/main/scala/com/twitter/util/routing/BUILD @@ -7,7 +7,7 @@ scala_library( ), dependencies = [ "util/util-core:util-core-util", - "util/util-logging/src/main/scala", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], - exports = ["util/util-logging/src/main/scala"], + exports = ["util/util-slf4j-api/src/main/scala/com/twitter/util/logging"], ) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala b/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala new file mode 100644 index 0000000000..ed0a598e56 --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala @@ -0,0 +1,24 @@ +package com.twitter.util.routing + +import java.lang.{Iterable => JIterable} +import java.util.function.BiFunction +import scala.jdk.CollectionConverters._ + +object Generator { + + /** Java-friendly creation of a [[Generator]] */ + def create[Input, Route, RouterType <: Router[Input, Route]]( + fn: BiFunction[String, JIterable[Route], RouterType] + ): Generator[Input, Route, RouterType] = new Generator[Input, Route, RouterType] { + override def apply(label: String, routes: Iterable[Route]): RouterType = + fn(label, routes.asJava) + } + +} + +/** + * Functional alias for creating a [[RouterType router]] given a [[String label]] and + * all of the defined [[Route routes]] for the router to be generated. + */ +abstract class Generator[Input, Route, +RouterType <: Router[Input, Route]] + extends ((String, Iterable[Route]) => RouterType) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/RouteValidationException.scala b/util-routing/src/main/scala/com/twitter/util/routing/RouteValidationException.scala deleted file mode 100644 index ddfa222c64..0000000000 --- a/util-routing/src/main/scala/com/twitter/util/routing/RouteValidationException.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.twitter.util.routing - -/** - * Exception thrown when a [[RouterBuilder]] encounters a `Route` that is not valid for - * the [[Router]] type it is building. - */ -case class RouteValidationException(msg: String, cause: Throwable) extends Exception(msg, cause) { - def this(msg: String) = this(msg, null) - def this(cause: Throwable) = this(null, cause) -} diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Router.scala b/util-routing/src/main/scala/com/twitter/util/routing/Router.scala index ad30a5b4b4..35818c1a39 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/Router.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/Router.scala @@ -1,13 +1,9 @@ package com.twitter.util.routing -import com.twitter.logging.Logger +import com.twitter.util.logging.Logging import com.twitter.util.{Closable, ClosableOnce, Future, Time} import scala.util.control.NonFatal -object Router { - val Log = Logger.get(Router.getClass) -} - /** * A generic interface for routing an input to an optional matching route. * @@ -32,11 +28,7 @@ object Router { * * @note A [[Router]] should be considered immutable unless explicitly noted. */ -trait Router[Input, Route] extends (Input => Option[Route]) with ClosableOnce { - import Router.Log - - /** The [[Logger]] to use for this [[Router]]. Can be overridden to customize. */ - protected def logger: Logger = Log +trait Router[Input, +Route] extends (Input => Option[Route]) with ClosableOnce with Logging { /** * A label used for identifying this Router (i.e. for distinguishing between [[Router]] instances @@ -96,9 +88,9 @@ trait Router[Input, Route] extends (Input => Option[Route]) with ClosableOnce { case closable: Closable => closable.close(deadline).rescue { case NonFatal(e) => - logger.warning( - e, - s"Error encountered when attempting to close route '$route' in router '$label'") + logger.warn( + s"Error encountered when attempting to close route '$route' in router '$label'", + e) Future.Done } case _ => diff --git a/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala b/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala index b779c9fc4d..6889c61ca9 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala @@ -1,70 +1,59 @@ package com.twitter.util.routing -import scala.collection.mutable.ArrayBuffer +import scala.collection.immutable.Queue + +object RouterBuilder { + + def newBuilder[Input, Route, RouterType <: Router[Input, Route]]( + generator: Generator[Input, Route, RouterType] + ): RouterBuilder[Input, Route, RouterType] = + RouterBuilder(generator) + +} /** - * Contract for a [[Router]] builder. The resulting [[RouterType]] - * should be considered immutable, unless the [[RouterType]] implementation + * Utility for building and creating [[RouterType routers]]. The resulting [[RouterType router]] + * should be considered immutable, unless the [[RouterType router's]] implementation * explicitly states otherwise. * - * @param routeValidator A function that will be used to validate a [[Route]] when it is - * added to this builder via [[withRoute]]. If a [[Route]] is not - * valid, the [[routeValidator]] should throw a [[RouteValidationException]]. - * - * Ex: - * {{{ - * case class ExampleRoute(in: Int, out: Int) - * - * val routeValidator: ExampleRoute => Unit = { - * case ExampleRoute(in, out) if in < out => - * // route is valid - * () - * case ExampleRoute(in, out) => - * // route is not valid - * throw InvalidRouteException(s"'$in' is not less than '$out'") - * } - * }}} - * @tparam Input The [[Router]]'s `Input` type. - * @tparam Route The [[Router]]'s destination `Route` type. It is recommended that the `Route` + * @tparam Input The [[Router router's]] `Input` type. + * @tparam Route The [[Router router's]] destination `Route` type. It is recommended that the `Route` * is a self-contained/self-describing type for the purpose of validation via - * the [[routeValidator]]. Put differently, the `Route` should know of the + * the [[validator]]. Put differently, the `Route` should know of the * `Input` that maps to itself. * @tparam RouterType The type of [[Router]] to build. */ -abstract class RouterBuilder[Input, Route, RouterType <: Router[Input, Route]]( - routeValidator: Route => Unit) { +case class RouterBuilder[Input, Route, +RouterType <: Router[Input, Route]] private ( + private val generator: Generator[Input, Route, RouterType], + private val label: String = "router", + private val routes: Queue[Route] = Queue.empty, + private val validator: Validator[Route] = Validator.None) { - def this() = this((_: Route) => ()) - - private[this] val routes: ArrayBuffer[Route] = new ArrayBuffer[Route]() + /** Set the [[Router.label label]] for the resulting [[RouterType router]] */ + def withLabel(label: String): RouterBuilder[Input, Route, RouterType] = + copy(label = label) /** - * Add the [[route]] to the routes that will be present in - * the [[RouterType]] when [[newRouter()]] is called. - * - * @param route The [[Route]] to be added. - * - * @return - * - * @note The [[routeValidator]] will be exercised with the [[route]] - * and may throw an [[RouteValidationException]] + * Add the [[Route route]] to the routes that will be present in + * the [[RouterType router]] when [[newRouter()]] is called. */ - def withRoute(route: Route): this.type = synchronized { - routeValidator(route) - routes += route - this - } + def withRoute(route: Route): RouterBuilder[Input, Route, RouterType] = + copy(routes = routes :+ route) /** - * Generate a new [[RouterType]] from the defined [[routes]]. - * @param routes The [[Route]]s to be used for building the resulting [[RouterType]]. - * @return The resulting [[RouterType]] made up of the underlying [[routes]]. + * Configure the route validation logic for this builder */ - protected def newRouter(routes: Iterable[Route]): RouterType + def withValidator( + validator: Validator[Route] + ): RouterBuilder[Input, Route, RouterType] = + copy(validator = validator) - /** - * @return A new [[RouterType]] made up of this builder's defined routes. - */ - def newRouter(): RouterType = newRouter(routes) + /** Generate a new [[RouterType router]] from the defined [[routes]] */ + def newRouter(): RouterType = { + val failures = validator(routes) + + if (failures.nonEmpty) throw ValidationException(failures) + generator(label, routes) + } } diff --git a/util-routing/src/main/scala/com/twitter/util/routing/ValidationError.scala b/util-routing/src/main/scala/com/twitter/util/routing/ValidationError.scala new file mode 100644 index 0000000000..27c72e30fa --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/ValidationError.scala @@ -0,0 +1,9 @@ +package com.twitter.util.routing + +/** + * Container class for an error that is encountered as part of validating routes via + * a [[RouteBuilder]]. + * + * @param msg The message that explains the error. + */ +case class ValidationError(msg: String) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/ValidationException.scala b/util-routing/src/main/scala/com/twitter/util/routing/ValidationException.scala new file mode 100644 index 0000000000..dd436156a5 --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/ValidationException.scala @@ -0,0 +1,17 @@ +package com.twitter.util.routing + +private object ValidationException { + def errorMsg(failures: Iterable[ValidationError]): String = failures match { + case head :: Nil => head.msg + case Nil => throw new IllegalArgumentException("RouteValidationError failures cannot be empty") + case failures => failures.map(_.msg).mkString(", ") + } +} + +/** + * Exception thrown when a [[RouterBuilder]] observes any [[Route routes]] that are not valid for + * the [[Router]] type it is building. + */ +case class ValidationException private[routing] (failures: Iterable[ValidationError]) + extends Exception( + s"Route Validation Failed! Errors encountered: [${ValidationException.errorMsg(failures)}]") diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Validator.scala b/util-routing/src/main/scala/com/twitter/util/routing/Validator.scala new file mode 100644 index 0000000000..acb6f60173 --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/Validator.scala @@ -0,0 +1,27 @@ +package com.twitter.util.routing + +import java.lang.{Iterable => JIterable} +import java.util.function.Function +import scala.jdk.CollectionConverters._ + +object Validator { + + /** [[Validator]] that does no validation and always returns empty error results */ + val None: Validator[Any] = new Validator[Any] { + override def apply(routes: Iterable[Any]): Iterable[ValidationError] = Iterable.empty + } + + /** Java-friendly creation of a [[Validator]] */ + def create[T](fn: Function[JIterable[T], JIterable[ValidationError]]): Validator[T] = + new Validator[T] { + override def apply(routes: Iterable[T]): Iterable[ValidationError] = + fn(routes.asJava).asScala + } + +} + +/** + * Functional alias for determining whether all defined results are valid for a specific + * [[Router router]] type. + */ +abstract class Validator[-Route] extends (Iterable[Route] => Iterable[ValidationError]) diff --git a/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java b/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java index b1a3a3c2eb..c079172a51 100644 --- a/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java +++ b/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java @@ -3,14 +3,11 @@ import org.junit.Assert; import org.junit.Test; -import scala.collection.Iterable; - import java.util.Map; import java.util.HashMap; import java.util.Optional; - -import static com.twitter.util.Function.exfunc; -import static com.twitter.util.Function.func; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; public class RouterCompilationTest { @@ -25,7 +22,13 @@ private static final class StringRoute { } private static final class StringRouter extends AbstractRouter { - static StringRouterBuilder newBuilder() { return new StringRouterBuilder(); } + static RouterBuilder newBuilder() { + return RouterBuilder.newBuilder(Generator.create((label, routes) -> { + Map routeMap = new HashMap<>(); + routes.forEach(r -> routeMap.put(r.input, r)); + return new StringRouter(routeMap); + })); + } private final Map routeMap; @@ -42,35 +45,14 @@ public Optional findAny(String input) { } - private static final class StringRouterBuilder - extends RouterBuilder { - - @Override - public StringRouter newRouter(Iterable routes) { - Map routeMap = new HashMap<>(); - routes.foreach(func(r -> routeMap.put(r.input, r))); - return new StringRouter(routeMap); + private static final class ValidatingStringRouterBuilder { + static RouterBuilder newBuilder() { + return StringRouter.newBuilder().withValidator(Validator.create(routes -> + StreamSupport.stream(routes.spliterator(), false) + .filter(r -> r.input.equals("invalid")) + .map(msg -> new ValidationError("INVALID!")) + .collect(Collectors.toList()))); } - - } - - private static final class ValidatingStringRouterBuilder - extends RouterBuilder { - - ValidatingStringRouterBuilder() { - super(exfunc(r -> { - if (r.input.equals("invalid")) throw new RouteValidationException("INVALID!"); - return null; - })); - } - - @Override - public StringRouter newRouter(Iterable routes) { - Map routeMap = new HashMap<>(); - routes.foreach(func(r -> routeMap.put(r.input, r))); - return new StringRouter(routeMap); - } - } @Test @@ -103,10 +85,11 @@ public void testRouterBuilder() { Assert.assertEquals(router.route("oh-no"), Optional.empty()); } - @Test(expected = RouteValidationException.class) + @Test(expected = ValidationException.class) public void testValidatingRouterBuilder() { - new ValidatingStringRouterBuilder() - .withRoute(new StringRoute("invalid", "other")); + ValidatingStringRouterBuilder.newBuilder() + .withRoute(new StringRoute("invalid", "other")) + .newRouter(); } } diff --git a/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala new file mode 100644 index 0000000000..f5bb2a48f5 --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala @@ -0,0 +1,134 @@ +package com.twitter.util.routing + +import org.scalatest.FunSuite + +private object PolymorphicRouterCompilationTest { + + /* polymorphic output type router */ + + sealed trait Status + case object Success extends Status + case object Failure extends Status + + sealed trait StatusHandler { + def str: String + def status: Status + } + case class StatusHandlerA(str: String, status: Status) extends StatusHandler + case class StatusHandlerB(str: String, status: Status) extends StatusHandler + + object StatusRouter { + def newBuilder: RouterBuilder[String, StatusHandler, StatusRouter] = + RouterBuilder.newBuilder[String, StatusHandler, StatusRouter]( + new Generator[String, StatusHandler, StatusRouter] { + override def apply(label: String, routes: Iterable[StatusHandler]): StatusRouter = + StatusRouter(label, routes) + }) + } + + case class StatusRouter(label: String, routes: Iterable[StatusHandler]) + extends Router[String, StatusHandler] { + override protected def find(input: String): Option[StatusHandler] = routes.find(_.str == input) + } + + val statusRouter: StatusRouter = StatusRouter.newBuilder + .withRoute(StatusHandlerA("success", Success)) + .withRoute(StatusHandlerB("fail", Failure)) + .newRouter() + + def checkStatusRouterType: Router[String, StatusHandler] = statusRouter + + val covariantStatusRouter: Router[String, StatusHandler] = new Router[String, StatusHandlerA] { + override def label: String = "covariant-status" + override def routes: Iterable[StatusHandlerA] = Iterable.empty + override protected def find(input: String): Option[StatusHandlerA] = None + } + + /* polymorphic input type router */ + + sealed trait Command + case class IntCommand(value: Int) extends Command + case class StringCommand(value: String) extends Command + case object NullCommand extends Command + + case class CommandHandler(canHandle: Command => Boolean, status: Status) + + object CommandRouter { + def newBuilder: RouterBuilder[Command, CommandHandler, CommandRouter] = + RouterBuilder.newBuilder(new Generator[Command, CommandHandler, CommandRouter] { + override def apply(label: String, routes: Iterable[CommandHandler]): CommandRouter = + CommandRouter(label, routes) + }) + } + + case class CommandRouter(label: String, routes: Iterable[CommandHandler]) + extends Router[Command, CommandHandler] { + override protected def find(input: Command): Option[CommandHandler] = + routes.find(_.canHandle(input)) + } + + val CommandHandlerA: CommandHandler = CommandHandler(_.isInstanceOf[IntCommand], Success) + val CommandHandlerB: CommandHandler = CommandHandler(_.isInstanceOf[StringCommand], Failure) + + val commandRouter: CommandRouter = CommandRouter.newBuilder + .withRoute(CommandHandlerA) + .withRoute(CommandHandlerB) + .newRouter() + + def checkCommandRouterType: Router[Command, CommandHandler] = commandRouter + +} + +// these tests ensure that we properly support polymorphic Router and RouterBuilder types +class PolymorphicRouterCompilationTest extends FunSuite { + import PolymorphicRouterCompilationTest._ + + test("Supports routing with polymorphic destinations") { + assert(statusRouter("success") == Some(StatusHandlerA("success", Success))) + assert(statusRouter("fail") == Some(StatusHandlerB("fail", Failure))) + assert(statusRouter("meh") == None) + } + + test("Supports routing with polymorphic inputs") { + assert(commandRouter(IntCommand(1)) == Some(CommandHandlerA)) + assert(commandRouter(StringCommand("1")) == Some(CommandHandlerB)) + assert(commandRouter(NullCommand) == None) + } + + test("Input type cannot be covariant") { + assertDoesNotCompile { + """ + | val contravariantInputRouter: Router[Command, CommandHandler] = new Router[IntCommand, CommandHandler] { + | override def label: String = "int-command-router" + | override def routes: Iterable[CommandHandler] = Iterable.empty + | override protected def find(input: IntCommand): Option[CommandHandler] = None + | } + |""".stripMargin + } + } + + test("Input type cannot be contravariant") { + assertDoesNotCompile { + """ + | val contravariantInputRouter: Router[IntCommand, CommandHandler] = new Router[Command, CommandHandler] { + | override def label: String = "int-command-router" + | override def routes: Iterable[CommandHandler] = Iterable.empty + | override protected def find(input: Command): Option[CommandHandler] = None + | } + |""".stripMargin + } + } + + test("Destination type cannot be contravariant") { + assertDoesNotCompile { + """ + | val contravariantStatusRouter: Router[String, StatusHandlerA] = new Router[String, StatusHandler] { + | override def label: String = "contravariant-status" + | override def routes: Iterable[StatusHandler] = Iterable.empty + | override protected def find(input: String): Option[StatusHandler] = None + | } + |""".stripMargin + } + } + +} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala index 29a2521be6..151705f01f 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala @@ -6,24 +6,23 @@ import org.scalatest.FunSuite class RouterBuilderTest extends FunSuite { private object TestRouter { - def newBuilder(): TestRouterBuilder = new TestRouterBuilder() + def newBuilder(): RouterBuilder[String, SimpleRoute, SimpleRouter] = + RouterBuilder.newBuilder(new Generator[String, SimpleRoute, SimpleRouter] { + override def apply(label: String, routes: Iterable[SimpleRoute]): SimpleRouter = + new SimpleRouter(routes.map { r => r.in -> r }.toMap) + }) } - private class TestRouterBuilder private[RouterBuilderTest] () - extends RouterBuilder[String, SimpleRoute, SimpleRouter] { - - protected def newRouter(routes: Iterable[SimpleRoute]): SimpleRouter = - new SimpleRouter(routes.map { r => r.in -> r }.toMap) - - } - - private class ValidatingTestRouterBuilder private[RouterBuilderTest] () - extends RouterBuilder[String, SimpleRoute, SimpleRouter](routeValidator = r => - if (r.in == "invalid") throw new IllegalArgumentException("INVALID!")) { - - protected def newRouter(routes: Iterable[SimpleRoute]): SimpleRouter = - new SimpleRouter(routes.map { r => r.in -> r }.toMap) - + private object ValidatingTestRouterBuilder { + def newBuilder(): RouterBuilder[String, SimpleRoute, SimpleRouter] = + TestRouter.newBuilder.withValidator( + new Validator[SimpleRoute] { + override def apply(routes: Iterable[SimpleRoute]): Iterable[ValidationError] = + routes.collect { + case r if r.in == "invalid" => ValidationError(s"INVALID @ $r") + } + } + ) } test("can build routes") { @@ -41,13 +40,47 @@ class RouterBuilderTest extends FunSuite { } test("throws when invalid route is passed") { - val router = new ValidatingTestRouterBuilder() + val router = ValidatingTestRouterBuilder.newBuilder + .withRoute(SimpleRoute("x", true)) + .withRoute(SimpleRoute("y", false)) + .withRoute(SimpleRoute("z", true)) + + router.newRouter() // everything is valid up to here + + val err = intercept[ValidationException] { + router.withRoute(SimpleRoute("invalid", false)).newRouter() + } + assert( + err.getMessage == "Route Validation Failed! Errors encountered: [INVALID @ SimpleRoute(invalid,false)]") + } + + test("throws when multiple invalid routes are passed") { + val router = ValidatingTestRouterBuilder.newBuilder .withRoute(SimpleRoute("x", true)) .withRoute(SimpleRoute("y", false)) .withRoute(SimpleRoute("z", true)) - intercept[IllegalArgumentException] { - router.withRoute(SimpleRoute("invalid", false)) + router.newRouter() // everything is valid up to here + + val err = intercept[ValidationException] { + router + .withRoute(SimpleRoute("invalid", false)) + .withRoute(SimpleRoute("invalid", true)) + .newRouter() + } + assert( + err.getMessage == "Route Validation Failed! Errors encountered: [INVALID @ SimpleRoute(invalid,false), INVALID @ SimpleRoute(invalid,true)]") + } + + test("support contravariant builder") { + assertCompiles { + """ + | val typed: RouterBuilder[String, SimpleRoute, SimpleRouter] = ValidatingTestRouterBuilder.newBuilder + | val generic: RouterBuilder[String, SimpleRoute, Router[String, SimpleRoute]] = typed + | val simpleRouter: SimpleRouter = typed.newRouter() + | val simpleRouterB: Router[String, SimpleRoute] = generic.newRouter() + | val simpleRouterC: Router[String, SimpleRoute] = typed.newRouter() + |""".stripMargin } } diff --git a/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala index 6ad177903a..130b74be52 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala @@ -33,7 +33,7 @@ class RouterTest extends FunSuite { val defWriteHandler = DynamicRoute(Method.Write, _.query == "def", _ => Future.value(Result.Failure)) - val router = DynamicRouter + val router: DynamicRouter = DynamicRouter .newBuilder() .withRoute(abcReadHandler) .withRoute(xyzReadHandler) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala index 5e3c86e477..9cd946d163 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala @@ -1,14 +1,13 @@ package com.twitter.util.routing.dynamic -import com.twitter.util.routing.{Router, RouterBuilder} +import com.twitter.util.routing.{Generator, Router, RouterBuilder} private[routing] object DynamicRouter { - def newBuilder(): DynamicTestRouterBuilder = new DynamicTestRouterBuilder() - - class DynamicTestRouterBuilder extends RouterBuilder[Request, DynamicRoute, DynamicRouter] { - override protected def newRouter(routes: Iterable[DynamicRoute]): DynamicRouter = - new DynamicRouter(routes.toSeq) - } + def newBuilder(): RouterBuilder[Request, DynamicRoute, DynamicRouter] = + RouterBuilder.newBuilder(new Generator[Request, DynamicRoute, DynamicRouter] { + override def apply(label: String, routes: Iterable[DynamicRoute]): DynamicRouter = + new DynamicRouter(routes.toSeq) + }) } // example to show how a router might be built using dynamic inputs From 9aa75d2b27e7092c17542e0a129a49a755a63c56 Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Wed, 5 Aug 2020 00:07:51 +0000 Subject: [PATCH 360/796] util-inject: added Injector to util-inject Problem we want users to take advantage of dependency injection with out depending on finatra. So, the first step in moving finatra/inject to util is moving Injector Solution Created a directory in util-inject and added 3rd party dependencies in BUILD and changed the package name accordingly Result This will help us in completing the first step in moving finatra/inject to util-inject JIRA Issues: CSL-10022 Differential Revision: https://phabricator.twitter.biz/D522862 --- .../main/scala/com/twitter/util/inject/BUILD | 22 ++ .../com/twitter/util/inject/Injector.scala | 142 +++++++++++ .../com/twitter/util/inject/TypeUtils.scala | 76 ++++++ .../test/java/com/twitter/util/inject/BUILD | 7 + .../util/inject/TestBindingAnnotation.java | 19 ++ .../test/scala/com/twitter/util/inject/BUILD | 16 ++ .../twitter/util/inject/InjectorTest.scala | 227 ++++++++++++++++++ .../twitter/util/inject/TypeUtilsTest.scala | 46 ++++ .../com/twitter/util/inject/classes.scala | 70 ++++++ .../util/inject/module/TestModule.scala | 25 ++ 10 files changed, 650 insertions(+) create mode 100644 util-inject/src/main/scala/com/twitter/util/inject/BUILD create mode 100644 util-inject/src/main/scala/com/twitter/util/inject/Injector.scala create mode 100644 util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala create mode 100644 util-inject/src/test/java/com/twitter/util/inject/BUILD create mode 100644 util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java create mode 100644 util-inject/src/test/scala/com/twitter/util/inject/BUILD create mode 100644 util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala create mode 100644 util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala create mode 100644 util-inject/src/test/scala/com/twitter/util/inject/classes.scala create mode 100644 util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala diff --git a/util-inject/src/main/scala/com/twitter/util/inject/BUILD b/util-inject/src/main/scala/com/twitter/util/inject/BUILD new file mode 100644 index 0000000000..cf545d03d6 --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/util/inject/BUILD @@ -0,0 +1,22 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-inject", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "//:scala-reflect", + "3rdparty/jvm/com/google/inject:guice", + "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", + "3rdparty/jvm/javax/inject:javax.inject", + "3rdparty/jvm/net/codingwell:scala-guice", + ], + exports = [ + "3rdparty/jvm/com/google/inject:guice", + "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", + "3rdparty/jvm/net/codingwell:scala-guice", + ], +) diff --git a/util-inject/src/main/scala/com/twitter/util/inject/Injector.scala b/util-inject/src/main/scala/com/twitter/util/inject/Injector.scala new file mode 100644 index 0000000000..7264bcc95d --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/util/inject/Injector.scala @@ -0,0 +1,142 @@ +package com.twitter.util.inject + +import com.google.inject.{Key, Injector => UnderlyingInjector} +import java.lang.annotation.Annotation +import net.codingwell.scalaguice.KeyExtensions._ +import net.codingwell.scalaguice.typeLiteral +import scala.reflect.ClassTag +import scala.reflect.runtime.universe._ + +final case class Injector private[inject] (underlying: UnderlyingInjector) { + + /** + * Returns the appropriate instance for the given key constructed from the + * passed type [[T]]. + * + * When feasible, avoid using this method in favor of having the Injector + * inject your dependencies ahead of time via annotating your constructor + * with `@Inject`. + * + * @tparam T type of the bound instance to return from the object graph. + * + * @return bound instance of type [[T]]. + */ + def instance[T: TypeTag]: T = + underlying.getInstance(typeLiteral[T].toKey) + + /** + * Returns the appropriate instance for the given key constructed from the + * passed type [[T]] and given [[java.lang.annotation.Annotation]] type [[Ann]]. + * + * When feasible, avoid using this method in favor of having the Injector + * inject your dependencies ahead of time via annotating your constructor + * with `@Inject`. + * + * @tparam T type of the bound instance to return from the object graph. + * @tparam Ann type of the annotation used to disambiguate the bound type [[T]]. + * + * @return bound instance of type [[T]] annotated with annotation type [[Ann]]. + */ + def instance[T: TypeTag, Ann <: Annotation: ClassTag]: T = + underlying.getInstance(typeLiteral[T].annotatedWith[Ann]) + + /** + * Returns the appropriate instance for the given key constructed from the + * passed type [[T]] and given [[java.lang.annotation.Annotation]] annotation. + * + * When feasible, avoid using this method in favor of having the Injector + * inject your dependencies ahead of time via annotating your constructor + * with `@Inject`. + * + * @param annotation [[java.lang.annotation.Annotation]] instance used to + * disambiguate the bound type [[T]]. + * @tparam T type of the bound instance to return from the object graph. + * + * @return bound instance of type [[T]] annotated with annotation. + */ + def instance[T: TypeTag](annotation: Annotation): T = + underlying.getInstance(typeLiteral[T].annotatedWith(annotation)) + + /** + * Returns the appropriate instance for the given key constructed from the + * passed type [[T]] and given [[java.lang.annotation.Annotation]] class. + * + * When feasible, avoid using this method in favor of having the Injector + * inject your dependencies ahead of time via annotating your constructor + * with `@Inject`. + * + * @param annotationClazz class of [[java.lang.annotation.Annotation]] used + * to disambiguate the bound type [[T]]. + * @tparam T type of the bound instance to return from the object graph. + * @return bound instance of type [[T]] annotated with annotation class. + */ + def instance[T: TypeTag](annotationClazz: Class[_ <: Annotation]): T = + underlying.getInstance(typeLiteral[T].annotatedWith(annotationClazz)) + + /** + * Returns the appropriate instance for the given injection type. + * + * When feasible, avoid using this method, in favor of having the Injector + * inject your dependencies ahead of time via annotating your constructor + * with `@Inject`. + * + * @param clazz the class of type [[T]] of the bound instance to return + * from the object graph. + * @tparam T type of the bound instance to return from the object graph. + * + * @return bound instance of type [[T]]. + */ + def instance[T](clazz: Class[T]): T = underlying.getInstance(clazz) + + /** + * Returns the appropriate instance for the given key constructed from the + * passed class and given [[java.lang.annotation.Annotation]] annotation. + * + * When feasible, avoid using this method, in favor of having the Injector + * inject your dependencies ahead of time via annotating your constructor + * with `@Inject`. + * + * @param clazz the class of type [[T]] of the bound instance to return from + * the object graph. + * @param annotation [[java.lang.annotation.Annotation]] instance used to + * disambiguate the bound type [[T]]. + * @tparam T type of the bound instance to return from the object graph. + * + * @return bound instance of type [[T]]. + */ + def instance[T](clazz: Class[T], annotation: Annotation): T = + underlying.getInstance(Key.get(clazz, annotation)) + + /** + * Returns the appropriate instance for the given key constructed from the + * passed class and given [[java.lang.annotation.Annotation]] class. + * + * @param clazz the class of type [[T]] of the bound instance to return from + * the object graph. + * @param annotationClazz [[java.lang.annotation.Annotation]] class used to + * disambiguate the bound type [[T]]. + * @tparam T type of the bound instance to return from the object graph. + * @tparam Ann type of the annotation class used to disambiguate the bound type [[T]]. + * + * @return bound instance of type [[T]]. + */ + def instance[T, Ann <: Annotation](clazz: Class[T], annotationClazz: Class[Ann]): T = + underlying.getInstance(Key.get(clazz, annotationClazz)) + + /** + * Returns the appropriate instance for the given injection key. + * + * When feasible, avoid using this method in favor of having the Injector + * inject your dependencies ahead of time via annotating your constructor + * with `@Inject`. + * + * @param key [[com.google.inject.Key]] binding key of the bound instance to return + * from the object graph. + * @tparam T type of the bound instance to return from the object graph. + * + * @return bound instance of type [[T]] represented by [[com.google.inject.Key]] key. + * + * @see [[https://google.github.io/guice/api-docs/latest/javadoc/com/google/inject/Key.html com.google.inject.Key]] + */ + def instance[T](key: Key[T]): T = underlying.getInstance(key) +} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala b/util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala new file mode 100644 index 0000000000..15a162e68c --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala @@ -0,0 +1,76 @@ +package com.twitter.util.inject + +import com.google.inject.TypeLiteral +import com.google.inject.internal.MoreTypes.ParameterizedTypeImpl +import scala.reflect.api.TypeCreator +import scala.reflect.runtime.universe._ +import scala.reflect.{ClassTag, ManifestFactory} + +private[inject] object TypeUtils { + + def singleTypeParam[T](objType: java.lang.reflect.Type): java.lang.reflect.Type = objType match { + case parametricType: ParameterizedTypeImpl => + parametricType.getActualTypeArguments.head + } + + def superTypeFromClass(clazz: Class[_], superClazz: Class[_]): java.lang.reflect.Type = { + TypeLiteral.get(clazz).getSupertype(superClazz).getType + } + + /** + * Convert a [[TypeTag]] to a [[Manifest]]. Recursively attempts to + * convert any type arguments from the given [[TypeTag]] to use for + * creating a Manifest[T]. + * + * @tparam T - the [[TypeTag]] to convert + * @return a Manifest[T] representation from the given type [T]. + */ + def asManifest[T: TypeTag]: Manifest[T] = { + val t = typeTag[T] + val mirror = t.mirror + def manifestFromType(t: Type): Manifest[_] = { + t match { + case n if n =:= typeOf[Nothing] => ManifestFactory.Nothing + case n if n =:= typeOf[Null] => ManifestFactory.Null + case n if n =:= typeOf[Any] => ManifestFactory.Any + case n if n =:= typeOf[AnyVal] => ManifestFactory.AnyVal + case _ => + try { + val clazz = ClassTag[T](mirror.runtimeClass(t)).runtimeClass + if (t.typeArgs.length == 1) { + val arg = manifestFromType(t.typeArgs.head) + ManifestFactory.classType(clazz, arg) + } else if (t.typeArgs.length > 1) { + // recursively walk each type arg to create a Manifest + val args = t.typeArgs.map(manifestFromType) + ManifestFactory.classType(clazz, args.head, args.tail: _*) + } else { + ManifestFactory.classType(clazz) + } + } catch { + case _: NoClassDefFoundError => ManifestFactory.Any + } + } + } + manifestFromType(t.tpe).asInstanceOf[Manifest[T]] + } + + /** + * Convert from the given Class[T] to a TypeTag[T] + * @param clazz the class for which to build the resultant TypeTag + * @return a TypeTag[T] representing the given Class[T] + */ + def asTypeTag[T](clazz: Class[_ <: T]): TypeTag[T] = { + val clazzMirror = runtimeMirror(clazz.getClassLoader) + val tpe = clazzMirror.classSymbol(clazz).toType + val typeCreator = new TypeCreator() { + def apply[U <: scala.reflect.api.Universe with scala.Singleton]( + m: scala.reflect.api.Mirror[U] + ): U#Type = { + if (clazzMirror != m) throw new RuntimeException("wrong mirror") + else tpe.asInstanceOf[U#Type] + } + } + TypeTag[T](clazzMirror, typeCreator) + } +} diff --git a/util-inject/src/test/java/com/twitter/util/inject/BUILD b/util-inject/src/test/java/com/twitter/util/inject/BUILD new file mode 100644 index 0000000000..961efa2be8 --- /dev/null +++ b/util-inject/src/test/java/com/twitter/util/inject/BUILD @@ -0,0 +1,7 @@ +java_library( + sources = ["*.java"], + compiler_option_sets = [], + dependencies = [ + "3rdparty/jvm/com/google/inject:guice", + ], +) diff --git a/util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java b/util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java new file mode 100644 index 0000000000..588c744cc7 --- /dev/null +++ b/util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java @@ -0,0 +1,19 @@ +package com.twitter.util.inject; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import com.google.inject.BindingAnnotation; + +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * HELPER ANNOTATION FOR TESTS. + *

    + * Note: This is not to be included in any published test:jar. + */ +@BindingAnnotation +@Target(PARAMETER) +@Retention(RUNTIME) +public @interface TestBindingAnnotation {} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/BUILD b/util-inject/src/test/scala/com/twitter/util/inject/BUILD new file mode 100644 index 0000000000..c13d7fd0bc --- /dev/null +++ b/util-inject/src/test/scala/com/twitter/util/inject/BUILD @@ -0,0 +1,16 @@ +junit_tests( + sources = ["**/*.scala"], + compiler_option_sets = ["fatal_warnings"], + dependencies = [ + "3rdparty/jvm/com/google/inject:guice", + "3rdparty/jvm/com/google/inject/extensions:guice-testlib", + "3rdparty/jvm/javax/inject:javax.inject", + "3rdparty/jvm/junit", + "3rdparty/jvm/net/codingwell:scala-guice", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:scalacheck", + "util/util-inject/src/main/scala/com/twitter/util/inject", + "util/util-inject/src/test/java/com/twitter/util/inject", + ], +) diff --git a/util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala b/util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala new file mode 100644 index 0000000000..982cb753d2 --- /dev/null +++ b/util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala @@ -0,0 +1,227 @@ +package com.twitter.util.inject + +import com.google.inject._ +import com.google.inject.name.Names +import com.twitter.util.inject.classes.{TypeProvider, _} +import com.twitter.util.inject.module.TestModule +import javax.inject.{Inject, Named, Singleton} +import net.codingwell.scalaguice.ScalaModule +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +object SeqAbstractModule extends AbstractModule with ScalaModule { + @Singleton + @Provides + def provideSeq: Seq[Array[Byte]] = + Seq.empty[Array[Byte]] +} + +trait MyServiceInterface { + + def add2(int: Int): Int +} + +class MyServiceImpl extends MyServiceInterface { + override def add2(int: Int): Int = { + int + 2 + } +} + +class SomeClient { + protected val underlying: String = "Hello, Alice." + def get: String = underlying +} + +trait Augmentation { self: SomeClient => + override def get: String = s"${self.underlying} Welcome to Wonderland." + def onlyHere: String = "Only on augmented classes." +} + +class Baz @Inject() (client: SomeClient with Augmentation) { + client.onlyHere +} + +object InjectorTest { + type AugmentedSomeClient = SomeClient with Augmentation +} + +@RunWith(classOf[JUnitRunner]) +class InjectorTest extends AnyFunSuite with Matchers { + import InjectorTest._ + + private case class W[T](t: T) + // Users should prefer the `c.t.inject.app.TestInjector` but it is not in scope here. + private val injector: Injector = Injector(Guice.createInjector(new TestModule {})) + + test("byte array generics") { + import net.codingwell.scalaguice.InjectorExtensions._ + val i = Guice.createInjector(SeqAbstractModule) + + i.instance[Seq[Array[Byte]]] should equal(Seq.empty[Array[Byte]]) + } + + test("can't bind class with trait and class") { + intercept[CreationException] { + Guice.createInjector( + new AbstractModule with ScalaModule { + @Provides + @Singleton + def provideSomeClient: SomeClient with Augmentation = { + new SomeClient with Augmentation + } + }, + new AbstractModule with ScalaModule { + @Provides + @Singleton + def provideSomeClient: SomeClient = { + new SomeClient + } + } + ) + } + } + + test("class with trait type") { + import net.codingwell.scalaguice.InjectorExtensions._ + val i = Guice.createInjector( + Stage.PRODUCTION, + new AbstractModule with ScalaModule { + @Provides + @Singleton + def provideSomeClient: SomeClient with Augmentation = { + new SomeClient with Augmentation + } + }) + + val r1 = i.instance[SomeClient with Augmentation] + r1.get should equal("Hello, Alice. Welcome to Wonderland.") + + val r2 = i.instance[AugmentedSomeClient] + r2.get should equal("Hello, Alice. Welcome to Wonderland.") + } + + test("class only type") { + import net.codingwell.scalaguice.InjectorExtensions._ + val i = Guice.createInjector( + Stage.PRODUCTION, + new AbstractModule with ScalaModule { + @Provides + @Singleton + def provideSomeClient: SomeClient = { + new SomeClient + } + }) + + // fails since only SomeClient is bound + val e = intercept[ProvisionException] { + i.instance[Baz] + } + e.getCause.isInstanceOf[ClassCastException] should be(true) + } + + test("get additional instances from injector") { + injector.instance[String](Names.named("str1")) should be("string1") + injector.instance[String, TestBindingAnnotation] should be("prod string") + injector.instance[String](Names.named("name")) should equal("Steve") + } + + test("allow binding target type using a type parameter") { + val module = new TestModule { + override protected def configure(): Unit = { + bind[X].to[Y] + } + } + + Injector(Guice.createInjector(module)).instance[X] + } + + test("allow binding target provider type using a type parameter") { + val module = new TestModule { + override protected def configure(): Unit = { + bind[X].toProvider[YProvider] + } + } + + Injector(Guice.createInjector(module)).instance[X] + } + + test("allow binding to provider of subtype using type parameter") { + val module = new TestModule { + override protected def configure(): Unit = { + bind[Gen[String]].toProvider[ZProvider] + } + } + + Injector(Guice.createInjector(module)).instance(new Key[Gen[String]] {}) + } + + test("allow binding to provider with injected type literal") { + val module = new TestModule { + override protected def configure(): Unit = { + bind[String].toProvider[TypeProvider[B]] + } + } + + Injector(Guice.createInjector(module)).instance(new Key[String] {}) + } + + test("allow binding in scope using a type parameter") { + val module = new TestModule { + override protected def configure(): Unit = { + bind[X].to[Y].in[Singleton] + } + } + + Injector(Guice.createInjector(module)).instance[X] + } + + test("allow binding a container with a generic singleton type") { + val module = new TestModule { + override protected def configure(): Unit = { + bind[SealedTraitContainer[FinalSealedTrait.type]] + .toProvider[SealedTraitContainerFinalSealedTraitProvider] + } + } + + Injector(Guice.createInjector(module)) + .instance(new Key[SealedTraitContainer[FinalSealedTrait.type]] {}) + } + + test("allow binding with annotation using a type parameter") { + val module = new TestModule { + override protected def configure(): Unit = { + bind[X].annotatedWith[Named].to[Y] + } + } + + Injector(Guice.createInjector(module)).instance(Key.get(classOf[X], classOf[Named])) + } + + test("allow use provider form javax.inject.Provider") { + val module = new TestModule { + override def configure(): Unit = { + bind[Foo].toProvider[FooProviderWithJavax] + } + } + + Injector(Guice.createInjector(module)).instance[Foo] + } + + test("provide helpful error when bound to self") { + val module = new TestModule { + override def configure(): Unit = { + bind[X].to[X] + } + } + val e = intercept[CreationException] { + Injector(Guice.createInjector(module)).instance[X] + } + val messages = e.getErrorMessages + messages.size should equal(1) + e.getStackTrace.exists(_.getClassName == this.getClass.getName) + val sources = messages.iterator.next.getSource + sources.contains(this.getClass.getName) should be(true) + } +} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala b/util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala new file mode 100644 index 0000000000..1c7cb6bbb4 --- /dev/null +++ b/util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala @@ -0,0 +1,46 @@ +package com.twitter.util.inject + +import org.scalacheck.Arbitrary +import org.scalatest.funsuite.AnyFunSuite +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import scala.reflect.runtime.universe._ + +trait A +trait B + +class TypeUtilsTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { + + test("asManifest handles AnyVal/Any/Null/Nothing") { + forAll(Arbitrary.arbAnyVal.arbitrary) { anyVal => + assert(manifestedTypesEquals(anyVal)) + } + assert(TypeUtils.asManifest[AnyVal] == manifest[AnyVal]) + assert(TypeUtils.asManifest[Any] == manifest[Any]) + assert(TypeUtils.asManifest[Null] == manifest[Null]) + assert(TypeUtils.asManifest[Nothing] == manifest[Nothing]) + } + + test("asManifest handles NoClassDefFound exceptions") { + val t = typeTag[A with B] + intercept[NoClassDefFoundError] { + t.mirror.runtimeClass(t.tpe) + } + assert(TypeUtils.asManifest[A with B] == manifest[Any]) + } + + test("asTypeTag handles classes") { + typeTagEquals(classOf[String], TypeUtils.asTypeTag(classOf[String])) + typeTagEquals(classOf[AnyVal], TypeUtils.asTypeTag(classOf[AnyVal])) + typeTagEquals(classOf[Any], TypeUtils.asTypeTag(classOf[Any])) + typeTagEquals(classOf[Null], TypeUtils.asTypeTag(classOf[Null])) + typeTagEquals(classOf[Nothing], TypeUtils.asTypeTag(classOf[Nothing])) + } + + def manifestedTypesEquals[T: TypeTag: Manifest](a: T): Boolean = { + TypeUtils.asManifest[T] == manifest[T] + } + + def typeTagEquals[T](clazz: Class[T], typeTag: TypeTag[T]): Boolean = { + clazz.isAssignableFrom(typeTag.mirror.runtimeClass(typeTag.tpe.typeSymbol.asClass)) + } +} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/classes.scala b/util-inject/src/test/scala/com/twitter/util/inject/classes.scala new file mode 100644 index 0000000000..86373267f2 --- /dev/null +++ b/util-inject/src/test/scala/com/twitter/util/inject/classes.scala @@ -0,0 +1,70 @@ +package com.twitter.util.inject + +import com.google.inject.{Provider, TypeLiteral} +import javax.inject.{Inject, Named} + +object classes { + object Outer { + trait X + class Y extends X + trait Gen[T] { + def get: T + } + + class C extends Gen[String] { + def get = "String" + } + } + + trait X + class Y extends X + + class YProvider extends Provider[Y] { + def get = new Y + } + + trait Gen[T] { + def get: T + } + + class TypeProvider[T] @Inject() (typ: TypeLiteral[T]) extends Provider[String] { + def get = typ.toString + } + + class Z extends Gen[String] { + def get = "String" + } + + class GenStringProvider extends Provider[Gen[String]] { + def get = new Z + } + + class ZProvider extends Provider[Z] { + def get = new Z + } + + trait Foo { + def foo(): String + } + + class FooProviderWithJavax extends javax.inject.Provider[Foo] { + def get(): Foo = new Foo { + def foo() = "foo" + } + } + + case class TwoStrings @Inject() (@Named("first") first: String, @Named("second") second: String) + + trait D + + sealed trait SealedTrait + + final case object FinalSealedTrait extends SealedTrait + + case class SealedTraitContainer[T <: SealedTrait](inner: T) + + class SealedTraitContainerFinalSealedTraitProvider + extends javax.inject.Provider[SealedTraitContainer[FinalSealedTrait.type]] { + def get(): SealedTraitContainer[FinalSealedTrait.type] = SealedTraitContainer(FinalSealedTrait) + } +} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala b/util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala new file mode 100644 index 0000000000..1ed54c9181 --- /dev/null +++ b/util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala @@ -0,0 +1,25 @@ +package com.twitter.util.inject.module + +import com.google.inject.AbstractModule +import com.google.inject.name.Names +import com.twitter.util.inject.{MyServiceImpl, MyServiceInterface, TestBindingAnnotation} +import java.util.Properties +import javax.inject.Singleton +import net.codingwell.scalaguice.ScalaModule + +trait TestModule extends AbstractModule with ScalaModule { + + override protected def configure(): Unit = { + bind[String].annotatedWith(Names.named("str1")).toInstance("string1") + bind[String].annotatedWith(Names.named("str2")).toInstance("string2") + + bind[Int].toInstance(11) + bind[String].toInstance("default string") + bind[String].annotatedWith[TestBindingAnnotation].toInstance("prod string") + + bind[MyServiceInterface].to[MyServiceImpl].in[Singleton] + val properties = new Properties() + properties.setProperty("name", "Steve") + Names.bindProperties(TestModule.super.binder(), properties) + } +} From 0ef5ea679123dfdbd50d2b8dc8111365506036c5 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 6 Aug 2020 19:30:19 +0000 Subject: [PATCH 361/796] util-stats: Fix InMemoryStatsReceiverTest so it passes on 2.13 Problem The test was broken on 2.13 as it relied on specific collection implementations (via toString). Solution The test is fixed now and build passes on 2.13. JIRA Issues: CSL-10070 Differential Revision: https://phabricator.twitter.biz/D528521 --- .../stats/InMemoryStatsReceiverTest.scala | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 2984fd618f..40b6cfd3d0 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -332,25 +332,24 @@ class InMemoryStatsReceiverTest extends FunSuite with Eventually with Integratio stats.addGauge("coolGauge") { 3 } stats.counter("sweetCounter") stats.stat("radHisto") + val gauges = Array("coolGauge").toSeq + val histograms = Array("radHisto").toSeq + val counters = Array("sweetCounter").toSeq val baos = new ByteArrayOutputStream() val ps = new PrintStream(baos, true, "utf-8") try { stats.printSchemas(ps) val content = new String(baos.toByteArray, StandardCharsets.UTF_8) - println(content) val parts = content.split('\n') assert(parts.length == 3) - assert( - parts( - 0) == "coolGauge GaugeSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, WrappedArray(coolGauge), None, Vector(), InMemoryStatsReceiver))") - assert( - parts( - 1) == "radHisto HistogramSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, WrappedArray(radHisto), None, Vector(), InMemoryStatsReceiver))") - assert( - parts( - 2) == "sweetCounter CounterSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, WrappedArray(sweetCounter), None, Vector(), InMemoryStatsReceiver))") + assert(parts( + 0) == s"coolGauge GaugeSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, $gauges, None, Vector(), InMemoryStatsReceiver))") + assert(parts( + 1) == s"radHisto HistogramSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, $histograms, None, Vector(), InMemoryStatsReceiver))") + assert(parts( + 2) == s"sweetCounter CounterSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, $counters, None, Vector(), InMemoryStatsReceiver))") } finally { ps.close() } From 8fcc7e0456a2a8fa37525bbbd75376fbb8a01ebc Mon Sep 17 00:00:00 2001 From: Marcy Reyes Date: Sat, 8 Aug 2020 00:14:07 +0000 Subject: [PATCH 362/796] Migrating Remaining OWNERS Files in source to PROJECT/PROJECT.yml We are finishing up our migration of OWNERS files in source to PROJECT files. In an effort to ensure PROJECT files are compatible with directories that already have project subdirectories, we are migrating ownership files in those particular directories to PROJECT.yml. For more information about PROJECT files, please see [[ https://docbird.twitter.biz/code-review/general/project_files.html | go/PROJECT ]]. JIRA Issues: CR-6197 Differential Revision: https://phabricator.twitter.biz/D528770 --- GROUPS | 1 - OWNERS | 6 ------ PROJECT.yml | 11 +++++++++++ util-app/OWNERS | 8 -------- util-app/PROJECT | 11 +++++++++++ util-benchmark/OWNERS | 7 ------- util-benchmark/PROJECT | 10 ++++++++++ util-codec/OWNERS | 6 ------ util-codec/PROJECT | 9 +++++++++ util-core/OWNERS | 7 ------- util-core/PROJECT | 10 ++++++++++ util-intellij/OWNERS | 7 ------- util-intellij/PROJECT | 10 ++++++++++ util-logging/OWNERS | 7 ------- util-logging/PROJECT | 10 ++++++++++ util-security/OWNERS | 6 ------ util-security/PROJECT | 9 +++++++++ util-slf4j-api/OWNERS | 6 ------ util-slf4j-api/PROJECT | 9 +++++++++ util-slf4j-jul-bridge/OWNERS | 6 ------ util-slf4j-jul-bridge/PROJECT | 9 +++++++++ util-stats/OWNERS | 6 ------ util-stats/PROJECT | 9 +++++++++ util-zk/GROUPS | 3 --- util-zk/OWNERS | 5 ----- util-zk/PROJECT | 12 ++++++++++++ 26 files changed, 119 insertions(+), 81 deletions(-) delete mode 100644 GROUPS delete mode 100644 OWNERS create mode 100644 PROJECT.yml delete mode 100644 util-app/OWNERS create mode 100644 util-app/PROJECT delete mode 100644 util-benchmark/OWNERS create mode 100644 util-benchmark/PROJECT delete mode 100644 util-codec/OWNERS create mode 100644 util-codec/PROJECT delete mode 100644 util-core/OWNERS create mode 100644 util-core/PROJECT delete mode 100644 util-intellij/OWNERS create mode 100644 util-intellij/PROJECT delete mode 100644 util-logging/OWNERS create mode 100644 util-logging/PROJECT delete mode 100644 util-security/OWNERS create mode 100644 util-security/PROJECT delete mode 100644 util-slf4j-api/OWNERS create mode 100644 util-slf4j-api/PROJECT delete mode 100644 util-slf4j-jul-bridge/OWNERS create mode 100644 util-slf4j-jul-bridge/PROJECT delete mode 100644 util-stats/OWNERS create mode 100644 util-stats/PROJECT delete mode 100644 util-zk/GROUPS delete mode 100644 util-zk/OWNERS create mode 100644 util-zk/PROJECT diff --git a/GROUPS b/GROUPS deleted file mode 100644 index e57fe3a541..0000000000 --- a/GROUPS +++ /dev/null @@ -1 +0,0 @@ -csl diff --git a/OWNERS b/OWNERS deleted file mode 100644 index 477af14021..0000000000 --- a/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -banderson -koliver -mnakamura -roanta -ryano -vkostyukov \ No newline at end of file diff --git a/PROJECT.yml b/PROJECT.yml new file mode 100644 index 0000000000..0b31cc64f9 --- /dev/null +++ b/PROJECT.yml @@ -0,0 +1,11 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - koliver + - mnakamura + - roanta + - ryano + - vkostyukov +watchers: + - csl@twitter.com diff --git a/util-app/OWNERS b/util-app/OWNERS deleted file mode 100644 index ddd4683efe..0000000000 --- a/util-app/OWNERS +++ /dev/null @@ -1,8 +0,0 @@ -banderson -ccoco -ibennett -koliver -mnakamura -roanta -ryano -vkostyukov diff --git a/util-app/PROJECT b/util-app/PROJECT new file mode 100644 index 0000000000..01c397156f --- /dev/null +++ b/util-app/PROJECT @@ -0,0 +1,11 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - ccoco + - ibennett + - koliver + - mnakamura + - roanta + - ryano + - vkostyukov diff --git a/util-benchmark/OWNERS b/util-benchmark/OWNERS deleted file mode 100644 index 24a8b106dd..0000000000 --- a/util-benchmark/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -banderson -koliver -mnakamura -roanta -ryano -vkostyukov -yufang diff --git a/util-benchmark/PROJECT b/util-benchmark/PROJECT new file mode 100644 index 0000000000..8a8fabfcdc --- /dev/null +++ b/util-benchmark/PROJECT @@ -0,0 +1,10 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - koliver + - mnakamura + - roanta + - ryano + - vkostyukov + - yufang diff --git a/util-codec/OWNERS b/util-codec/OWNERS deleted file mode 100644 index 95b52649d9..0000000000 --- a/util-codec/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -banderson -mnakamura -roanta -ryano -vkostyukov -yufang diff --git a/util-codec/PROJECT b/util-codec/PROJECT new file mode 100644 index 0000000000..b15a45699b --- /dev/null +++ b/util-codec/PROJECT @@ -0,0 +1,9 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - mnakamura + - roanta + - ryano + - vkostyukov + - yufang diff --git a/util-core/OWNERS b/util-core/OWNERS deleted file mode 100644 index 24a8b106dd..0000000000 --- a/util-core/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -banderson -koliver -mnakamura -roanta -ryano -vkostyukov -yufang diff --git a/util-core/PROJECT b/util-core/PROJECT new file mode 100644 index 0000000000..8a8fabfcdc --- /dev/null +++ b/util-core/PROJECT @@ -0,0 +1,10 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - koliver + - mnakamura + - roanta + - ryano + - vkostyukov + - yufang diff --git a/util-intellij/OWNERS b/util-intellij/OWNERS deleted file mode 100644 index 450160e1be..0000000000 --- a/util-intellij/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -banderson -ccoco -ibennett -mnakamura -roanta -ryano -vkostyukov diff --git a/util-intellij/PROJECT b/util-intellij/PROJECT new file mode 100644 index 0000000000..141b6376e6 --- /dev/null +++ b/util-intellij/PROJECT @@ -0,0 +1,10 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - ccoco + - ibennett + - mnakamura + - roanta + - ryano + - vkostyukov diff --git a/util-logging/OWNERS b/util-logging/OWNERS deleted file mode 100644 index 450160e1be..0000000000 --- a/util-logging/OWNERS +++ /dev/null @@ -1,7 +0,0 @@ -banderson -ccoco -ibennett -mnakamura -roanta -ryano -vkostyukov diff --git a/util-logging/PROJECT b/util-logging/PROJECT new file mode 100644 index 0000000000..141b6376e6 --- /dev/null +++ b/util-logging/PROJECT @@ -0,0 +1,10 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - ccoco + - ibennett + - mnakamura + - roanta + - ryano + - vkostyukov diff --git a/util-security/OWNERS b/util-security/OWNERS deleted file mode 100644 index 5b1829a750..0000000000 --- a/util-security/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -banderson -bdimmick -mnakamura -roanta -ryano -vkostyukov diff --git a/util-security/PROJECT b/util-security/PROJECT new file mode 100644 index 0000000000..4b4968d8d4 --- /dev/null +++ b/util-security/PROJECT @@ -0,0 +1,9 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - bdimmick + - mnakamura + - roanta + - ryano + - vkostyukov diff --git a/util-slf4j-api/OWNERS b/util-slf4j-api/OWNERS deleted file mode 100644 index 9a1030a1b3..0000000000 --- a/util-slf4j-api/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -banderson -ccoco -ibennett -mnakamura -roanta -ryano diff --git a/util-slf4j-api/PROJECT b/util-slf4j-api/PROJECT new file mode 100644 index 0000000000..32d03a0ee9 --- /dev/null +++ b/util-slf4j-api/PROJECT @@ -0,0 +1,9 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - ccoco + - ibennett + - mnakamura + - roanta + - ryano diff --git a/util-slf4j-jul-bridge/OWNERS b/util-slf4j-jul-bridge/OWNERS deleted file mode 100644 index 9a1030a1b3..0000000000 --- a/util-slf4j-jul-bridge/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -banderson -ccoco -ibennett -mnakamura -roanta -ryano diff --git a/util-slf4j-jul-bridge/PROJECT b/util-slf4j-jul-bridge/PROJECT new file mode 100644 index 0000000000..32d03a0ee9 --- /dev/null +++ b/util-slf4j-jul-bridge/PROJECT @@ -0,0 +1,9 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - ccoco + - ibennett + - mnakamura + - roanta + - ryano diff --git a/util-stats/OWNERS b/util-stats/OWNERS deleted file mode 100644 index b749f10796..0000000000 --- a/util-stats/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -banderson -koliver -mnakamura -roanta -ryano -vkostyukov diff --git a/util-stats/PROJECT b/util-stats/PROJECT new file mode 100644 index 0000000000..244fc53688 --- /dev/null +++ b/util-stats/PROJECT @@ -0,0 +1,9 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - koliver + - mnakamura + - roanta + - ryano + - vkostyukov diff --git a/util-zk/GROUPS b/util-zk/GROUPS deleted file mode 100644 index 82a0574ba7..0000000000 --- a/util-zk/GROUPS +++ /dev/null @@ -1,3 +0,0 @@ -coord-team -csl -observability diff --git a/util-zk/OWNERS b/util-zk/OWNERS deleted file mode 100644 index 10222fe988..0000000000 --- a/util-zk/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -banderson -mnakamura -roanta -ryano -vkostyukov diff --git a/util-zk/PROJECT b/util-zk/PROJECT new file mode 100644 index 0000000000..646a189266 --- /dev/null +++ b/util-zk/PROJECT @@ -0,0 +1,12 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - mnakamura + - roanta + - ryano + - vkostyukov +watchers: + - coord-team@twitter.com + - csl@twitter.com + - observability@twitter.com From c213046946ae611b3bb37e6e70fe270191f1e6f2 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 11 Aug 2020 02:53:48 +0000 Subject: [PATCH 363/796] Twitter-oss: Prepare OSS libraries for release 20.8.0 Problem We want to release the next versions of our Twitter OSS libraries 20.8.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D530623 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 853284dcb6..fd6b32ad73 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.8.0 +------ + New Features ~~~~~~~~~~~~ * util-stats: Store MetricSchemas in InMemoryStatsReceiver to enable further testing. ``PHAB_ID=D518195`` diff --git a/README.md b/README.md index 64557283bf..ecdef08697 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.7.0" +val utilCore = "com.twitter" %% "util-core" % "20.8.0" ``` # Units diff --git a/build.sbt b/build.sbt index 2b85fd52c5..25f3b7f2b1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.8.0-SNAPSHOT" +val releaseVersion = "20.8.0" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From 5928e3d3af03d1e7a6ea697aee119c7e38ab48d3 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 11 Aug 2020 18:26:35 +0000 Subject: [PATCH 364/796] Twitter-oss: update OSS libraries post-release to 20.9.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.9.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D531226 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ecdef08697..d55c2e55a8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 25f3b7f2b1..625ca670d8 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.8.0" +val releaseVersion = "20.9.0-SNAPSHOT" val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" From b2fb4b3a9e063a2183d4f14de4ae04f5e1b2abb2 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 11 Aug 2020 20:49:42 +0000 Subject: [PATCH 365/796] util-mock: Introduce mockito-scala based mocking library Problem We'd like to have "idiomatic" Mockito stubbing available as a library for tests to use. We'd also like to update the current Mockito dependency where possible to a later version. Solution Add util/util-mock which defines `c.t.mock.Mockito` for idiomatic stubbing with mockito. Update all Mockito dependencies except for util/util-test which defines a mocking adaptation for argument capture with Mockito. Result A new mockito-scala based idiomatic mocking library which others can use. Almost all code updated to a more recent version of Mockito. Differential Revision: https://phabricator.twitter.biz/D530995 --- CHANGELOG.rst | 8 + build.sbt | 107 ++- util-app/src/test/scala/BUILD | 4 +- util-cache/src/test/java/BUILD | 2 +- .../src/test/scala/com/twitter/cache/BUILD | 4 +- .../cache/LazilyEvictingCacheTest.scala | 2 +- .../scala/com/twitter/cache/caffeine/BUILD | 4 +- .../test/scala/com/twitter/concurrent/BUILD | 4 +- util-core/src/test/scala/com/twitter/io/BUILD | 4 +- .../src/test/scala/com/twitter/util/BUILD | 4 +- .../scala/com/twitter/util/FutureTest.scala | 2 +- .../scala/com/twitter/util/MonitorTest.scala | 2 +- .../scala/com/twitter/util/TimerTest.scala | 2 +- util-jvm/src/test/scala/BUILD | 5 +- .../test/scala/com/twitter/jvm/JvmTest.scala | 2 +- .../test/scala/com/twitter/util/lint/BUILD | 4 +- util-logging/src/test/scala/BUILD | 2 +- util-mock/PROJECT | 11 + .../src/main/scala/com/twitter/mock/BUILD | 17 + .../main/scala/com/twitter/mock/Mockito.scala | 167 ++++ .../src/test/java/com/twitter/mock/BUILD | 7 + .../test/java/com/twitter/mock/JavaFoo.java | 5 + .../src/test/scala/com/twitter/mock/BUILD | 14 + .../com/twitter/mock/MockitoTraitTest.scala | 728 ++++++++++++++++++ .../mock/MockitoTraitWithFixtureTest.scala | 50 ++ .../ResetMocksAfterEachTestTraitTest.scala | 89 +++ util-registry/src/test/scala/BUILD | 4 +- .../twitter/util/registry/RosterTest.scala | 2 +- .../test/scala/com/twitter/util/routing/BUILD | 4 +- .../test/scala/com/twitter/util/logging/BUILD | 4 +- .../twitter/util/logging/LoggingTest.scala | 2 +- .../util/logging/MarkerLoggingTest.scala | 2 +- util-stats/src/test/scala/BUILD | 2 +- util-zk/src/test/scala/com/twitter/zk/BUILD | 4 +- .../scala/com/twitter/zk/ZkClientTest.scala | 2 +- .../scala/com/twitter/zk/coordination/BUILD | 4 +- 36 files changed, 1209 insertions(+), 71 deletions(-) create mode 100644 util-mock/PROJECT create mode 100644 util-mock/src/main/scala/com/twitter/mock/BUILD create mode 100644 util-mock/src/main/scala/com/twitter/mock/Mockito.scala create mode 100644 util-mock/src/test/java/com/twitter/mock/BUILD create mode 100644 util-mock/src/test/java/com/twitter/mock/JavaFoo.java create mode 100644 util-mock/src/test/scala/com/twitter/mock/BUILD create mode 100644 util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala create mode 100644 util-mock/src/test/scala/com/twitter/mock/MockitoTraitWithFixtureTest.scala create mode 100644 util-mock/src/test/scala/com/twitter/mock/ResetMocksAfterEachTestTraitTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fd6b32ad73..674416b555 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,14 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-mock: Introduce `mockito-scala `__ based mocking + integration. Fix up and update mockito testing dependencies: + - `mockito-all:1.10.19` to `mockito-core:3.3.3` + - `scalatestplus:mockito-1-10:3.1.0.0` to `scalatestplus:mockito-3-2:3.1.2.0` ``PHAB_ID=D530995`` + 20.8.0 ------ diff --git a/build.sbt b/build.sbt index 625ca670d8..a7fee36057 100644 --- a/build.sbt +++ b/build.sbt @@ -3,6 +3,11 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "20.9.0-SNAPSHOT" +val slf4jVersion = "1.7.30" +val jacksonVersion = "2.11.0" +val mockitoVersion = "3.3.3" +val mockitoScalaVersion = "1.14.8" + val zkVersion = "3.5.0-alpha" val zkClientVersion = "0.0.81" val zkGroupVersion = "0.0.92" @@ -11,8 +16,6 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll ( ExclusionRule("com.sun.jmx", "jmxri"), ExclusionRule("javax.jms", "jms") ) -val slf4jVersion = "1.7.30" -val jacksonVersion = "2.11.0" val guavaLib = "com.google.guava" % "guava" % "25.1-jre" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.5" @@ -88,10 +91,8 @@ val baseSettings = Seq( "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.2", // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", - "org.mockito" % "mockito-all" % "1.10.19" % "test", "org.scalatest" %% "scalatest" % "3.1.2" % "test", - "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test", - "org.scalatestplus" %% "mockito-1-10" % "3.1.0.0" % "test" + "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test" ), fork in Test := true, // We have to fork to get the JavaOptions // Workaround for cross building HealthyQueue.scala, which is not compatible between @@ -200,6 +201,7 @@ lazy val util = Project( utilJvm, utilLint, utilLogging, + utilMock, utilReflect, utilRegistry, utilRouting, @@ -220,17 +222,21 @@ lazy val utilApp = Project( ).settings( sharedSettings ).settings( - name := "util-app" + name := "util-app", + libraryDependencies ++= Seq( + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + ) ).dependsOn(utilAppLifecycle, utilCore, utilRegistry) lazy val utilAppLifecycle = Project( id = "util-app-lifecycle", base = file("util-app-lifecycle") ).settings( - sharedSettings -).settings( - name := "util-app-lifecycle" -).dependsOn(utilCore) + sharedSettings + ).settings( + name := "util-app-lifecycle" + ).dependsOn(utilCore) lazy val utilBenchmark = Project( id = "util-benchmark", @@ -250,7 +256,12 @@ lazy val utilCache = Project( sharedSettings ).settings( name := "util-cache", - libraryDependencies ++= Seq(caffeineLib, jsr305Lib) + libraryDependencies ++= Seq( + caffeineLib, + jsr305Lib, + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + ) ).dependsOn(utilCore) lazy val utilCacheGuava = Project( @@ -296,6 +307,8 @@ lazy val utilCore = Project( scalacheckLib, "org.scala-lang" % "scala-reflect" % scalaVersion.value, "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2", + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ), resourceGenerators in Compile += Def.task { @@ -381,8 +394,12 @@ lazy val utilJvm = Project( ).settings( sharedSettings ).settings( - name := "util-jvm" - ).dependsOn(utilApp, utilCore, utilStats, utilTest % "test") + name := "util-jvm", + libraryDependencies ++= Seq( + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + ) + ).dependsOn(utilApp, utilCore, utilStats) lazy val utilLint = Project( id = "util-lint", @@ -402,6 +419,42 @@ lazy val utilLogging = Project( name := "util-logging" ).dependsOn(utilCore, utilApp, utilStats) +lazy val utilMock = Project( + id = "util-mock", + base = file("util-mock") +).settings( + sharedSettings + ).settings( + name := "util-mock", + libraryDependencies ++= Seq( + // only depend on mockito-scala; it will pull in the correct corresponding version of mockito. + "org.mockito" %% "mockito-scala" % mockitoScalaVersion, + "org.mockito" %% "mockito-scala-scalatest" % mockitoScalaVersion % "test" + ) + ).dependsOn(utilCore % "test") + +lazy val utilRegistry = Project( + id = "util-registry", + base = file("util-registry") +).settings( + sharedSettings + ).settings( + name := "util-registry", + libraryDependencies ++= Seq( + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + ) + ).dependsOn(utilCore) + +lazy val utilRouting = Project( + id = "util-routing", + base = file("util-routing") +).settings( + sharedSettings + ).settings( + name := "util-routing" + ).dependsOn(utilCore, utilSlf4jApi) + lazy val utilSlf4jApi = Project( id = "util-slf4j-api", base = file("util-slf4j-api") @@ -411,6 +464,8 @@ lazy val utilSlf4jApi = Project( name := "util-slf4j-api", libraryDependencies ++= Seq( slf4jApi, + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test", "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" ) ).dependsOn(utilCore % "test") @@ -425,15 +480,6 @@ lazy val utilSlf4jJulBridge = Project( libraryDependencies ++= Seq(slf4jApi, "org.slf4j" % "jul-to-slf4j" % slf4jVersion) ).dependsOn(utilCore, utilSlf4jApi) -lazy val utilRegistry = Project( - id = "util-registry", - base = file("util-registry") -).settings( - sharedSettings - ).settings( - name := "util-registry" - ).dependsOn(utilCore) - lazy val utilSecurity = Project( id = "util-security", base = file("util-security") @@ -458,6 +504,8 @@ lazy val utilStats = Project( caffeineLib, jsr305Lib, scalacheckLib, + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ) ++ { CrossVersion.partialVersion(scalaVersion.value) match { @@ -469,15 +517,6 @@ lazy val utilStats = Project( } ).dependsOn(utilCore, utilLint) -lazy val utilRouting = Project( - id = "util-routing", - base = file("util-routing") -).settings( - sharedSettings - ).settings( - name := "util-routing" - ).dependsOn(utilCore, utilSlf4jApi) - lazy val utilTest = Project( id = "util-test", base = file("util-test") @@ -529,7 +568,11 @@ lazy val utilZk = Project( sharedSettings ).settings( name := "util-zk", - libraryDependencies += zkDependency + libraryDependencies ++= Seq( + zkDependency, + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + ) ).dependsOn(utilCore, utilLogging) lazy val utilZkTest = Project( diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index cfd289bc65..1750c7515f 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -3,10 +3,10 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "util/util-app/src/main/scala", "util/util-app/src/test/resources", "util/util-core:util-core-util", diff --git a/util-cache/src/test/java/BUILD b/util-cache/src/test/java/BUILD index 5a0ea76ebc..d5d9a6b412 100644 --- a/util-cache/src/test/java/BUILD +++ b/util-cache/src/test/java/BUILD @@ -3,7 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scala-lang:scala-library", "3rdparty/jvm/org/scalatestplus:junit", "util/util-cache/src/main/scala/com/twitter/cache", diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index 41ce69098d..1af0fddf95 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -9,10 +9,10 @@ junit_tests( ":abstract_tests", "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-core:util-core-util", diff --git a/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala index e4cf2cf812..6947b61c0d 100644 --- a/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala @@ -3,7 +3,7 @@ package com.twitter.cache import com.github.benmanes.caffeine.cache.{CacheLoader, Caffeine} import com.twitter.cache.caffeine.LoadingFutureCache import com.twitter.util.{Await, Future, Promise} -import org.mockito.Matchers._ +import org.mockito.ArgumentMatchers._ import org.mockito.Mockito.{never, verify, when} import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index 851da83281..fc69ccd618 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -5,10 +5,10 @@ junit_tests( "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/com/google/code/findbugs:jsr305", "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index b60d3d986d..b0fd41e8f9 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -3,11 +3,11 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 6d1367e4f4..2092607878 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -3,11 +3,11 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index 56199e4890..8f2b611846 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -3,11 +3,11 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index cf67ca7886..c0b29829b9 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -3,7 +3,7 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import java.util.concurrent.{ConcurrentLinkedQueue, ExecutionException, Future => JFuture} import java.util.concurrent.atomic.AtomicInteger -import org.mockito.Matchers.any +import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{never, times, verify, when} import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer diff --git a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala index 35aeeddb1c..e6d6b0913a 100644 --- a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala @@ -1,6 +1,6 @@ package com.twitter.util -import org.mockito.Matchers._ +import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar diff --git a/util-core/src/test/scala/com/twitter/util/TimerTest.scala b/util-core/src/test/scala/com/twitter/util/TimerTest.scala index 02f7382164..7776d57fe5 100644 --- a/util-core/src/test/scala/com/twitter/util/TimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimerTest.scala @@ -4,7 +4,7 @@ import com.twitter.conversions.DurationOps._ import java.util.concurrent.{CancellationException, ExecutorService, TimeUnit, CountDownLatch} import java.util.concurrent.atomic.AtomicInteger import org.mockito.ArgumentCaptor -import org.mockito.Matchers.any +import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{never, verify, when} import org.scalatest.FunSuite import org.scalatest.concurrent.{IntegrationPatience, Eventually} diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index d7258d2dfb..a6581d7a1f 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -1,13 +1,12 @@ COMMON_DEPS = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "util/util-app/src/main/scala", "util/util-core:util-core-util", "util/util-jvm/src/main/scala/com/twitter/jvm", "util/util-logging/src/main/scala", - "util/util-test/src/main/scala", ] junit_tests( diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala index 28e737b8de..e050f19d45 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala @@ -3,7 +3,7 @@ package com.twitter.jvm import com.twitter.conversions.DurationOps._ import com.twitter.util.Time import java.util.logging.{Level, Logger} -import org.mockito.Matchers.contains +import org.mockito.ArgumentMatchers.contains import org.mockito.Mockito.{times, verify, when} import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index 6dae7fa1ff..7099a7af87 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -3,10 +3,10 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "util/util-lint/src/main/scala/com/twitter/util/lint", ], ) diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 479b5b021f..30b94630a2 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -3,7 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "util/util-app/src/main/scala", diff --git a/util-mock/PROJECT b/util-mock/PROJECT new file mode 100644 index 0000000000..791a30f039 --- /dev/null +++ b/util-mock/PROJECT @@ -0,0 +1,11 @@ +owners: + - banderson + - ccoco + - ibennett + - koliver + - mnakamura + - roanta + - ryano + - vkostyukov +watchers: + - csl-team:ldap diff --git a/util-mock/src/main/scala/com/twitter/mock/BUILD b/util-mock/src/main/scala/com/twitter/mock/BUILD new file mode 100644 index 0000000000..3e1050d2a9 --- /dev/null +++ b/util-mock/src/main/scala/com/twitter/mock/BUILD @@ -0,0 +1,17 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-mockito", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/org/mockito:mockito-core", + "3rdparty/jvm/org/mockito:mockito-scala", + ], + exports = [ + "3rdparty/jvm/org/mockito:mockito-scala", + ], +) diff --git a/util-mock/src/main/scala/com/twitter/mock/Mockito.scala b/util-mock/src/main/scala/com/twitter/mock/Mockito.scala new file mode 100644 index 0000000000..1ff639b676 --- /dev/null +++ b/util-mock/src/main/scala/com/twitter/mock/Mockito.scala @@ -0,0 +1,167 @@ +package com.twitter.mock + +import org.mockito.{ArgumentMatchersSugar, IdiomaticMockito} + +/** + * Helper for Mockito Scala sugar with [[https://github.com/mockito/mockito-scala#idiomatic-mockito idiomatic stubbing]]. + * Java users are encouraged to use [[org.mockito.Mockito]] directly. + * + * Note that the Specs2 `smartMock[]` or `mock[].smart` is the default behavior + * for [[https://github.com/mockito/mockito-scala Mockito Scala]]. + * + * =Usage= + * + * This trait uses [[org.mockito.IdiomaticMockito]] which is heavily influenced by ScalaTest Matchers. + * + * To use, mix in the [[com.twitter.mock.Mockito]] trait where desired. + * + * ==Create a new mock== + * + * {{{ + * + * trait Foo { + * def bar: String + * def bar(v: Int): Int + * } + * + * class MyTest extends AnyFunSuite with Mockito { + * val aMock = mock[Foo] + * } + * }}} + * + * ==Expect behavior== + * + * {{{ + * // "when" equivalents + * aMock.bar returns "mocked!" + * aMock.bar returns "mocked!" andThen "mocked again!" + * aMock.bar shouldCall realMethod + * aMock.bar.shouldThrow[IllegalArgumentException] + * aMock.bar throws new IllegalArgumentException + * aMock.bar answers "mocked!" + * aMock.bar(*) answers ((i: Int) => i * 10) + * + * // "do-when" equivalents + * "mocked!" willBe returned by aMock.bar + * "mocked!" willBe answered by aMock.bar + * ((i: Int) => i * 10) willBe answered by aMock.bar(*) + * theRealMethod willBe called by aMock.bar + * new IllegalArgumentException willBe thrown by aMock.bar + * aMock.bar.doesNothing() // doNothing().when(aMock).bar + * + * // verifications + * aMock wasNever called // verifyZeroInteractions(aMock) + * aMock.bar was called + * aMock.bar(*) was called // '*' is shorthand for 'any()' or 'any[T]' + * aMock.bar(any[Int]) was called // same as above but with typed input matcher + * + * aMock.bar wasCalled onlyHere + * aMock.bar wasNever called + * + * aMock.bar wasCalled twice + * aMock.bar wasCalled 2.times + * + * aMock.bar wasCalled fourTimes + * aMock.bar wasCalled 4.times + * + * aMock.bar wasCalled atLeastFiveTimes + * aMock.bar wasCalled atLeast(fiveTimes) + * aMock.bar wasCalled atLeast(5.times) + * + * aMock.bar wasCalled atMostSixTimes + * aMock.bar wasCalled atMost(sixTimes) + * aMock.bar wasCalled atMost(6.times) + * + * aMock.bar wasCalled (atLeastSixTimes within 2.seconds) // verify(aMock, timeout(2000).atLeast(6)).bar + * + * aMock wasNever calledAgain // verifyNoMoreInteractions(aMock) + * + * InOrder(mock1, mock2) { implicit order => + * mock2.someMethod() was called + * mock1.anotherMethod() was called + * } + * }}} + * + * Note the 'dead code' warning that can happen when using 'any' or '*' + * [[https://github.com/mockito/mockito-scala#dead-code-warning matchers]]. + * + * ==Mixing and matching matchers== + * + * Using the idiomatic syntax also allows for mixing argument matchers with real values. E.g., you + * are no longer forced to use argument matchers for all parameters as soon as you use one. E.g., + * + * {{{ + * trait Foo { + * def bar(v: Int, v2: Int, v3: Int = 42): Int + * } + * + * class MyTest extends AnyFunSuite with Mockito { + * val aMock = mock[Foo] + * + * aMock.bar(1, 2) returns "mocked!" + * aMock.bar(1, *) returns "mocked!" + * aMock.bar(1, any[Int]) returns "mocked!" + * aMock.bar(*, *) returns "mocked!" + * aMock.bar(any[Int], any[Int]) returns "mocked!" + * aMock.bar(*, *, 3) returns "mocked!" + * aMock.bar(any[Int], any[Int], 3) returns "mocked!" + * + * "mocked!" willBe returned by aMock.bar(1, 2) + * "mocked!" willBe returned by aMock.bar(1, *) + * "mocked!" willBe returned by aMock.bar(1, any[Int]) + * "mocked!" willBe returned by aMock.bar(*, *) + * "mocked!" willBe returned by aMock.bar(any[Int], any[Int]) + * "mocked!" willBe returned by aMock.bar(*, *, 3) + * "mocked!" willBe returned by aMock.bar(any[Int], any[Int], 3) + * + * aMock.bar(1, 2) was called + * aMock.bar(1, *) was called + * aMock.bar(1, any[Int]) was called + * aMock.bar(*, *) was called + * aMock.bar(any[Int], any[Int]) was called + * aMock.bar(*, *, 3) was called + * aMock.bar(any[Int], any[Int], 3) was called + * } + * }}} + * + * See [[https://github.com/mockito/mockito-scala#mixing-normal-values-with-argument-matchers Mix-and-Match]] + * for more information including a caveat around curried functions with default arguments. + * + * ==Numeric Matchers== + * + * Numeric comparisons are possible for argument matching, e.g., + * + * {{{ + * aMock.method(5) + * + * aMock.method(n > 4.99) was called + * aMock.method(n >= 5) was called + * aMock.method(n < 5.1) was called + * aMock.method(n <= 5) was called + * }}} + * + * See [[https://github.com/mockito/mockito-scala#numeric-matchers Numeric Matchers]]. + * + * ==Vargargs== + * + * Most matches will deal with varargs out of the box, just note when using the 'eqTo' matcher to + * apply it to all the arguments as one (not individually). + * + * See [[https://github.com/mockito/mockito-scala#varargs Varargs]]. + * + * ==More Information== + * + * See the [[https://github.com/mockito/mockito-scala#idiomatic-mockito IdiomaticMockito]] documentation + * for more specific information and the [[https://github.com/mockito/mockito-scala Mockito Scala]] + * [[https://github.com/mockito/mockito-scala#getting-started Getting Started]] documentation for general + * information. + * + * @see [[org.mockito.IdiomaticMockito]] + * @see [[org.mockito.ArgumentMatchersSugar]] + */ +trait Mockito extends IdiomaticMockito with ArgumentMatchersSugar + +/** + * Simple object to allow the usage of the trait without mixing it in + */ +object Mockito extends Mockito diff --git a/util-mock/src/test/java/com/twitter/mock/BUILD b/util-mock/src/test/java/com/twitter/mock/BUILD new file mode 100644 index 0000000000..f81ed67772 --- /dev/null +++ b/util-mock/src/test/java/com/twitter/mock/BUILD @@ -0,0 +1,7 @@ +java_library( + sources = ["*.java"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = "True", + dependencies = [ + ], +) diff --git a/util-mock/src/test/java/com/twitter/mock/JavaFoo.java b/util-mock/src/test/java/com/twitter/mock/JavaFoo.java new file mode 100644 index 0000000000..3eae8a1c32 --- /dev/null +++ b/util-mock/src/test/java/com/twitter/mock/JavaFoo.java @@ -0,0 +1,5 @@ +package com.twitter.mock; + +public interface JavaFoo { + Integer varargMethod(Integer... arg); +} diff --git a/util-mock/src/test/scala/com/twitter/mock/BUILD b/util-mock/src/test/scala/com/twitter/mock/BUILD new file mode 100644 index 0000000000..3a88794a25 --- /dev/null +++ b/util-mock/src/test/scala/com/twitter/mock/BUILD @@ -0,0 +1,14 @@ +junit_tests( + sources = ["**/*.scala"], + compiler_option_sets = ["fatal_warnings"], + dependencies = [ + "3rdparty/jvm/org/mockito:mockito-core", + "3rdparty/jvm/org/mockito:mockito-scala", + "3rdparty/jvm/org/mockito:mockito-scala-scalatest", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "util/util-core:scala", + "util/util-mock/src/main/scala/com/twitter/mock", + "util/util-mock/src/test/java/com/twitter/mock", + ], +) diff --git a/util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala b/util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala new file mode 100644 index 0000000000..30af1d5858 --- /dev/null +++ b/util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala @@ -0,0 +1,728 @@ +package com.twitter.mock + +import com.twitter.conversions.DurationOps._ +import com.twitter.util.{Return, Throw} +import com.twitter.util.{Await, Awaitable, Future, Monitor} +import java.io.{File, FileOutputStream, ObjectOutputStream} +import java.util.concurrent.atomic.AtomicInteger +import org.junit.runner.RunWith +import org.mockito.MockitoSugar +import org.mockito.captor.ArgCaptor +import org.mockito.exceptions.verification.WantedButNotInvoked +import org.mockito.invocation.InvocationOnMock +import org.mockito.stubbing.{CallsRealMethods, DefaultAnswer, ScalaFirstStubbing} +import org.scalatest.concurrent.ScalaFutures.{FutureConcept, whenReady} +import org.scalatestplus.junit.JUnitRunner +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import scala.annotation.varargs +import scala.language.implicitConversions +import scala.reflect.io.AbstractFile + +object MockitoTraitTest { + + /* Test classes pulled from MockitoScala: + https://github.com/mockito/mockito-scala/blob/release/1.x/scalatest/src/test/scala/user/org/mockito/TestModel.scala*/ + + class X { + def bar = "not mocked" + def iHaveByNameArgs(normal: String, byName: => String, byName2: => String): String = + "not mocked" + def iHaveSomeDefaultArguments(noDefault: String, default: String = "default value"): String = + "not mocked" + def iStartWithByNameArgs(byName: => String, normal: String): String = "not mocked" + def iHavePrimitiveByNameArgs(byName: => Int, normal: String): String = "not mocked" + def iHaveByNameAndVarArgs( + byName: => String, + normal: String, + byName2: => String, + normal2: String, + vararg: String* + )( + byName3: => String, + normal3: String, + vararg3: String* + ): String = "not mocked" + + def iHaveFunction0Args(normal: String, f0: () => String): String = "not mocked" + def iHaveByNameAndFunction0Args(normal: String, f0: () => String, byName: => String): String = + "not mocked" + def returnBar: Bar = new Bar + def doSomethingWithThisIntAndString(v: Int, v2: String): ValueCaseClassInt = ValueCaseClassInt( + v) + def returnsValueCaseClass: ValueCaseClassInt = ValueCaseClassInt(-1) + def returnsValueCaseClass(i: Int): ValueCaseClassInt = ValueCaseClassInt(i) + def baz(i: Int, b: Baz2): String = "not mocked" + } + + trait Y { + def varargMethod(s: String, arg: Int*): Int = -1 + def varargMethod(arg: Int*): Int = -1 + @varargs def javaVarargMethod(arg: Int*): Int = -1 + def byNameMethod(arg: => Int): Int = -1 + def traitMethod(arg: Int): ValueCaseClassInt = ValueCaseClassInt(arg) + def traitMethodWithDefaultArgs(defaultArg: Int = 30, anotherDefault: String = "hola"): Int = -1 + } + + class XWithY extends X with Y + + class Complex extends X + + class Foo { + def bar(a: String) = "bar" + } + + class Foo2 { + def valueClass(n: Int, v: ValueClass): String = ??? + def valueCaseClass(n: Int, v: ValueCaseClassInt): String = ??? + } + + class Bar { + def iAlsoHaveSomeDefaultArguments( + noDefault: String, + default: String = "default value" + ): String = "not mocked" + def iHaveDefaultArgs(v: String = "default"): String = "not mocked" + } + + trait Baz { + def varargMethod(s: String, arg: Int*): Int = -1 + def varargMethod(arg: Int*): Int = -1 + @varargs def javaVarargMethod(arg: Int*): Int = -1 + def byNameMethod(arg: => Int): Int = -1 + def traitMethod(arg: Int): ValueCaseClassInt = ValueCaseClassInt(arg) + def traitMethodWithDefaultArgs(defaultArg: Int = 30, anotherDefault: String = "hola"): Int = -1 + } + + class ValueClass(val v: String) extends AnyVal { + override def toString = s"ValueClass($v)" + } + case class ValueCaseClassInt(v: Int) extends AnyVal + case class ValueCaseClassString(v: String) extends AnyVal + case class GenericValue[T](v: Int) extends AnyVal + + class HigherKinded[F[_]] { + def method: F[Either[String, String]] = null.asInstanceOf[F[Either[String, String]]] + def method2: F[Either[String, String]] = null.asInstanceOf[F[Either[String, String]]] + } + + class ConcreteHigherKinded extends HigherKinded[Option] + + class Implicit[T] + + class Org { + def bar = "not mocked" + def baz = "not mocked" + + def defaultParams(a: String, defaultParam1: Boolean = false, defaultParam2: Int = 1): String = + "not mocked" + def curriedDefaultParams( + a: String, + defaultParam1: Boolean = false + )( + defaultParam2: Int = a.length + ): String = "not mocked" + + def doSomethingWithThisInt(v: Int): Int = -1 + def doSomethingWithThisIntAndString(v: Int, v2: String): String = "not mocked" + def doSomethingWithThisIntAndStringAndBoolean(v: Int, v2: String, v3: Boolean): String = + "not mocked" + def returnBar: Bar = new Bar + def highOrderFunction(f: Int => String): String = "not mocked" + def iReturnAFunction(v: Int): Int => String = i => (i * v).toString + def iBlowUp(v: Int, v2: String): String = throw new IllegalArgumentException("I was called!") + def iHaveTypeParamsAndImplicits[A, B](a: A, b: B)(implicit v3: Implicit[A]): String = + "not mocked" + def valueClass(n: Int, v: ValueClass): String = "not mocked" + def valueCaseClass(n: Int, v: ValueCaseClassInt): String = "not mocked" + def returnsValueCaseClassInt: ValueCaseClassInt = ValueCaseClassInt(-1) + def returnsValueCaseClassString: ValueCaseClassString = ValueCaseClassString("not mocked") + def takesManyValueClasses( + v: ValueClass, + v1: ValueCaseClassInt, + v2: ValueCaseClassString + ): String = "not mocked" + def baz(i: Int, b: Baz): String = "not mocked" + def fooWithVarArg(bells: String*): Unit = () + def fooWithActualArray(bells: Array[String]): Unit = () + def fooWithSecondParameterList(bell: String)(awaitable: Awaitable[Int]): Unit = () + def fooWithVarArgAndSecondParameterList(bells: String*)(awaitable: Awaitable[Int]): Unit = + () + def fooWithActualArrayAndSecondParameterList( + bells: Array[String] + )( + awaitable: Awaitable[Int] + ): Unit = () + def valueClassWithVarArg(monitors: Monitor*): Unit = () + def valueClassWithSecondParameterList(monitor: Monitor)(awaitable: Awaitable[Int]): Unit = () + def valueClassWithVarArgAndSecondParameterList( + monitors: Monitor* + )( + awaitable: Awaitable[Int] + ): Unit = () + + def option: Option[String] = None + def option(a: String, b: Int): Option[String] = None + def printGenericValue(value: GenericValue[String]): String = "not mocked" + def unit(): Unit = ??? + } + + case class Baz2(param1: Int, param2: String) + + trait ParameterizedTrait[+E] { def m(): E } + class ParameterizedTraitInt extends ParameterizedTrait[Int] { def m() = -1 } +} + +@RunWith(classOf[JUnitRunner]) +class MockitoTraitTest extends AnyFunSuite with Matchers /*with WhenReadyMixin*/ with Mockito { + import MockitoTraitTest._ + + test("mocks are called with the right arguments") { + val foo = mock[Foo] + + foo.bar(any[String]) returns "mocked" + foo.bar("alice") should equal("mocked") + foo.bar("alice") was called + } + + test("create a mock with name") { + val aMock = mock[Complex]("Named Mock") + aMock.toString should be("Named Mock") + } + + test("work with mixins") { + val aMock = mock[XWithY] + + aMock.bar returns "mocked!" + aMock.traitMethod(any[Int]) returns ValueCaseClassInt(69) + + aMock.bar should be("mocked!") + aMock.traitMethod(30) should be(ValueCaseClassInt(69)) + + MockitoSugar.verify(aMock).traitMethod(30) + } + + test("work with inline mixins") { + val aMock: X with Baz = mock[X with Baz] + + aMock.bar returns "mocked!" + aMock.traitMethod(any[Int]) returns ValueCaseClassInt(69) + aMock.varargMethod(1, 2, 3) returns 42 + aMock.javaVarargMethod(1, 2, 3) returns 42 + aMock.byNameMethod(69) returns 42 + + aMock.bar should be("mocked!") + aMock.traitMethod(30) should be(ValueCaseClassInt(69)) + aMock.varargMethod(1, 2, 3) should equal(42) + aMock.javaVarargMethod(1, 2, 3) should equal(42) + aMock.byNameMethod(69) should equal(42) + + MockitoSugar.verify(aMock).traitMethod(30) + MockitoSugar.verify(aMock).varargMethod(1, 2, 3) + MockitoSugar.verify(aMock).javaVarargMethod(1, 2, 3) + MockitoSugar.verify(aMock).byNameMethod(69) + a[WantedButNotInvoked] should be thrownBy MockitoSugar.verify(aMock).varargMethod(1, 2) + a[WantedButNotInvoked] should be thrownBy MockitoSugar.verify(aMock).javaVarargMethod(1, 2) + a[WantedButNotInvoked] should be thrownBy MockitoSugar.verify(aMock).byNameMethod(71) + } + + test("work with java varargs") { + val aMock = mock[JavaFoo] + + aMock.varargMethod(1, 2, 3) returns 42 + aMock.varargMethod(1, 2, 3) should equal(42) + + MockitoSugar.verify(aMock).varargMethod(1, 2, 3) + a[WantedButNotInvoked] should be thrownBy MockitoSugar.verify(aMock).varargMethod(1, 2) + } + + test("work when getting varargs from collections") { + val aMock = mock[Baz] + + aMock.varargMethod("Hello", List(1, 2, 3): _*) returns 42 + aMock.varargMethod("Hello", 1, 2, 3) should equal(42) + MockitoSugar.verify(aMock).varargMethod("Hello", 1, 2, 3) + } + + test("work when getting varargs from collections (with matchers)") { + val aMock = mock[Baz] + + aMock.varargMethod(eqTo("Hello"), eqTo(List(1, 2, 3)): _*) returns 42 + aMock.varargMethod("Hello", 1, 2, 3) should equal(42) + MockitoSugar.verify(aMock).varargMethod("Hello", 1, 2, 3) + } + + test("be serializable") { + val list = mock[java.util.List[String]](withSettings.name("list1").serializable()) + list.get(eqTo(3)) returns "mocked" + list.get(3) should be("mocked") + + val file = File.createTempFile("mock", "tmp") + file.deleteOnExit() + + val oos = new ObjectOutputStream(new FileOutputStream(file)) + try { + oos.writeObject(list) + } finally { + oos.close() + } + } + + test("work with type aliases") { + type MyType = String + + val aMock = mock[MyType => String] + + aMock("Hello") returns "Goodbye" + aMock("Hello") should be("Goodbye") + } + + test("not mock final methods") { + val abstractFile = mock[AbstractFile] + + abstractFile.path returns "SomeFile.scala" + abstractFile.path should be("SomeFile.scala") + } + + test("create a valid spy for lambdas and anonymous classes") { + val aSpy = spyLambda((arg: String) => s"Got: $arg") + + aSpy.apply(any[String]) returns "mocked!" + aSpy("hi!") should be("mocked!") + MockitoSugar.verify(aSpy).apply("hi!") + } + + test("eqTo syntax") { + val aMock = mock[Foo2] + + aMock.valueClass(1, eqTo(new ValueClass("meh"))) returns "mocked!" + aMock.valueClass(1, new ValueClass("meh")) should be("mocked!") + aMock.valueClass(1, eqTo(new ValueClass("meh"))) was called + + aMock.valueClass(any[Int], new ValueClass("moo")) returns "mocked!" + aMock.valueClass(11, new ValueClass("moo")) should be("mocked!") + aMock.valueClass(any[Int], new ValueClass("moo")) was called + + val valueClass = new ValueClass("blah") + aMock.valueClass(1, eqTo(valueClass)) returns "mocked!" + aMock.valueClass(1, valueClass) should be("mocked!") + aMock.valueClass(1, eqTo(valueClass)) was called + + aMock.valueCaseClass(2, eqTo(ValueCaseClassInt(100))) returns "mocked!" + aMock.valueCaseClass(2, ValueCaseClassInt(100)) should be("mocked!") + aMock.valueCaseClass(2, eqTo(ValueCaseClassInt(100))) was called + + val caseClassValue = ValueCaseClassInt(100) + aMock.valueCaseClass(3, eqTo(caseClassValue)) returns "mocked!" + aMock.valueCaseClass(3, caseClassValue) should be("mocked!") + aMock.valueCaseClass(3, eqTo(caseClassValue)) was called + + aMock.valueCaseClass(any[Int], ValueCaseClassInt(200)) returns "mocked!" + aMock.valueCaseClass(4, ValueCaseClassInt(200)) should be("mocked!") + aMock.valueCaseClass(any[Int], ValueCaseClassInt(200)) was called + } + + test("create a mock with default answer") { + val aMock = mock[Complex](CallsRealMethods) + + aMock.bar should be("not mocked") + } + + test("create a mock with default answer from implicit scope") { + implicit val defaultAnswer: DefaultAnswer = CallsRealMethods + + val aMock = mock[Complex] + aMock.bar should be("not mocked") + } + + test("stub a value class return value") { + val aMock = mock[Complex] + + aMock.returnsValueCaseClass returns ValueCaseClassInt(100) andThen ValueCaseClassInt(200) + + aMock.returnsValueCaseClass should be(ValueCaseClassInt(100)) + aMock.returnsValueCaseClass should be(ValueCaseClassInt(200)) + } + + test("stub a map") { + val mocked = mock[Map[String, String]] + mocked(any[String]) returns "123" + mocked("key") should be("123") + } + + test("work with stubbing value type parameters") { + val aMock = mock[ParameterizedTraitInt] + + aMock.m() returns 100 + + aMock.m() should be(100) + aMock.m() was called + } + + test("deal with verifying value type parameters") { + val aMock = mock[ParameterizedTraitInt] + //this has to be done separately as the verification in the other test would return the stubbed value so the + //null problem on the primitive would not happen + aMock.m() wasNever called + } + + test("stub a spy that would fail if the real impl is called") { + val aSpy = spy(new Org) + + an[IllegalArgumentException] should be thrownBy { + aSpy.iBlowUp(any[Int], any[String]) returns "mocked!" + } + + "mocked!" willBe returned by aSpy.iBlowUp(any[Int], "ok") + + aSpy.iBlowUp(1, "ok") should be("mocked!") + aSpy.iBlowUp(2, "ok") should be("mocked!") + + an[IllegalArgumentException] should be thrownBy { + aSpy.iBlowUp(2, "not ok") + } + } + + test("work with parameterized return types") { + val aMock = mock[HigherKinded[Option]] + + aMock.method returns Some(Right("Mocked!")) + aMock.method.get.right.get should be("Mocked!") + aMock.method2 + } + + test("work with parameterized return types declared in parents") { + val aMock = mock[ConcreteHigherKinded] + + aMock.method returns Some(Right("Mocked!")) + + aMock.method.get.right.get should be("Mocked!") + aMock.method2 + } + + test("work with higher kinded types and auxiliary methods") { + def whenGetById[F[_]](algebra: HigherKinded[F]): ScalaFirstStubbing[F[Either[String, String]]] = + MockitoSugar.when(algebra.method) + + val aMock = mock[HigherKinded[Option]] + + whenGetById(aMock) thenReturn Some(Right("Mocked!")) + aMock.method.get.right.get should be("Mocked!") + } + + test("stub a spy with an answer") { + val aSpy = spy(new Org) + + ((i: Int) => i * 10 + 2) willBe answered by aSpy.doSomethingWithThisInt(any[Int]) + ((i: Int, s: String) => (i * 10 + s.toInt).toString) willBe answered by aSpy + .doSomethingWithThisIntAndString(any[Int], any[String]) + ( + ( + i: Int, + s: String, + boolean: Boolean + ) => (i * 10 + s.toInt).toString + boolean + ) willBe answered by aSpy + .doSomethingWithThisIntAndStringAndBoolean(any[Int], any[String], v3 = true) + val counter = new AtomicInteger(1) + (() => counter.getAndIncrement().toString) willBe answered by aSpy.bar + counter.getAndIncrement().toString willBe answered by aSpy.baz + + counter.get should equal(1) + aSpy.bar should be("1") + counter.get should equal(2) + aSpy.baz should be("2") + counter.get should equal(3) + + aSpy.doSomethingWithThisInt(4) should equal(42) + aSpy.doSomethingWithThisIntAndString(4, "2") should be("42") + aSpy.doSomethingWithThisIntAndStringAndBoolean(4, "2", v3 = true) should be("42true") + aSpy.doSomethingWithThisIntAndStringAndBoolean(4, "2", v3 = false) should be("not mocked") + } + + test("create a mock where with mixed matchers and normal parameters (answer)") { + val org = mock[Org] + + org.doSomethingWithThisIntAndString(*, "test") answers "mocked!" + + org.doSomethingWithThisIntAndString(3, "test") should be("mocked!") + org.doSomethingWithThisIntAndString(5, "test") should be("mocked!") + org.doSomethingWithThisIntAndString(5, "est") should not be "mocked!" + } + + test("create a mock with nice answer API (multiple params)") { + val aMock = mock[Complex] + + aMock.doSomethingWithThisIntAndString(any[Int], any[String]) answers ((i: Int, s: String) => + ValueCaseClassInt(i * 10 + s.toInt)) andThenAnswer ((i: Int, _: String) => + ValueCaseClassInt(i)) + + aMock.doSomethingWithThisIntAndString(4, "2") should be(ValueCaseClassInt(42)) + aMock.doSomethingWithThisIntAndString(4, "2") should be(ValueCaseClassInt(4)) + } + + test("simplify answer API (invocation usage)") { + val org = mock[Org] + + org.doSomethingWithThisInt(any[Int]) answers ((i: InvocationOnMock) => i.arg[Int](0) * 10 + 2) + org.doSomethingWithThisInt(4) should equal(42) + } + + test("chain answers") { + val org = mock[Org] + + org.doSomethingWithThisInt(any[Int]) answers ((i: Int) => i * 10 + 2) andThenAnswer ((i: Int) => + i * 15 + 9) + + org.doSomethingWithThisInt(4) should equal(42) + org.doSomethingWithThisInt(4) should equal(69) + } + + test("chain answers (invocation usage)") { + val org = mock[Org] + + org.doSomethingWithThisInt(any[Int]) answers ((i: InvocationOnMock) => + i.arg[Int](0) * 10 + 2) andThenAnswer ((i: InvocationOnMock) => i.arg[Int](0) * 15 + 9) + + org.doSomethingWithThisInt(4) should equal(42) + org.doSomethingWithThisInt(4) should equal(69) + } + + test("allow using less params than method on answer stubbing") { + val org = mock[Org] + + org.doSomethingWithThisIntAndStringAndBoolean(any[Int], any[String], any[Boolean]) answers ( + (i: Int, s: String) => (i * 10 + s.toInt).toString) + org.doSomethingWithThisIntAndStringAndBoolean(4, "2", v3 = true) should be("42") + } + + test("stub a mock inline that has default args") { + val aMock = mock[Org] + + aMock.returnBar returns mock[Bar] andThen mock[Bar] + + aMock.returnBar should be(a[Bar]) + aMock.returnBar should be(a[Bar]) + } + + test("stub a high order function") { + val org = mock[Org] + + org.highOrderFunction(any[Int => String]) returns "mocked!" + + org.highOrderFunction(_.toString) should be("mocked!") + } + + test("stub a method that returns a function") { + val org = mock[Org] + + org + .iReturnAFunction(any[Int]).shouldReturn(_.toString).andThen(i => + (i * 2).toString).andThenCallRealMethod() + + org.iReturnAFunction(0)(42) should be("42") + org.iReturnAFunction(0)(42) should be("84") + org.iReturnAFunction(3)(3) should be("9") + } + + //useful if we want to delay the evaluation of whatever we are returning until the method is called + test("simplify stubbing an answer where we don't care about any param") { + val org = mock[Complex] + + val counter = new AtomicInteger(1) + org.bar answers counter.getAndIncrement().toString + + counter.get should equal(1) + org.bar should be("1") + counter.get should equal(2) + org.bar should be("2") + } + + test("create a mock while stubbing another") { + val aMock = mock[Complex] + + aMock.returnBar returns mock[Bar] + aMock.returnBar should be(a[Bar]) + } + + test("stub a runtime exception instance to be thrown") { + val aMock = mock[Foo] + + aMock.bar(any[String]) throws new IllegalArgumentException + an[IllegalArgumentException] should be thrownBy aMock.bar("alice") + } + + test("stub a checked exception instance to be thrown") { + val aMock = mock[Foo] + + aMock.bar(any[String]) throws new Exception + an[Exception] should be thrownBy aMock.bar("fred") + } + + test("stub a multiple exceptions instance to be thrown") { + val aMock = mock[Complex] + + aMock.bar throws new Exception andThenThrow new IllegalArgumentException + + an[Exception] should be thrownBy aMock.bar + an[IllegalArgumentException] should be thrownBy aMock.bar + + aMock.returnBar throws new IllegalArgumentException andThenThrow new Exception + + an[IllegalArgumentException] should be thrownBy aMock.returnBar + an[Exception] should be thrownBy aMock.returnBar + } + + test("default answer should deal with default arguments") { + val aMock = mock[Complex] + + aMock.iHaveSomeDefaultArguments("I will not pass the second argument") + aMock.iHaveSomeDefaultArguments("I will pass the second argument", "second argument") + + MockitoSugar + .verify(aMock) + .iHaveSomeDefaultArguments("I will not pass the second argument", "default value") + MockitoSugar + .verify(aMock).iHaveSomeDefaultArguments("I will pass the second argument", "second argument") + } + + test("work with by-name arguments (argument order doesn't matter when not using matchers)") { + val aMock = mock[Complex] + + aMock.iStartWithByNameArgs("arg1", "arg2") returns "mocked!" + + aMock.iStartWithByNameArgs("arg1", "arg2") should be("mocked!") + aMock.iStartWithByNameArgs("arg111", "arg2") should not be "mocked!" + + MockitoSugar.verify(aMock).iStartWithByNameArgs("arg1", "arg2") + MockitoSugar.verify(aMock).iStartWithByNameArgs("arg111", "arg2") + } + + test("work with primitive by-name arguments") { + val aMock = mock[Complex] + + aMock.iHavePrimitiveByNameArgs(1, "arg2") returns "mocked!" + + aMock.iHavePrimitiveByNameArgs(1, "arg2") should be("mocked!") + aMock.iHavePrimitiveByNameArgs(2, "arg2") should not be "mocked!" + + MockitoSugar.verify(aMock).iHavePrimitiveByNameArgs(1, "arg2") + MockitoSugar.verify(aMock).iHavePrimitiveByNameArgs(2, "arg2") + } + + test("function0 arguments") { + val aMock = mock[Complex] + + aMock.iHaveFunction0Args(eqTo("arg1"), function0("arg2")) returns "mocked!" + + aMock.iHaveFunction0Args("arg1", () => "arg2") should be("mocked!") + aMock.iHaveFunction0Args("arg1", () => "arg3") should not be "mocked!" + + MockitoSugar.verify(aMock).iHaveFunction0Args(eqTo("arg1"), function0("arg2")) + MockitoSugar.verify(aMock).iHaveFunction0Args(eqTo("arg1"), function0("arg3")) + } + + test("ignore the calls to the methods that provide default arguments") { + val aMock = mock[Complex] + + aMock.iHaveSomeDefaultArguments("I will not pass the second argument") + + MockitoSugar + .verify(aMock) + .iHaveSomeDefaultArguments("I will not pass the second argument", "default value") + verifyNoMoreInteractions(aMock) + } + + test("support an ArgCaptor and deal with default arguments") { + val aMock = mock[Complex] + + aMock.iHaveSomeDefaultArguments("I will not pass the second argument") + + val captor1 = ArgCaptor[String] + val captor2 = ArgCaptor[String] + MockitoSugar.verify(aMock).iHaveSomeDefaultArguments(captor1, captor2) + + captor1 hasCaptured "I will not pass the second argument" + captor2 hasCaptured "default value" + } + + test("work with Future types 1") { + val mockFn = mock[() => Int] + mockFn() returns 42 + + Future(mockFn.apply()).map { v => + v should equal(42) + mockFn() was called + } + } + + test("work with Future types 2") { + val mockFutureFn = mock[() => Future[Int]] + mockFutureFn() returns Future(42) + + Await.result(mockFutureFn(), 2.seconds) should equal(42) + } + + test("works with Future whenReady 1") { + val mockFn = mock[() => Unit] + + val f = Future(mockFn.apply()) + whenReady(f)(_ => mockFn() was called) + } + + test("works with Future whenReady 2") { + val mockFutureFn = mock[() => Future[Int]] + mockFutureFn() returns Future(42) + + whenReady(mockFutureFn())(result => result should equal(42)) + } + + test("works with Function types") { + val expected = Future("Hello, world") + val mockFunction = mock[Int => Future[String]] + mockFunction(any[Int]) returns expected + + mockFunction(42) should equal(expected) + } + + test("works with Function types no sugar") { + val expected = Future("Hello, world") + val mockFunction = mock[Function1[Int, Future[String]]] + mockFunction(any[Int]) returns expected + + mockFunction(42) should equal(expected) + } + + test("works with Function whenReady") { + val mockFunction = mock[Int => Future[String]] + mockFunction(any[Int]) returns Future("Hello, world") + + whenReady(mockFunction(42))(result => result should equal("Hello, world")) + } + + test("works with Function whenReady no sugar") { + val mockFunction = mock[Function1[Int, Future[String]]] + mockFunction(any[Int]) returns Future("Hello, world") + + whenReady(mockFunction(42))(result => result should equal("Hello, world")) + } + + test("disallow passing traits in the settings") { + a[IllegalArgumentException] should be thrownBy { + mock[Foo](withSettings.extraInterfaces(classOf[Baz])) + } + } + + private final implicit def convertTwitterFuture[T](twitterFuture: Future[T]): FutureConcept[T] = + new FutureConcept[T] { + def eitherValue: Option[Either[Throwable, T]] = + twitterFuture.poll.map { + case Return(result) => Right(result) + case Throw(e) => Left(e) + } + // As per the ScalaTest docs if the underlying future does not support these they + // must return false. + def isExpired: Boolean = false + def isCanceled: Boolean = false + } +} diff --git a/util-mock/src/test/scala/com/twitter/mock/MockitoTraitWithFixtureTest.scala b/util-mock/src/test/scala/com/twitter/mock/MockitoTraitWithFixtureTest.scala new file mode 100644 index 0000000000..6eeaa325a3 --- /dev/null +++ b/util-mock/src/test/scala/com/twitter/mock/MockitoTraitWithFixtureTest.scala @@ -0,0 +1,50 @@ +package com.twitter.mock + +import org.junit.runner.RunWith +import org.mockito.exceptions.verification.{NeverWantedButInvoked, NoInteractionsWanted} +import org.scalatest.Outcome +import org.scalatest.funsuite.FixtureAnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class MockitoTraitWithFixtureTest extends FixtureAnyFunSuite with Matchers with Mockito { + class Foo { + def bar(a: String) = "bar" + def baz = "baz" + } + + class FixtureParam { + val foo: Foo = mock[Foo] + } + + protected def withFixture(test: OneArgTest): Outcome = { + val theFixture = new FixtureParam + super.withFixture(test.toNoArgTest(theFixture)) + } + + test("verify no calls on fixture objects methods") { f: FixtureParam => + "mocked" willBe returned by f.foo.bar("jane") + "mocked" willBe returned by f.foo.baz + + f.foo.bar("jane") should be("mocked") + f.foo.baz should be("mocked") + + a[NeverWantedButInvoked] should be thrownBy { + f.foo.bar(any[String]) wasNever called + } + a[NeverWantedButInvoked] should be thrownBy { + f.foo.baz wasNever called + } + } + + test("verify no calls on a mock inside a fixture object") { f: FixtureParam => + f.foo.bar("jane") returns "mocked" + f.foo wasNever called + + f.foo.bar("jane") should be("mocked") + a[NoInteractionsWanted] should be thrownBy { + f.foo wasNever called + } + } +} diff --git a/util-mock/src/test/scala/com/twitter/mock/ResetMocksAfterEachTestTraitTest.scala b/util-mock/src/test/scala/com/twitter/mock/ResetMocksAfterEachTestTraitTest.scala new file mode 100644 index 0000000000..43d7af5e99 --- /dev/null +++ b/util-mock/src/test/scala/com/twitter/mock/ResetMocksAfterEachTestTraitTest.scala @@ -0,0 +1,89 @@ +package com.twitter.mock + +import org.junit.runner.RunWith +import org.mockito.MockitoSugar +import org.mockito.scalatest.ResetMocksAfterEachTest +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.scalatestplus.junit.JUnitRunner + +object ResetMocksAfterEachTestTraitTest { + + trait ReturnInteger { + def get: Int + } + + trait Foo { + def bar(a: String) = "bar" + } + + trait Baz { + def qux(a: String) = "qux" + } +} + +/** + * Ensure [[ResetMocksAfterEachTest]] works with [[com.twitter.mock.Mockito]] + */ +@RunWith(classOf[JUnitRunner]) +class ResetMocksAfterEachTestTraitTest + extends AnyFunSuite + with Matchers + with BeforeAndAfterAll + with BeforeAndAfterEach + with Mockito + with ResetMocksAfterEachTest { + import ResetMocksAfterEachTestTraitTest._ + + val foo: Foo = mock[Foo] + val baz: Baz = mock[Baz] + + private val mockReturnInteger: ReturnInteger = mock[ReturnInteger] + mockReturnInteger.get returns 100 + + override def beforeAll(): Unit = { + mockReturnInteger.get should equal(100) + super.beforeAll() + } + + override def afterEach(): Unit = { + super.afterEach() + + mockingDetails(mockReturnInteger).getInvocations.isEmpty should be(true) + } + + test("no interactions 1") { + MockitoSugar.verifyZeroInteractions(foo) + + foo.bar("bar") returns "mocked" + foo.bar("bar") shouldBe "mocked" + } + + test("no interactions 2") { + MockitoSugar.verifyZeroInteractions(foo) + + foo.bar("bar") returns "mocked2" + foo.bar("bar") shouldBe "mocked2" + } + + test("no interactions for all 1") { + MockitoSugar.verifyZeroInteractions(foo, baz) + + foo.bar("bar") returns "mocked3" + baz.qux("qux") returns "mocked4" + + foo.bar("bar") shouldBe "mocked3" + baz.qux("qux") shouldBe "mocked4" + } + + test("no interactions for all 2") { + MockitoSugar.verifyZeroInteractions(foo, baz) + + foo.bar("bar") returns "mocked5" + baz.qux("qux") returns "mocked6" + + foo.bar("bar") shouldBe "mocked5" + baz.qux("qux") shouldBe "mocked6" + } +} diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index d012dfc789..35775d0ab3 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -3,10 +3,10 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala index 2745158de2..be11ae916e 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala @@ -2,7 +2,7 @@ package com.twitter.util.registry import java.util.logging.Logger import org.mockito.Mockito.{never, verify} -import org.mockito.Matchers.anyObject +import org.mockito.ArgumentMatchers.anyObject import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index 58314f4931..35acdc1be9 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -3,10 +3,10 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "util/util-routing/src/main/scala/com/twitter/util/routing", "util/util-routing/src/test/scala/com/twitter/util/routing/dynamic", "util/util-routing/src/test/scala/com/twitter/util/routing/simple", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index 12d8fa0184..6a2230fad2 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -23,10 +23,10 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ ":testutil", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "3rdparty/jvm/org/slf4j:slf4j-simple", ], ) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala index 3c914e0291..28ccedbcfe 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala @@ -1,6 +1,6 @@ package com.twitter.util.logging -import org.mockito.Matchers._ +import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ import org.scalatestplus.mockito.MockitoSugar import org.scalatest.{FunSuite, Matchers} diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala index aeba68ea36..c897aed22b 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala @@ -1,6 +1,6 @@ package com.twitter.util.logging -import org.mockito.Matchers._ +import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ import org.scalatestplus.mockito.MockitoSugar import org.scalatest.{FunSuite, Matchers} diff --git a/util-stats/src/test/scala/BUILD b/util-stats/src/test/scala/BUILD index d7261205f1..ee68d7b328 100644 --- a/util-stats/src/test/scala/BUILD +++ b/util-stats/src/test/scala/BUILD @@ -3,7 +3,7 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index 5314eebd64..529da964b6 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -4,10 +4,10 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-logging/src/main/scala/com/twitter/logging", diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index a66547ffbb..5196a1e6a6 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -5,7 +5,7 @@ import scala.jdk.CollectionConverters._ import org.apache.zookeeper._ import org.apache.zookeeper.data.{ACL, Stat} -import org.mockito.Matchers.{eq => meq, _} +import org.mockito.ArgumentMatchers.{eq => meq, _} import org.mockito.Mockito._ import org.mockito._ import org.mockito.invocation.InvocationOnMock diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index 35c70bac58..13299b3a06 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -4,10 +4,10 @@ junit_tests( dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", - "3rdparty/jvm/org/mockito:mockito-all", + "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "3rdparty/jvm/org/scalatestplus:mockito-3-2", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-zk/src/main/scala/com/twitter/zk", From 8fd310b14eb9e97c5f7cc07bdd40b5ffc8805123 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 12 Aug 2020 19:09:47 +0000 Subject: [PATCH 366/796] twitter-server: Add SecurityManager check to ContentionHandler Problem Attempting to access the `ManagementFactory.getThreadMXBean` to enable contention monitoring can throw a `SecurityException` if the `SecurityManager` does not allow for that permission. Solution Ensure that the `ContentionHandler` checks the `SecurityManager` before attempting to instantiate a `ContentionSnapshot`. JIRA Issues: CSL-10056 Differential Revision: https://phabricator.twitter.biz/D531873 --- .../src/main/scala/com/twitter/jvm/ContentionSnapshot.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala b/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala index 69041d5a0b..5ce32a7205 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala @@ -9,6 +9,9 @@ import java.lang.management.{ManagementFactory, ThreadInfo} * * While this could be an object, we use instantiation as a signal of * intent and enable contention monitoring. + * + * @note users should ensure that the [[java.lang.management.ManagementPermission("control")]] is + * allowed via the [[java.lang.SecurityManager]]. */ class ContentionSnapshot { ManagementFactory.getThreadMXBean.setThreadContentionMonitoringEnabled(true) From 4657258dfe0d7e1a70ad7460a78601c715bfb4bd Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Wed, 12 Aug 2020 21:56:14 +0000 Subject: [PATCH 367/796] util-app: Support Java Enum type flags Problem/Solution: Add support to inject flags of Java Enum types JIRA Issues: CSL-10078 Differential Revision: https://phabricator.twitter.biz/D530205 --- CHANGELOG.rst | 2 ++ .../scala/com/twitter/app/Flaggable.scala | 16 ++++++++++ .../java/com/twitter/app/JavaFlagTest.java | 31 +++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 674416b555..dfc2818633 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,8 @@ New Features - `mockito-all:1.10.19` to `mockito-core:3.3.3` - `scalatestplus:mockito-1-10:3.1.0.0` to `scalatestplus:mockito-3-2:3.1.2.0` ``PHAB_ID=D530995`` +* util-app: Add support for flags of Java Enum types. ``PHAB_ID=D530205`` + 20.8.0 ------ diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index 7eb566fe36..37ba726c0c 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -166,6 +166,22 @@ object Flaggable { } } + /** + * Create a Flaggable of Java Enum. + * @param clazz The [[java.lang.Enum]] class. + * @note [[java.lang.Enum]] enumeration constant value look up is case insensitive. + */ + def ofJavaEnum[T <: Enum[T]](clazz: Class[T]): Flaggable[T] = + mandatory { s: String => + clazz.getEnumConstants.find(_.name.equalsIgnoreCase(s)) match { + case Some(prop) => prop + case _ => + throw new IllegalArgumentException( + s"The property $s does not belong to Java Enum ${clazz.getName}, the constants defined " + + s"in the class are: ${clazz.getEnumConstants.toSeq}.") + } + } + private[app] class SetFlaggable[T: Flaggable] extends Flaggable[Set[T]] { private val flag = implicitly[Flaggable[T]] assert(flag.default.isEmpty) diff --git a/util-app/src/test/java/com/twitter/app/JavaFlagTest.java b/util-app/src/test/java/com/twitter/app/JavaFlagTest.java index 8fab172321..a83270b6b9 100644 --- a/util-app/src/test/java/com/twitter/app/JavaFlagTest.java +++ b/util-app/src/test/java/com/twitter/app/JavaFlagTest.java @@ -39,6 +39,11 @@ static class Named { } } + enum Disney { + MICKEY, + MINNIE + } + @Test public void testJavaFlags() { @@ -98,6 +103,32 @@ public void testJavaLongFlags() { } } + @Test + public void testJavaEnumFlags() { + Flags flags = new Flags("ApplicationName"); + Flag enumFlag = flags.create( + "disney", + Disney.MICKEY, + "", + Flaggable.ofJavaEnum(Disney.class) + ); + assertEquals(Disney.MICKEY, enumFlag.apply()); + // should parse Strings with matching cases without errors + enumFlag.parse("MICKEY"); + // should parse Strings without matching cases without errors + enumFlag.parse("minnie"); + // should thrown IllegalArgumentException when property doesn't exist in the Enum + try{ + enumFlag.parse("MICKY"); + } catch (IllegalArgumentException e) { + assertEquals( + "The property MICKY does not belong to Java Enum com.twitter.app.JavaFlagTest$Disney, " + + "the constants defined in the class are: WrappedArray(MICKEY, MINNIE).", + e.getMessage() + ); + } + } + @Test public void testLet() { Flags flags = new Flags("ApplicationName"); From 1236912790a2de330d7592d2148028a569077c38 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 13 Aug 2020 00:48:24 +0000 Subject: [PATCH 368/796] util-mock: Fix up tests in Scala 2.11 Problem/Solution Some test cases in the `MockitoTraitTest` fail in Scala 2.11 due to differences in the interaction between the Mockito Java library and the Scala code. Let's adapt the tests to work in the multiple supported versions and note some of the issues. JIRA Issues: CSL-10087 Differential Revision: https://phabricator.twitter.biz/D532590 --- .../com/twitter/mock/MockitoTraitTest.scala | 65 ++++++++++++++----- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala b/util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala index 30af1d5858..1ee36d241e 100644 --- a/util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala +++ b/util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala @@ -1,8 +1,7 @@ package com.twitter.mock import com.twitter.conversions.DurationOps._ -import com.twitter.util.{Return, Throw} -import com.twitter.util.{Await, Awaitable, Future, Monitor} +import com.twitter.util._ import java.io.{File, FileOutputStream, ObjectOutputStream} import java.util.concurrent.atomic.AtomicInteger import org.junit.runner.RunWith @@ -11,12 +10,14 @@ import org.mockito.captor.ArgCaptor import org.mockito.exceptions.verification.WantedButNotInvoked import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.{CallsRealMethods, DefaultAnswer, ScalaFirstStubbing} -import org.scalatest.concurrent.ScalaFutures.{FutureConcept, whenReady} -import org.scalatestplus.junit.JUnitRunner +import org.scalatest.concurrent.PatienceConfiguration.{Interval, Timeout} +import org.scalatest.concurrent.ScalaFutures.{FutureConcept, PatienceConfig, whenReady} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import org.scalatest.time.{Millis, Seconds, Span} +import org.scalatestplus.junit.JUnitRunner import scala.annotation.varargs -import scala.language.implicitConversions +import scala.language.{higherKinds, implicitConversions} import scala.reflect.io.AbstractFile object MockitoTraitTest { @@ -94,7 +95,9 @@ object MockitoTraitTest { def traitMethodWithDefaultArgs(defaultArg: Int = 30, anotherDefault: String = "hola"): Int = -1 } - class ValueClass(val v: String) extends AnyVal { + // this is a case class in order to have equals/hashcode properly defined which + // is necessary for the eqTo syntax to work properly in Scala 2.11 + case class ValueClass(v: String) extends AnyVal { override def toString = s"ValueClass($v)" } case class ValueCaseClassInt(v: Int) extends AnyVal @@ -174,7 +177,7 @@ object MockitoTraitTest { } @RunWith(classOf[JUnitRunner]) -class MockitoTraitTest extends AnyFunSuite with Matchers /*with WhenReadyMixin*/ with Mockito { +class MockitoTraitTest extends AnyFunSuite with Matchers with Mockito { import MockitoTraitTest._ test("mocks are called with the right arguments") { @@ -295,15 +298,15 @@ class MockitoTraitTest extends AnyFunSuite with Matchers /*with WhenReadyMixin*/ test("eqTo syntax") { val aMock = mock[Foo2] - aMock.valueClass(1, eqTo(new ValueClass("meh"))) returns "mocked!" - aMock.valueClass(1, new ValueClass("meh")) should be("mocked!") - aMock.valueClass(1, eqTo(new ValueClass("meh"))) was called + aMock.valueClass(1, eqTo(ValueClass("meh"))) returns "mocked!" + aMock.valueClass(1, ValueClass("meh")) should be("mocked!") + aMock.valueClass(1, eqTo(ValueClass("meh"))) was called - aMock.valueClass(any[Int], new ValueClass("moo")) returns "mocked!" - aMock.valueClass(11, new ValueClass("moo")) should be("mocked!") - aMock.valueClass(any[Int], new ValueClass("moo")) was called + aMock.valueClass(any[Int], ValueClass("moo")) returns "mocked!" + aMock.valueClass(11, ValueClass("moo")) should be("mocked!") + aMock.valueClass(any[Int], ValueClass("moo")) was called - val valueClass = new ValueClass("blah") + val valueClass = ValueClass("blah") aMock.valueClass(1, eqTo(valueClass)) returns "mocked!" aMock.valueClass(1, valueClass) should be("mocked!") aMock.valueClass(1, eqTo(valueClass)) was called @@ -663,11 +666,18 @@ class MockitoTraitTest extends AnyFunSuite with Matchers /*with WhenReadyMixin*/ Await.result(mockFutureFn(), 2.seconds) should equal(42) } + // Function0 types seem to not play well with Mockito and Future.apply + // in Scala 2.11 so we use Future.value here -- this test is perhaps a bit + // tautological since mockFn() is called immediately in Future.value but it + // did expose a difference between Scala 2.11 and Scala 2.12. test("works with Future whenReady 1") { - val mockFn = mock[() => Unit] + val mockFn = mock[() => Boolean] + mockFn() returns true - val f = Future(mockFn.apply()) - whenReady(f)(_ => mockFn() was called) + val f: Future[Boolean] = Future.value(mockFn()) + whenReady(f) { _: Boolean => + mockFn() was called + } } test("works with Future whenReady 2") { @@ -677,6 +687,16 @@ class MockitoTraitTest extends AnyFunSuite with Matchers /*with WhenReadyMixin*/ whenReady(mockFutureFn())(result => result should equal(42)) } + test("works with Future whenReady 3") { + val mockFn = mock[Int => Boolean] + mockFn(any[Int]) returns true + + val f: Future[Boolean] = Future.apply(mockFn(42)) + whenReady(f) { result: Boolean => + result should be(true) + } + } + test("works with Function types") { val expected = Future("Hello, world") val mockFunction = mock[Int => Future[String]] @@ -713,6 +733,9 @@ class MockitoTraitTest extends AnyFunSuite with Matchers /*with WhenReadyMixin*/ } } + private final implicit val patienceConfig: PatienceConfig = + PatienceConfig(timeout = Span(5, Seconds), interval = Span(20, Millis)) + private final implicit def convertTwitterFuture[T](twitterFuture: Future[T]): FutureConcept[T] = new FutureConcept[T] { def eitherValue: Option[Either[Throwable, T]] = @@ -725,4 +748,12 @@ class MockitoTraitTest extends AnyFunSuite with Matchers /*with WhenReadyMixin*/ def isExpired: Boolean = false def isCanceled: Boolean = false } + + private final implicit def twitterDurationToScalaTestTimeout(duration: Duration): Timeout = { + Timeout(Span(duration.inMillis, Millis)) + } + + private final implicit def twitterDurationToScalaTestInterval(duration: Duration): Interval = { + Interval(Span(duration.inMillis, Millis)) + } } From 947b9cb1e5b4f5b8ae10d900e41349cd615c3cbb Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Thu, 13 Aug 2020 21:51:57 +0000 Subject: [PATCH 369/796] util-stats: enhance MetricBuilder's toString This reverts commit 5086494c3f940fe0a753374e48fd8787685468b7. Problem ------- MetricBuilder's toString produced inconsistent results on differing versions of scala. Solution -------- Avoid the problem and increase readability for humans by stringify-ing the metric name, joining the fields with "/"s as is done on the metrics.json endpoint. Result ------ MetricBuilder's toString produces consistent, human-readable output across scala versions. JIRA Issues: CSL-10070 Differential Revision: https://phabricator.twitter.biz/D531211 --- .../twitter/finagle/stats/MetricBuilder.scala | 10 ++++---- .../stats/InMemoryStatsReceiverTest.scala | 23 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 395721d812..1c1d58bbaa 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -35,10 +35,10 @@ class MetricBuilder( val role: SourceRole = NoRoleSpecified, val verbosity: Verbosity = Verbosity.Default, val sourceClass: Option[String] = None, - val name: Seq[String] = Seq(), + val name: Seq[String] = Seq.empty, val processPath: Option[String] = None, // Only persisted and relevant when building histograms. - val percentiles: IndexedSeq[Double] = IndexedSeq(), + val percentiles: IndexedSeq[Double] = IndexedSeq.empty, val statsReceiver: StatsReceiver) { /** @@ -164,6 +164,8 @@ class MetricBuilder( hashCodes.foldLeft(0)((a, b) => 31 * a + b) } - override def toString = - s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $name, $processPath, $percentiles, $statsReceiver)" + override def toString(): String = { + val nameString = name.mkString("/") + s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $nameString, $processPath, $percentiles, $statsReceiver)" + } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 40b6cfd3d0..fde38e8111 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -330,11 +330,9 @@ class InMemoryStatsReceiverTest extends FunSuite with Eventually with Integratio test("printSchemas should print schemas") { val stats = new InMemoryStatsReceiver() stats.addGauge("coolGauge") { 3 } - stats.counter("sweetCounter") - stats.stat("radHisto") - val gauges = Array("coolGauge").toSeq - val histograms = Array("radHisto").toSeq - val counters = Array("sweetCounter").toSeq + stats.counter("sweet", "counter") + val radSr = stats.scope("rad") + radSr.stat("histo") val baos = new ByteArrayOutputStream() val ps = new PrintStream(baos, true, "utf-8") @@ -344,12 +342,15 @@ class InMemoryStatsReceiverTest extends FunSuite with Eventually with Integratio val parts = content.split('\n') assert(parts.length == 3) - assert(parts( - 0) == s"coolGauge GaugeSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, $gauges, None, Vector(), InMemoryStatsReceiver))") - assert(parts( - 1) == s"radHisto HistogramSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, $histograms, None, Vector(), InMemoryStatsReceiver))") - assert(parts( - 2) == s"sweetCounter CounterSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, $counters, None, Vector(), InMemoryStatsReceiver))") + assert( + parts( + 0) == "coolGauge GaugeSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, coolGauge, None, Vector(), InMemoryStatsReceiver))") + assert( + parts( + 1) == "rad/histo HistogramSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, rad/histo, None, Vector(), InMemoryStatsReceiver/rad))") + assert( + parts( + 2) == "sweet/counter CounterSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, sweet/counter, None, Vector(), InMemoryStatsReceiver))") } finally { ps.close() } From dc172c5581a706745d8fcbed4726e72ad73f3002 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Thu, 13 Aug 2020 22:14:15 +0000 Subject: [PATCH 370/796] util-stats: retain metadata for CumulativeGauges Problem ------- Metric Metadata is not retained properly when a gauge is constructed via CumulativeGauge's addGauge method. Solution -------- Remove the optimization that was leading to the lost MetricMetadata. The optimization was to avoid creating a new function on each call to whenGaugeNotPresent by having functions for each of the Verbosity levels. Now, there is more Metadata to store than simply Verbosity and we cannot have a cached function for all possible Metadata values. Result ------ Metric Metadata is now properly populated and retained when CumulativeGauge's addGauge method. JIRA Issues: CSL-9958 Differential Revision: https://phabricator.twitter.biz/D531151 --- .../finagle/stats/CumulativeGauge.scala | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index 6e004fdd43..d60c8458c2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -120,27 +120,18 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => * The StatsReceiver implements these. They provide the cumulative * gauges. */ - protected[this] def registerGauge(verbosity: Verbosity, name: Seq[String], f: => Float): Unit + protected[this] def registerGauge(schema: GaugeSchema, f: => Float): Unit protected[this] def deregisterGauge(name: Seq[String]): Unit - // To avoid creating a new function on each call to `whenGaugeNotPresent`, we cache these - // functions for [[Verbosity.Default]] and [[Verbosity.Debug]], the most common cases. - private[this] val whenDefaultNotPresent = whenNotPresent(Verbosity.Default) - private[this] val whenDebugNotPresent = whenNotPresent(Verbosity.Debug) - - private[this] def getWhenNotPresent(verbosity: Verbosity) = verbosity match { - case Verbosity.Default => whenDefaultNotPresent - case Verbosity.Debug => whenDebugNotPresent - case _ => whenNotPresent(verbosity) - } + private[this] def getWhenNotPresent(schema: GaugeSchema) = whenNotPresent(schema) /** The executor that will be used for expiring gauges */ def executor: Executor = ForkJoinPool.commonPool() - private[this] def whenNotPresent(verbosity: Verbosity) = + private[this] def whenNotPresent(schema: GaugeSchema) = new JFunction[Seq[String], CumulativeGauge] { def apply(key: Seq[String]): CumulativeGauge = new CumulativeGauge(executor) { - self.registerGauge(verbosity, key, getValue) + self.registerGauge(schema, getValue) // The number of registers starts at `0` because every new gauge will cause a // registration, even the one that created the `CumulativeGauge`. Because 0 is @@ -179,9 +170,8 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => def addGauge(schema: GaugeSchema)(f: => Float): Gauge = { var gauge: Gauge = null while (gauge == null) { - val cumulativeGauge = gauges.computeIfAbsent( - schema.metricBuilder.name, - getWhenNotPresent(schema.metricBuilder.verbosity)) + val cumulativeGauge = + gauges.computeIfAbsent(schema.metricBuilder.name, getWhenNotPresent(schema)) gauge = cumulativeGauge.addGauge(f) } gauge From 1030e37842b37181df9c0dc7abe5987e338845cc Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Sat, 15 Aug 2020 00:31:25 +0000 Subject: [PATCH 371/796] util-app: Fix oss Scala 2.13 build Problem: The travis build failed to run `c.t.app.JavaFlagTest.testJavaEnumFlags` because calling `Array.toSeq` is wrapping a Java array in `ArraySeq` in Scala 2.13, but in pre Scala 2.13 world, it was wrapped in `WrappedArray`, which is deprecated in Scala 2.13. Solution: Construct the error message with `Array.toList` instead of `Array.toSeq` Parse the flag before reading from it. JIRA Issues: CSL-10090 Differential Revision: https://phabricator.twitter.biz/D533992 --- util-app/src/main/scala/com/twitter/app/Flaggable.scala | 2 +- util-app/src/test/java/com/twitter/app/JavaFlagTest.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index 37ba726c0c..b350ad397e 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -178,7 +178,7 @@ object Flaggable { case _ => throw new IllegalArgumentException( s"The property $s does not belong to Java Enum ${clazz.getName}, the constants defined " + - s"in the class are: ${clazz.getEnumConstants.toSeq}.") + s"in the class are: ${clazz.getEnumConstants.mkString(",")}.") } } diff --git a/util-app/src/test/java/com/twitter/app/JavaFlagTest.java b/util-app/src/test/java/com/twitter/app/JavaFlagTest.java index a83270b6b9..bdbfc93d72 100644 --- a/util-app/src/test/java/com/twitter/app/JavaFlagTest.java +++ b/util-app/src/test/java/com/twitter/app/JavaFlagTest.java @@ -112,6 +112,7 @@ public void testJavaEnumFlags() { "", Flaggable.ofJavaEnum(Disney.class) ); + enumFlag.parse(); assertEquals(Disney.MICKEY, enumFlag.apply()); // should parse Strings with matching cases without errors enumFlag.parse("MICKEY"); @@ -123,7 +124,7 @@ public void testJavaEnumFlags() { } catch (IllegalArgumentException e) { assertEquals( "The property MICKY does not belong to Java Enum com.twitter.app.JavaFlagTest$Disney, " + - "the constants defined in the class are: WrappedArray(MICKEY, MINNIE).", + "the constants defined in the class are: MICKEY,MINNIE.", e.getMessage() ); } From 61d1810158b8e6a71af2c8faaf3e5ff0a6cdf249 Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Tue, 18 Aug 2020 17:27:27 +0000 Subject: [PATCH 372/796] util-inject: added TwitterModule to util-inject Problem The second step in moving finatra/inject to util is moving TwitterModule Solution Created a directory under util-inject and added TwitterModule's dependencies Result This will help us to in moving almost all of finatra/inject/inject-core to util/inject JIRA Issues: CSL-10069 Differential Revision: https://phabricator.twitter.biz/D528700 --- util-inject/PROJECT | 11 + .../main/java/com/twitter/inject/flag/BUILD | 13 + .../java/com/twitter/inject/flag/Flagged.java | 24 + .../com/twitter/inject/flag/FlaggedImpl.java | 49 ++ .../java/com/twitter/inject/flag/Flags.java | 30 + .../scala/com/twitter/{util => }/inject/BUILD | 4 + .../main/scala/com/twitter/inject/Flags.scala | 17 + .../twitter/{util => }/inject/Injector.scala | 2 +- .../com/twitter/inject/InjectorModule.scala | 16 + .../twitter/inject/TwitterBaseModule.scala | 24 + .../com/twitter/inject/TwitterModule.scala | 368 ++++++++++++ .../twitter/inject/TwitterModuleFlags.scala | 116 ++++ .../inject/TwitterModuleLifecycle.scala | 83 +++ .../twitter/{util => }/inject/TypeUtils.scala | 2 +- .../scala/com/twitter/inject/internal/BUILD | 24 + .../twitter/inject/internal/FlagsModule.scala | 69 +++ .../inject/internal/InstalledModules.scala | 82 +++ .../com/twitter/inject/internal/Modules.scala | 113 ++++ .../TwitterTypeConvertersModule.scala | 46 ++ .../twitter/inject/internal/iterable.scala | 37 ++ .../com/twitter/inject/internal/package.scala | 5 + .../java/com/twitter/{util => }/inject/BUILD | 0 .../inject/TestBindingAnnotation.java | 2 +- .../scala/com/twitter/{util => }/inject/BUILD | 4 +- .../{util => }/inject/InjectorTest.scala | 40 +- .../twitter/inject/TwitterModuleTest.scala | 543 ++++++++++++++++++ .../{util => }/inject/TypeUtilsTest.scala | 2 +- .../twitter/{util => }/inject/classes.scala | 36 +- .../inject/module/SeqAbstractModule.scala | 12 + .../{util => }/inject/module/TestModule.scala | 5 +- .../inject/module/TestTwitterModule.scala | 136 +++++ .../com/twitter/inject/servicefactory.scala | 32 ++ 32 files changed, 1902 insertions(+), 45 deletions(-) create mode 100644 util-inject/PROJECT create mode 100644 util-inject/src/main/java/com/twitter/inject/flag/BUILD create mode 100644 util-inject/src/main/java/com/twitter/inject/flag/Flagged.java create mode 100644 util-inject/src/main/java/com/twitter/inject/flag/FlaggedImpl.java create mode 100644 util-inject/src/main/java/com/twitter/inject/flag/Flags.java rename util-inject/src/main/scala/com/twitter/{util => }/inject/BUILD (80%) create mode 100644 util-inject/src/main/scala/com/twitter/inject/Flags.scala rename util-inject/src/main/scala/com/twitter/{util => }/inject/Injector.scala (99%) create mode 100644 util-inject/src/main/scala/com/twitter/inject/InjectorModule.scala create mode 100644 util-inject/src/main/scala/com/twitter/inject/TwitterBaseModule.scala create mode 100644 util-inject/src/main/scala/com/twitter/inject/TwitterModule.scala create mode 100644 util-inject/src/main/scala/com/twitter/inject/TwitterModuleFlags.scala create mode 100644 util-inject/src/main/scala/com/twitter/inject/TwitterModuleLifecycle.scala rename util-inject/src/main/scala/com/twitter/{util => }/inject/TypeUtils.scala (98%) create mode 100644 util-inject/src/main/scala/com/twitter/inject/internal/BUILD create mode 100644 util-inject/src/main/scala/com/twitter/inject/internal/FlagsModule.scala create mode 100644 util-inject/src/main/scala/com/twitter/inject/internal/InstalledModules.scala create mode 100644 util-inject/src/main/scala/com/twitter/inject/internal/Modules.scala create mode 100644 util-inject/src/main/scala/com/twitter/inject/internal/TwitterTypeConvertersModule.scala create mode 100644 util-inject/src/main/scala/com/twitter/inject/internal/iterable.scala create mode 100644 util-inject/src/main/scala/com/twitter/inject/internal/package.scala rename util-inject/src/test/java/com/twitter/{util => }/inject/BUILD (100%) rename util-inject/src/test/java/com/twitter/{util => }/inject/TestBindingAnnotation.java (93%) rename util-inject/src/test/scala/com/twitter/{util => }/inject/BUILD (79%) rename util-inject/src/test/scala/com/twitter/{util => }/inject/InjectorTest.scala (86%) create mode 100644 util-inject/src/test/scala/com/twitter/inject/TwitterModuleTest.scala rename util-inject/src/test/scala/com/twitter/{util => }/inject/TypeUtilsTest.scala (98%) rename util-inject/src/test/scala/com/twitter/{util => }/inject/classes.scala (65%) create mode 100644 util-inject/src/test/scala/com/twitter/inject/module/SeqAbstractModule.scala rename util-inject/src/test/scala/com/twitter/{util => }/inject/module/TestModule.scala (84%) create mode 100644 util-inject/src/test/scala/com/twitter/inject/module/TestTwitterModule.scala create mode 100644 util-inject/src/test/scala/com/twitter/inject/servicefactory.scala diff --git a/util-inject/PROJECT b/util-inject/PROJECT new file mode 100644 index 0000000000..791a30f039 --- /dev/null +++ b/util-inject/PROJECT @@ -0,0 +1,11 @@ +owners: + - banderson + - ccoco + - ibennett + - koliver + - mnakamura + - roanta + - ryano + - vkostyukov +watchers: + - csl-team:ldap diff --git a/util-inject/src/main/java/com/twitter/inject/flag/BUILD b/util-inject/src/main/java/com/twitter/inject/flag/BUILD new file mode 100644 index 0000000000..56acef0a01 --- /dev/null +++ b/util-inject/src/main/java/com/twitter/inject/flag/BUILD @@ -0,0 +1,13 @@ +java_library( + sources = ["*.java"], + compiler_option_sets = ["fatal_warnings"], + provides = artifact( + org = "com.twitter", + name = "util-inject-java", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/google/inject:guice", + ], +) diff --git a/util-inject/src/main/java/com/twitter/inject/flag/Flagged.java b/util-inject/src/main/java/com/twitter/inject/flag/Flagged.java new file mode 100644 index 0000000000..9dcb73e668 --- /dev/null +++ b/util-inject/src/main/java/com/twitter/inject/flag/Flagged.java @@ -0,0 +1,24 @@ +package com.twitter.inject.flag; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import com.google.inject.BindingAnnotation; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * Annotates parsed flags. + * + * @see + * @see + */ +@Retention(RUNTIME) +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@BindingAnnotation +public @interface Flagged { + + /** Name of the flag */ + String value(); +} diff --git a/util-inject/src/main/java/com/twitter/inject/flag/FlaggedImpl.java b/util-inject/src/main/java/com/twitter/inject/flag/FlaggedImpl.java new file mode 100644 index 0000000000..c5fc11c566 --- /dev/null +++ b/util-inject/src/main/java/com/twitter/inject/flag/FlaggedImpl.java @@ -0,0 +1,49 @@ +package com.twitter.inject.flag; + +import java.io.Serializable; +import java.lang.annotation.Annotation; + +/** + * Do not use this class directly. Use {@link Flags#named(String)}. + * + * Pattern copied from com.google.inject.name.NamedImpl. + * @see + * @see Flags + * @see Flagged + */ +class FlaggedImpl implements Flagged, Serializable { + private final String value; + + FlaggedImpl(String value) { + this.value = value; + } + + public String value() { + return this.value; + } + + public int hashCode() { + // This is specified in java.lang.Annotation. + return (127 * "value".hashCode()) ^ value.hashCode(); + } + + /** Flag specific equals */ + public boolean equals(Object o) { + if (!(o instanceof Flagged)) { + return false; + } + + Flagged other = (Flagged) o; + return value.equals(other.value()); + } + + public String toString() { + return "@" + Flagged.class.getName() + "(value=" + value + ")"; + } + + public Class annotationType() { + return Flagged.class; + } + + private static final long serialVersionUID = 0; +} diff --git a/util-inject/src/main/java/com/twitter/inject/flag/Flags.java b/util-inject/src/main/java/com/twitter/inject/flag/Flags.java new file mode 100644 index 0000000000..6b7c3c94a3 --- /dev/null +++ b/util-inject/src/main/java/com/twitter/inject/flag/Flags.java @@ -0,0 +1,30 @@ +package com.twitter.inject.flag; + +import com.google.inject.Key; + +/** + * Utility methods for use with {@code @}{@link Flagged}. Pattern copied from + * {@code com.google.inject.name.Names}. + * + * @see + * @see Flagged + */ +public final class Flags { + private Flags() { + } + + /** Creates a {@link Flagged} annotation with {@code name} as the value. */ + public static Flagged named(String name) { + return new FlaggedImpl(name); + } + + /** + * Builds a {@code com.google.inject.Key} over a String type annotated + * with {@code @}{@link Flagged}(named) + * + * @see + */ + public static Key key(String name) { + return Key.get(String.class, Flags.named(name)); + } +} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/BUILD b/util-inject/src/main/scala/com/twitter/inject/BUILD similarity index 80% rename from util-inject/src/main/scala/com/twitter/util/inject/BUILD rename to util-inject/src/main/scala/com/twitter/inject/BUILD index cf545d03d6..47f79d4198 100644 --- a/util-inject/src/main/scala/com/twitter/util/inject/BUILD +++ b/util-inject/src/main/scala/com/twitter/inject/BUILD @@ -13,10 +13,14 @@ scala_library( "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/javax/inject:javax.inject", "3rdparty/jvm/net/codingwell:scala-guice", + "util/util-app/src/main/scala", + "util/util-slf4j-api/src/main/scala", ], exports = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", "3rdparty/jvm/net/codingwell:scala-guice", + "util/util-app/src/main/scala", + "util/util-slf4j-api/src/main/scala", ], ) diff --git a/util-inject/src/main/scala/com/twitter/inject/Flags.scala b/util-inject/src/main/scala/com/twitter/inject/Flags.scala new file mode 100644 index 0000000000..e562f443f3 --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/Flags.scala @@ -0,0 +1,17 @@ +package com.twitter.inject + +import com.twitter.app.Flag + +/** Transforms a [[com.twitter.app.Flags]] collection into a map look up */ +case class Flags private[inject] (private[inject] val flgs: com.twitter.app.Flags) { + private[this] val underlying: Map[String, Flag[_]] = { + val flags: Seq[Flag[_]] = flgs.getAll(includeGlobal = false).toSeq + flags.map(f => f.name -> f).toMap + } + + /** gets Flag from [com.twitter.app.Flags] map look up */ + def get(name: String): Option[Flag[_]] = underlying.get(name) + + /** gets set of defined Flags */ + def getAll: Seq[Flag[_]] = underlying.values.toSeq +} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/Injector.scala b/util-inject/src/main/scala/com/twitter/inject/Injector.scala similarity index 99% rename from util-inject/src/main/scala/com/twitter/util/inject/Injector.scala rename to util-inject/src/main/scala/com/twitter/inject/Injector.scala index 7264bcc95d..d39107684e 100644 --- a/util-inject/src/main/scala/com/twitter/util/inject/Injector.scala +++ b/util-inject/src/main/scala/com/twitter/inject/Injector.scala @@ -1,4 +1,4 @@ -package com.twitter.util.inject +package com.twitter.inject import com.google.inject.{Key, Injector => UnderlyingInjector} import java.lang.annotation.Annotation diff --git a/util-inject/src/main/scala/com/twitter/inject/InjectorModule.scala b/util-inject/src/main/scala/com/twitter/inject/InjectorModule.scala new file mode 100644 index 0000000000..72f741c64f --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/InjectorModule.scala @@ -0,0 +1,16 @@ +package com.twitter.inject + +import com.google.inject.{Provides, Injector => GuiceInjector} +import javax.inject.Singleton + +object InjectorModule extends TwitterModule { + + @Provides + @Singleton + private def providesInjector(injector: GuiceInjector): Injector = { + Injector(injector) + } + + /** Java-friendly way to access this module as a singleton instance */ + def get(): this.type = this +} diff --git a/util-inject/src/main/scala/com/twitter/inject/TwitterBaseModule.scala b/util-inject/src/main/scala/com/twitter/inject/TwitterBaseModule.scala new file mode 100644 index 0000000000..5938beefde --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/TwitterBaseModule.scala @@ -0,0 +1,24 @@ +package com.twitter.inject + +import com.google.inject.Module + +private[inject] trait TwitterBaseModule extends TwitterModuleFlags with TwitterModuleLifecycle { + + /** + * Additional modules to be composed into this module. This list of modules is generally used + * ''instead'' of the [[com.twitter.inject.TwitterModule.install]] method to properly support the + * [[TwitterModuleLifecycle]] for [[com.twitter.inject.TwitterModule]] instances. + * + * @note [[com.twitter.inject.TwitterModule.install(module: Module)]] can still be used for non-[[TwitterModule]] 1 + * instances, and is sometimes preferred due to `install` being deferred until after flag parsing occurs. + * @note Java users should prefer [[javaModules]]. + */ + protected[inject] def modules: Seq[Module] = Seq.empty + + /** Additional modules to be composed into this module from Java */ + protected[inject] def javaModules: java.util.Collection[Module] = + java.util.Collections.emptyList[Module]() + + /** Additional framework modules to be composed into this module. */ + protected[inject] def frameworkModules: Seq[Module] = Seq.empty +} diff --git a/util-inject/src/main/scala/com/twitter/inject/TwitterModule.scala b/util-inject/src/main/scala/com/twitter/inject/TwitterModule.scala new file mode 100644 index 0000000000..8e5f6459ac --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/TwitterModule.scala @@ -0,0 +1,368 @@ +package com.twitter.inject + +import com.google.inject.assistedinject.FactoryModuleBuilder +import com.google.inject.matcher.{Matcher, Matchers} +import com.google.inject.spi.TypeConverter +import com.google.inject.{Module, _} +import com.twitter.app.Flaggable +import com.twitter.util.logging.Logging +import java.lang.annotation.Annotation +import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder, ScalaOptionBinder, typeLiteral} + +/** + * =Overview= + * A support class for [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/Module.html Module]] + * implementations which exposes a DSL for binding via type parameters. Extend this class, + * override the `configure` method and call the `bind` methods, or define custom `@Provides` + * annotated methods. This class also provides an integration with [[com.twitter.app.Flag]] types + * which allows for passing external configuration to aid in the creation of bound types. Lastly, it + * is also possible to define a list of other [[TwitterModule]] instances which this [[TwitterModule]] + * "depends" on by setting the [[TwitterBaseModule.modules]] (or [[TwitterBaseModule.javaModules]]) + * to a non-empty list. This will ensure that when only this [[TwitterModule]] instance is used to + * compose an [[Injector]] the "dependent" list of modules will also be installed. + * + * =Lifecycle= + * A [[TwitterModule]] has a lifecycle. Executing this lifecycle is particularly important if the + * [[TwitterModule]] creates any [[com.twitter.app.Flag]] instances. + * + * @note Attempting to bind the same type multiple times with no discriminating + * [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/BindingAnnotation.html com.google.inject.BindingAnnotation]] + * will result in an exception during injector construction. + * @example + * {{{ + * object MyModule extends TwitterModule { + * flag[String](name = "card.gateway", help = "The processing gateway to use for credit cards.") + * + * override protected def configure(): Unit = { + * bind[Service].to[ServiceImpl].in[Singleton] + * bind[CreditCardPaymentService] + * bind[Bar[Foo]].to[FooBarImpl] + * bind[PaymentService].to[CreditCardPaymentService] + * } + * + * @Singleton + * @Provides + * def provideCreditCardServiceProcessor( + * @Flag("card.gateway") gateway: String + * ): CreditCardProcessor = { + * new CreditCardProcessor(gateway) + * } + * } + * }}} + * @see [[https://static.javadoc.io/com.google.inject/guice/4.1.0/com/google/inject/AbstractModule.html com.google.inject.AbstractModule]] + * @see [[https://twitter.github.io/finatra/user-guide/getting-started/modules.html Writing Modules in Finatra]] + * @define multiBinding + * An API to bind multiple values separately, only to later inject them as a complete collection. + * "Multibinding" is intended for use in your application's module: + * {{{ + * class SnacksModule extends TwitterModule { + * override protected def configure(): Unit = { + * bindMultiple[Snack].addBinding.toInstance(new Twix()) + * bindMultiple[Snack].addBinding.toProvider[SnickersProvider] + * bindMultiple[Snack].addBinding.to[Skittles] + * } + * } + * }}} + * + * With this binding, a `Set[Snack]` can now be injected: + * {{{ + * class SnackMachine @Inject()(Set[Snack] snacks) + * }}} + * + * Contributing multibindings from different modules is also supported. For example, it is okay + * for both `CandyModule` and `ChipsModule` to create their own `Multibinder[Snack]`, and to each + * contribute bindings to the set of snacks. When that set is injected, it will contain elements + * from both modules. + * + * The set's iteration order is consistent with the binding order. This is convenient when multiple + * elements are contributed by the same module because that module can order its bindings + * appropriately. Avoid relying on the iteration order of elements contributed by different modules, + * since there is no equivalent mechanism to order modules. + * + * The set is unmodifiable. Elements can only be added to the set by configuring the multibinder. + * Elements can never be removed from the set. + * + * Elements are resolved at set injection time. If an element is bound to a provider, that + * provider's get method will be called each time the set is injected (unless the binding is + * also scoped). + * + * Annotations can be used to create different sets of the same element type. Each distinct + * annotation gets its own independent collection of elements. + * + * Elements '''MUST''' be distinct. If multiple bound elements have the same value, set injection will fail. + * + * Elements '''MUST''' be non-null. If any set element is null, set injection will fail. + * @define optionalBinding + * Calling this method will always supply the bindings: `Option[T]` and `Option[Provider[T]]`. If + * `setBinding` or `setDefault` are called, it will also bind `T`. + * + * `setDefault` is intended for use by frameworks that need a default value. User code can call + * `setBinding` to override the default. + * + * '''Warning:''' even if `setBinding` is called, the default binding will still exist in the + * object graph. If it is a singleton, it will be instantiated in `Stage.PRODUCTION`. + * + * If `setDefault` or `setBinding` are linked to Providers, the Provider may return null. If it + * does, the Option bindings will be a None. Binding `setBinding` to a Provider that returns + * null will not cause `OptionalBinder` to fall back to the `setDefault` binding. + * + * If neither `setDefault` nor `setBinding` are called, it will try to link to a user-supplied + * binding of the same type. If no binding exists, the options will be absent. Otherwise, if a + * user-supplied binding of that type exists, or if `setBinding` or `setDefault` are called, the + * options will return `Some(T)` if they are bound to a non-null value, otherwise `None`. + * + * Values are resolved at injection time. If a value is bound to a provider, that provider's + * get method will be called each time the option is injected (unless the binding is also + * scoped, or an option of provider is injected). + * + * Annotations are used to create different options of the same key/value type. Each distinct + * annotation gets its own independent binding. For example: + * {{{ + * class FrameworkModule extends TwitterModule { + * override protected def configure(): Unit = { + * bindOption[Renamer] + * } + * } + * }}} + * + * With this module, an `Option[Renamer]` can now be injected. With no other bindings, the + * option will be None. Users can specify bindings in one of two ways: + * + * Option 1: + * {{{ + * class UserRenamerModule extends TwitterModule { + * override protected def configure(): Unit = { + * bind[Renamer].to[ReplacingRenamer] + * } + * } + * }}} + * or Option 2: + * {{{ + * class UserRenamerModule extends TwitterModule { + * override protected def configure(): Unit = { + * bindOption[Renamer].setBinding.to[ReplacingRenamer] + * } + * } + * }}} + * + * With both options, the `Option[Renamer]` will be present and supply the `ReplacingRenamer`. + * + * Default values can be supplied using: + * {{{ + * class FrameworkModule extends TwitterModule { + * override protected def configure(): Unit = { + * bindOption[String, LookupUrl].setDefault.toInstance(DefaultLookupUrl) + * } + * } + * }}} + * + * With the above module, code can inject an `@LookupUrl`-annotated String and it will supply + * the `DefaultLookupUrl`. A user can change this value by binding: + * {{{ + * class UserLookupModule extends TwitterModule { + * override protected def configure(): Unit = { + * bindOption[String, LookupUrl].setBinding.toInstance(CustomLookupUrl) + * } + * } + * }}} + * + * which will override the default value. + * + * If one module uses `setDefault` the only way to override the default is to use `setBinding`. + * It is an error for a user to specify the binding without using + * [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] + * if `setDefault` or `setBinding` are called. For example: + * {{{ + * class FrameworkModule extends TwitterModule { + * override protected def configure(): Unit = { + * bindOption[String, LookupUrl].setDefault.toInstance(DefaultLookupUrl) + * } + * } + * class UserLookupModule extends TwitterModule { + * override protected def configure(): Unit = { + * bind[String, LookupUrl].toInstance(CustomLookupUrl); + * } + * } + * }}} + * + * would generate an error, because both the framework and the user are trying to bind `@LookupUrl` String. + */ +abstract class TwitterModule + extends AbstractModule + with TwitterBaseModule + with ScalaModule + with Logging { + + /** + * Uses the given [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/Module.html com.google.inject.Module]] + * to configure more bindings. This is not supported for instances of type [[TwitterModule]] and + * will throw an [[UnsupportedOperationException]] if attempted on an instance of [[TwitterModule]]. + * This is to properly support the [[TwitterModuleLifecycle]] for [[TwitterModule]] instances. + * + * @note [[TwitterModule.install(module: Module)]] can still be used for non-[[TwitterModule]] instances, and is + * sometimes preferred due to `install` being deferred until after flag parsing occurs. + */ + @throws[UnsupportedOperationException] + override protected def install(module: Module): Unit = { + module match { + case _: TwitterModule => + throw new UnsupportedOperationException( + "Install not supported for TwitterModules. Please use 'override val modules = Seq(module1, module2, ...)'" + ) + case _ => + super.install(module) + } + } + + /* Protected */ + + /** + * Provides for installing and building a factory that combines the caller's arguments with + * injector-supplied values to construct objects. This is preferable to calling `install` + * on the [[TwitterModule]] which provides the factory as install is not supported for + * [[TwitterModule]] types. + * + * @tparam T type of the assisted injection factory + * + * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/assistedinject/AssistedInject.html com.google.inject.assistedinject.AssistedInject]] + * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/assistedinject/FactoryModuleBuilder.html com.google.inject.assistedinject.FactoryModuleBuilder]] + * @see [[https://github.com/google/guice/wiki/AssistedInject Assisted Injection]] + */ + protected def bindAssistedFactory[T: Manifest](): Unit = { + super.install(new FactoryModuleBuilder().build(manifest[T].runtimeClass)) + } + + /** + * Binds a type converter. The injector will use the given converter to + * convert string constants to matching types as needed. + * + * @param converter converts values + * @tparam T type to match that the converter can handle + */ + protected def addTypeConverter[T: Manifest](converter: TypeConverter): Unit = { + convertToTypes(Matchers.only(typeLiteral[T]), converter) + } + + /** + * Binds a type converter derived from a [[Flaggable]], making it possible to inject flags of all + * kinds. The injector will use a provided `Flaggable` to perform type conversion during an + * injection. + * + * For example (in Scala): + * + * {{{ + * addFlagConverter[List[(Int, Int)]] // support injecting flags of type List[(Int, Int)] + * }}} + * + * @see [[addFlagConverter(Matcher,Flaggable)]] for variant that's more suitable for Java. + */ + protected def addFlagConverter[T <: AnyRef: Manifest](implicit F: Flaggable[T]): Unit = { + addTypeConverter[T](new TypeConverter { + def convert(s: String, typeLiteral: TypeLiteral[_]): AnyRef = F.parse(s) + }) + } + + /** + * Binds a type converter derived from a [[Flaggable]], making it possible to inject flags of all + * kinds. The injector will use a provided `Flaggable` to perform type conversion during an + * injection. + * + * For example (in Java): + * + * {{{ + * import java.util.List; + * import com.google.inject.TypeLiteral; + * import com.google.inject.matcher.Matchers; + * import com.twitter.app.Flaggable; + * + * addFlagConverter( + * Matchers.only(new TypeLiteral>>() {}), + * Flaggable.ofJavaList(Flaggable.ofTuple(Flaggable.ofJavaInteger(), Flaggable.ofJavaInteger()) + * ); + * }}} + * + * @see [[addFlagConverter(Manifest,Flaggable]] for a variant that's more suitable for Scala. + */ + protected def addFlagConverter[T <: AnyRef]( + typeMatcher: Matcher[_ >: TypeLiteral[_]], + F: Flaggable[T] + ): Unit = { + convertToTypes( + typeMatcher, + new TypeConverter { + def convert(s: String, typeLiteral: TypeLiteral[_]): AnyRef = F.parse(s) + } + ) + } + + /** + * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] + * that collects instances of type [[T]] in a [[scala.collection.immutable.Set]] that is itself + * bound with no binding annotation. + * + * $multiBinding + * + * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] + */ + protected def bindMultiple[T: Manifest]: ScalaMultibinder[T] = + ScalaMultibinder.newSetBinder[T](binderAccess) + + /** + * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] + * that collects instances of type [[T]] in a [[scala.collection.immutable.Set]] that is itself bound with a binding + * annotation [[A]]. + * + * $multiBinding + * + * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] + */ + protected def bindMultiple[T: Manifest, A <: Annotation: Manifest]: ScalaMultibinder[T] = + ScalaMultibinder.newSetBinder[T, A](binderAccess) + + /** + * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] + * that collects instances of type [[T]] in a [[scala.collection.immutable.Set]] that is itself + * bound with a binding annotation. + * + * $multiBinding + * + * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] + */ + protected def bindMultiple[T: Manifest](annotation: Annotation): ScalaMultibinder[T] = + ScalaMultibinder.newSetBinder[T](binderAccess, annotation) + + /** + * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] + * that binds an instance of [[T]] in a [[scala.Option]]. + * + * $optionalBinding + * + * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] + */ + protected def bindOption[T: Manifest]: ScalaOptionBinder[T] = + ScalaOptionBinder.newOptionBinder[T](binderAccess) + + /** + * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] + * that binds an instance of [[T]] in a [[scala.Option]] that is itself bound with a binding + * annotation [[A]]. + * + * $optionalBinding + * + * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] + */ + protected def bindOption[T: Manifest, A <: Annotation: Manifest]: ScalaOptionBinder[T] = + ScalaOptionBinder.newOptionBinder[T, A](binderAccess) + + /** + * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] + * that binds an instance of [[T]] in a [[scala.Option]] that is itself bound with a + * binding annotation. + * + * $optionalBinding + * + * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] + */ + protected def bindOption[T: Manifest](annotation: Annotation): ScalaOptionBinder[T] = + ScalaOptionBinder.newOptionBinder[T](binderAccess, annotation) +} diff --git a/util-inject/src/main/scala/com/twitter/inject/TwitterModuleFlags.scala b/util-inject/src/main/scala/com/twitter/inject/TwitterModuleFlags.scala new file mode 100644 index 0000000000..4e17c209fc --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/TwitterModuleFlags.scala @@ -0,0 +1,116 @@ +package com.twitter.inject + +import com.twitter.app.{Flag, Flaggable} +import scala.collection.mutable.ArrayBuffer + +/** [[com.twitter.app.Flag]] integration for a [[TwitterModule]]. */ +private[inject] trait TwitterModuleFlags { + + /* Mutable State */ + + protected[inject] lazy val flags: ArrayBuffer[Flag[_]] = ArrayBuffer[Flag[_]]() + + /* Protected */ + + /** + * This is akin to the `c.t.inject.app.App#failfastOnFlagsNotParsed` and serves + * a similar purpose but for new [[com.twitter.app.Flag]] instances created in this + * [[TwitterModule]]. The value is 'true' by default. This is to ensure that the value of + * a [[com.twitter.app.Flag]] instance created in this [[TwitterModule]] cannot be incorrectly + * accessed before the application has parsed any passed command line input. This mirrors the + * framework default for `com.twitter.inject.app.App#failfastOnFlagsNotParsed` for Flag + * instances created within the application container. + * + * @return a [[Boolean]] indicating if [[com.twitter.app.Flag]] instances created in + * this [[TwitterModule]] should be set with [[com.twitter.app.Flag.failFastUntilParsed]] + * set to 'true' or 'false'. Default: 'true'. + * @note This value SHOULD NOT be changed to 'false' without a very good reason. + */ + protected[this] def failfastOnFlagsNotParsed: Boolean = true + + /** + * A Java-friendly method for creating a named [[Flag]]. + * + * @param name the name of the [[Flag]]. + * @param default a default value for the [[Flag]] when no value is given as an application + * argument. + * @param help the help text explaining the purpose of the [[Flag]]. + * @return the created [[Flag]]. + */ + final def createFlag[T]( + name: String, + default: T, + help: String, + flaggable: Flaggable[T] + ): Flag[T] = { + implicit val implicitFlaggable: Flaggable[T] = flaggable + val flag = new Flag[T](name, help, default, failFastUntilParsed = failfastOnFlagsNotParsed) { + override protected val flaggable: Flaggable[T] = implicitFlaggable + } + flags += flag + flag + } + + /** + * A Java-friendly way to create a "mandatory" [[Flag]]. "Mandatory" flags MUST have a value + * provided as an application argument (as they have no default value to be used). + * + * @param name the name of the [[Flag]]. + * @param help the help text explaining the purpose of the [[Flag]]. + * @param usage a string describing the type of the [[Flag]], i.e.: Integer. + * @return the created [[Flag]]. + */ + final def createMandatoryFlag[T]( + name: String, + help: String, + usage: String, + flaggable: Flaggable[T] + ): Flag[T] = { + implicit val implicitFlaggable: Flaggable[T] = flaggable + val flag = new Flag[T](name, help, usage, failFastUntilParsed = failfastOnFlagsNotParsed) { + override protected val flaggable: Flaggable[T] = implicitFlaggable + } + flags += flag + flag + } + + /** + * Create a [[Flag]] and add it to this Module's flags list. + * + * @note Java users: see the more Java-friendly [[createFlag]] or [[createMandatoryFlag]]. + * + * @param name the name of the [[Flag]]. + * @param default a default value for the [[Flag]] when no value is given as an application + * argument. + * @param help the help text explaining the purpose of the [[Flag]]. + * @tparam T must be a [[Flaggable]] type. + * @return the created [[Flag]]. + */ + final def flag[T: Flaggable](name: String, default: T, help: String): Flag[T] = { + val flag = new Flag[T](name, help, default, failFastUntilParsed = failfastOnFlagsNotParsed) + flags += flag + flag + } + + /** + * Create a "mandatory" flag and add it to this Module's flags list."Mandatory" flags MUST have + * a value provided as an application argument (as they have no default value to be used). + * + * @note Java users: see the more Java-friendly [[createFlag]] or [[createMandatoryFlag]]. + * + * @param name the name of the [[Flag]]. + * @param help the help text explaining the purpose of the [[Flag]]. + * @tparam T must be a [[Flaggable]] type. + * @return the created [[Flag]]. + */ + final def flag[T: Flaggable: Manifest](name: String, help: String): Flag[T] = { + val flag = + new Flag[T]( + name, + help, + manifest[T].runtimeClass.toString, + failFastUntilParsed = failfastOnFlagsNotParsed) + flags += flag + flag + } +} diff --git a/util-inject/src/main/scala/com/twitter/inject/TwitterModuleLifecycle.scala b/util-inject/src/main/scala/com/twitter/inject/TwitterModuleLifecycle.scala new file mode 100644 index 0000000000..a1c5a1bf55 --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/TwitterModuleLifecycle.scala @@ -0,0 +1,83 @@ +package com.twitter.inject + +import com.twitter.util.logging.Logging +import scala.collection.mutable +import scala.util.control.NonFatal + +/** + * [[com.twitter.inject.TwitterModule]] to com.twitter.inject.app.App lifecycle integration. + */ +private[inject] trait TwitterModuleLifecycle extends Logging { + + /* Mutable State */ + private lazy val closableFunctions = mutable.Buffer[() => Unit]() + + /* Protected */ + + /** + * Invoked after the injector is started. + * + * This method should only get singleton instances from the injector. + */ + protected[inject] def singletonStartup(injector: Injector): Unit = {} + + /** + * Invoke after external ports are bound and any clients are resolved + * + * This method should only get singleton instances from the injector. + */ + protected[inject] def singletonPostWarmupComplete(injector: Injector): Unit = {} + + /** + * Invoked on graceful shutdown of the application. + * + * This method should only get singleton instances from the injector. + */ + protected[inject] def singletonShutdown(injector: Injector): Unit = {} + + /** + * Collects functions over a [[com.twitter.util.Closable]]s. These functions will be passed to + * the application `onExit` function to be executed on graceful shutdown of the application. + * + * @note It is expected that the passed function is a function over a [[com.twitter.util.Closable]]. + * + * @param f A Function0 which returns Unit. It is expected that this function encapsulates awaiting + * on a [[com.twitter.util.Closable]] that the application would like to ensure is closed + * upon graceful shutdown. + * + * {{{ + * closeOnExit { + * val closable = ... + * Await.result( + * closable.close(after: Duration), timeout: Duration) + * } + * }}} + * + * @see [[com.twitter.app.App.onExit(f: => Unit)]] + * @see [[com.twitter.util.Awaitable]] + * @see [[com.twitter.util.Closable]] + */ + protected def closeOnExit(f: => Unit): Unit = { + // `c.t.app.App#onExit` will swallow any exception from a given function, thus we try to ensure + // some visibility into any potential exception resulting from executing the passed function. + def tryCatchFn(fn: => Unit): () => Unit = { () => + { + try { + fn + } catch { + case NonFatal(e) => + warn(e.getMessage) + debug(s"An error occurred in ${getClass.getSimpleName}#closeOnExit", e) + } + } + } + + closableFunctions += tryCatchFn(f) + } + + /* Private */ + + private[inject] def close(): Unit = { + closableFunctions.foreach(_()) + } +} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala b/util-inject/src/main/scala/com/twitter/inject/TypeUtils.scala similarity index 98% rename from util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala rename to util-inject/src/main/scala/com/twitter/inject/TypeUtils.scala index 15a162e68c..83af0d05b3 100644 --- a/util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala +++ b/util-inject/src/main/scala/com/twitter/inject/TypeUtils.scala @@ -1,4 +1,4 @@ -package com.twitter.util.inject +package com.twitter.inject import com.google.inject.TypeLiteral import com.google.inject.internal.MoreTypes.ParameterizedTypeImpl diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/BUILD b/util-inject/src/main/scala/com/twitter/inject/internal/BUILD new file mode 100644 index 0000000000..81cf77bb49 --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/internal/BUILD @@ -0,0 +1,24 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-inject-internal", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/google/inject:guice", + "3rdparty/jvm/javax/inject:javax.inject", + "util/util-app/src/main/scala", + "util/util-core:util-core-util", + "util/util-inject/src/main/java/com/twitter/inject/flag", + "util/util-inject/src/main/scala/com/twitter/inject", + "util/util-slf4j-api/src/main/scala", + ], + exports = [ + "util/util-app/src/main/scala", + "util/util-inject/src/main/java/com/twitter/inject/flag", + "util/util-slf4j-api/src/main/scala", + ], +) diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/FlagsModule.scala b/util-inject/src/main/scala/com/twitter/inject/internal/FlagsModule.scala new file mode 100644 index 0000000000..d98c2a6423 --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/internal/FlagsModule.scala @@ -0,0 +1,69 @@ +package com.twitter.inject.internal + +import com.google.inject.{AbstractModule, Key} +import com.twitter.inject.flag.Flags +import com.twitter.util.logging.Logging +import javax.inject.Provider + +/** + * Note this is purposely *not* an instance of a [[com.twitter.inject.TwitterModule]] + * as we use no lifecycle callbacks nor does this define any [[com.twitter.app.Flag]]s. + * Additionally, we do not want mistakenly use or bind the [[com.twitter.inject.TwitterModule#flags]] + * reference thus we extend [[AbstractModule]] directly. + * + * This module is solely intended to create object graph bindings for the parsed value of + * every non-global [[com.twitter.app.Flag]] contained in the passed [[com.twitter.app.Flags]] instance. + * If the [[com.twitter.app.Flag]] instance does not have a parsed value a `Provider[Nothing]` + * is provided which when de-referenced will throw an [[IllegalArgumentException]]. + */ +private[inject] class FlagsModule(flags: com.twitter.app.Flags) + extends AbstractModule + with Logging { + + /** + * Currently, we are unable to bind to anything other than a [[String]] type as Flags do not carry + * enough type information due to type erasure. We would need to update [[com.twitter.app.Flag]] + * or [[com.twitter.app.Flaggable]] to carry Manifest or TypeTag information which we could use + * here to determine the correct type for building an instance Key. Or re-create the + * [[com.twitter.app.Flaggable]] logic in a registered TypeConverter. + * + * In the interim, we install the Guice default type conversions along with the [[TwitterTypeConvertersModule]] + * to provide a limited set of conversions by the injector from [[String]] to another type. Thus + * you can inject a [[com.twitter.app.Flag]] of some non-[[String]] types. But this is not all of + * the same types supported by [[com.twitter.app.Flaggable]] and [[com.twitter.app.Flag]]. Some + * mismatch in the functionality is expected as the integration is done manually. That said, + * defining a custom `Flaggable` instance for type X doesn't guarantee it can be injected without + * registering a flag converter. + * + * @see [[TwitterTypeConvertersModule]] + * @see [[https://github.com/google/guice/blob/master/core/src/com/google/inject/internal/TypeConverterBindingProcessor.java Guice Default Type Conversions]] + */ + override def configure(): Unit = { + // bind the c.t.inject.Flags instance to the object graph + binder.bind(classOf[com.twitter.inject.Flags]).toInstance(com.twitter.inject.Flags(flags)) + + // bind every parsed (non-global) flag that has a value keyed by flag name + val asList = flags.getAll(includeGlobal = false).toSeq + val flagsMap = (for (flag <- asList) yield { + flag.name -> flag.getWithDefaultUnparsed + }).toMap + + for ((flagName, valueOpt) <- flagsMap) { + val key: Key[String] = Flags.key(flagName) + valueOpt match { + case Some(value) => + debug("Binding flag: " + flagName + " = " + value) + binder.bind(key).toInstance(value) + case None => + binder + .bind(key) + .toProvider(new Provider[Nothing] { + override def get() = + throw new IllegalArgumentException( + "flag: " + flagName + " has an unspecified value and is not eligible for @Flag injection" + ) + }) + } + } + } +} diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/InstalledModules.scala b/util-inject/src/main/scala/com/twitter/inject/internal/InstalledModules.scala new file mode 100644 index 0000000000..0faaf1cba2 --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/internal/InstalledModules.scala @@ -0,0 +1,82 @@ +package com.twitter.inject.internal + +import com.google.inject.Module +import com.twitter.inject.{Injector, TwitterModuleLifecycle} +import com.twitter.util.logging.Logging +import scala.PartialFunction.condOpt + +/** + * A representation of the "installed" [[com.google.inject.Module]]s, that is + * the created [[com.twitter.inject.Injector]] and the [[com.google.inject.Module]] of which it + * is comprised. Creating the [[com.twitter.inject.Injector]] loses the [[com.google.inject.Module]]s + * over which it was created, but we need to hold onto the list in order to apply the application lifecycle + * to the [[com.twitter.inject.TwitterModuleLifecycle]] versions of the contained [[com.google.inject.Module]]s. + * + * @param injector the [[com.twitter.inject.Injector]] built from the [[com.google.inject.Module]]s + * @param modules the list of [[com.google.inject.Module]]s from which the [[com.twitter.inject.Injector]] was created. + */ +private[inject] case class InstalledModules(injector: Injector, modules: Seq[Module]) + extends Logging { + + def postInjectorStartup(): Unit = { + modules.foreach { + case injectModule: TwitterModuleLifecycle => + try { + injectModule.singletonStartup(injector) + } catch { + case e: Throwable => + error("Startup method error in " + injectModule, e) + throw e + } + case _ => + } + } + + def postWarmupComplete(): Unit = { + modules.foreach { + case injectModule: TwitterModuleLifecycle => + try { + injectModule.singletonPostWarmupComplete(injector) + } catch { + case e: Throwable => + error("Post warmup complete method error in " + injectModule, e) + throw e + } + case _ => + } + } + + /** + * Collect shutdown `ExitFunctions` for [[com.google.inject.Module]] instances + * which implement the [[TwitterModuleLifecycle]] + */ + def shutdown(): Seq[ExitFunction] = { + condOptModules(modules)(_.singletonShutdown(injector)) + } + + /** + * Collect close `ExitFunctions` for [[com.google.inject.Module]] instances which + * implement the [[TwitterModuleLifecycle]] + */ + def close(): Seq[ExitFunction] = { + condOptModules(modules)(_.close()) + } + + /* Private */ + + /** + * Iterates through the list of Modules to match only instances of TwitterModuleLifecycle + * on which to create an `ExitFunction` over the passed in TwitterModuleLifecycle function. + * @see [[scala.PartialFunction.condOpt]] + */ + private[this] def condOptModules( + modules: Seq[Module] + )( + fn: TwitterModuleLifecycle => Unit + ): Seq[ExitFunction] = modules.flatMap { module => + condOpt(module) { + case injectModule: TwitterModuleLifecycle => + () => fn(injectModule) + } + } +} diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/Modules.scala b/util-inject/src/main/scala/com/twitter/inject/internal/Modules.scala new file mode 100644 index 0000000000..21944b0d18 --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/internal/Modules.scala @@ -0,0 +1,113 @@ +package com.twitter.inject.internal + +import com.google.inject.{Guice, Module, Stage} +import com.twitter.app.{Flag, Flags} +import com.twitter.inject.{Injector, TwitterBaseModule} +import com.twitter.inject.internal.iterable._ +import scala.collection.JavaConverters._ + +private[inject] object Modules { + + /** + * De-duplicate a given sequence of [[com.google.inject.Module]]. + * Exposed for testing. + */ + def distinctModules( + modules: Seq[com.google.inject.Module] + ): Seq[com.google.inject.Module] = { + // De-dupe using a `java.util.IdentityHashMap` which uses reference equality instead + // of object equality to check if both objects point to the same memory location + // and filter out modules already seen. We use `filter` because it's stable, and + // the order of module initialization may be important. + val identityHashMap = new java.util.IdentityHashMap[com.google.inject.Module, Boolean]() + modules.filter { module => + if (identityHashMap.containsKey(module)) false + else { + identityHashMap.put(module, true) + true + } + } + } + + /** Capture all flags in the [[com.google.inject.Module]] object hierarchy. */ + private def findModuleFlags(modules: Seq[com.google.inject.Module]): Seq[Flag[_]] = { + // Flags are stored in the App `com.twitter.app.Flags` member variable which is a + // Map[String, Flag[_]], where the key is the Flag#name. Thus, to ensure we correctly account + // for the "override" behavior of Flag#add (the last Flag with the same name added, wins), + // we need to ensure that we "reverse" our Sequence before performing a distinct operation + // discriminated by Flag#name. + modules + .collect { + case injectModule: TwitterBaseModule => injectModule.flags + }.flatten.reverse.distinctBy(_.name) + } + + /** Recursively find all 'composed' modules */ + private def findInstalledModules( + module: com.google.inject.Module + ): Seq[com.google.inject.Module] = module match { + case injectModule: TwitterBaseModule => + injectModule.modules ++ + injectModule.modules.flatMap(findInstalledModules) ++ + injectModule.javaModules.asScala.toSeq ++ + injectModule.javaModules.asScala.toSeq.flatMap(findInstalledModules) ++ + injectModule.frameworkModules ++ + injectModule.frameworkModules.flatMap(findInstalledModules) + case _ => + Seq.empty + } + +} + +/** + * Provides a flattened view of the given lists of [[Module]]s. That is if the [[Module]] is + * a [[[com.twitter.inject.TwitterBaseModule]] type it may reference other [[Module]]s to + * be included when creating the injector. This class will perform the recursive lookup of + * all the referenced [[Module]]s from the give input lists and store each as a flattened + * sequence to be used for injector creation. + * + * We only want to compute all of the composed modules and the flags contained therein + * once as calling the [[com.twitter.inject.TwitterBaseModule#modules]] or + * [[com.twitter.inject.TwitterBaseModule#javaModules]] or + * [[com.twitter.inject.TwitterBaseModule#frameworkModules]] methods is not guaranteed + * to be idempotent. This class is a holder of the computed modules and flags for + * the containing [[com.twitter.inject.app.App]]. + * + * @see [[com.twitter.inject.TwitterBaseModule]] + * @see [[com.twitter.inject.TwitterBaseModule#modules]] + * @see [[com.twitter.inject.TwitterBaseModule#javaModules]] + * @see [[com.twitter.inject.TwitterBaseModule#frameworkModules]] + */ +private[inject] class Modules(required: Seq[Module], overrides: Seq[Module]) { + import Modules._ + + val modules: Seq[Module] = { + val composedModules = required.flatMap(findInstalledModules) + required ++ composedModules + } + + val overrideModules: Seq[Module] = { + val composedOverrideModules = overrides.flatMap(findInstalledModules) + overrides ++ composedOverrideModules + } + + val moduleFlags: Seq[Flag[_]] = + findModuleFlags(distinctModules(modules ++ overrideModules)) + + def addFlags(flag: Flags): Unit = moduleFlags.foreach(flag.add) + + def install( + flags: Flags, + stage: Stage + ): InstalledModules = { + // ensure we add the FlagsModule and the TwitterTypeConvertersModule to the list to build the injector. + val requiredModules = modules ++ Seq(new FlagsModule(flags), TwitterTypeConvertersModule) + val combinedModule = com.google.inject.util.Modules + .`override`(requiredModules.asJava).`with`(overrideModules.asJava) + + InstalledModules( + injector = Injector(Guice.createInjector(stage, combinedModule)), + modules = distinctModules(requiredModules ++ overrideModules) + ) + } +} diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/TwitterTypeConvertersModule.scala b/util-inject/src/main/scala/com/twitter/inject/internal/TwitterTypeConvertersModule.scala new file mode 100644 index 0000000000..641d2a2c58 --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/internal/TwitterTypeConvertersModule.scala @@ -0,0 +1,46 @@ +package com.twitter.inject.internal + +import com.twitter.inject.TwitterModule +import com.twitter.util.{Duration, StorageUnit, Time} +import java.io.File +import java.net.InetSocketAddress +import java.time.LocalTime + +private[inject] object TwitterTypeConvertersModule extends TwitterModule { + + override def configure(): Unit = { + // Single value type converters + addFlagConverter[LocalTime] + addFlagConverter[Time] + addFlagConverter[Duration] + addFlagConverter[StorageUnit] + addFlagConverter[InetSocketAddress] + addFlagConverter[File] + + // Multi-value type converters (comma-separated) for scala.Seq + addFlagConverter[Seq[String]] + addFlagConverter[Seq[Int]] + addFlagConverter[Seq[Long]] + addFlagConverter[Seq[Double]] + addFlagConverter[Seq[Float]] + addFlagConverter[Seq[LocalTime]] + addFlagConverter[Seq[Time]] + addFlagConverter[Seq[Duration]] + addFlagConverter[Seq[StorageUnit]] + addFlagConverter[Seq[InetSocketAddress]] + addFlagConverter[Seq[File]] + + // Multi-value type converters (comma-separated) for java.util.List + addFlagConverter[java.util.List[String]] + addFlagConverter[java.util.List[Int]] + addFlagConverter[java.util.List[Long]] + addFlagConverter[java.util.List[Double]] + addFlagConverter[java.util.List[Float]] + addFlagConverter[java.util.List[LocalTime]] + addFlagConverter[java.util.List[Time]] + addFlagConverter[java.util.List[Duration]] + addFlagConverter[java.util.List[StorageUnit]] + addFlagConverter[java.util.List[InetSocketAddress]] + addFlagConverter[java.util.List[File]] + } +} diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/iterable.scala b/util-inject/src/main/scala/com/twitter/inject/internal/iterable.scala new file mode 100644 index 0000000000..c5128d8156 --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/internal/iterable.scala @@ -0,0 +1,37 @@ +package com.twitter.inject.internal + +import scala.collection.generic.CanBuildFrom +import scala.collection.mutable.{HashSet => MutableHashSet} +import scala.language.higherKinds + +private[inject] object iterable { + + implicit class RichIterable[Elem, From[Elem] <: Iterable[Elem]](val self: From[Elem]) + extends AnyVal { + + /** + * Distinct 'iterable' elements using the passed in 'hash' function + * @param hash Hash function to determine unique elements + * @return Distinct elements + */ + def distinctBy[HashCodeType]( + hash: Elem => HashCodeType + )( + implicit cbf: CanBuildFrom[From[Elem], Elem, From[Elem]] + ): From[Elem] = { + val builder = cbf() + val seen = MutableHashSet[HashCodeType]() + + for (elem <- self) { + if (!seen(hash(elem))) { + seen += hash(elem) + builder += elem + } + } + + builder.result() + } + + } + +} diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/package.scala b/util-inject/src/main/scala/com/twitter/inject/internal/package.scala new file mode 100644 index 0000000000..d121a18bec --- /dev/null +++ b/util-inject/src/main/scala/com/twitter/inject/internal/package.scala @@ -0,0 +1,5 @@ +package com.twitter.inject + +package object internal { + type ExitFunction = () => Unit +} diff --git a/util-inject/src/test/java/com/twitter/util/inject/BUILD b/util-inject/src/test/java/com/twitter/inject/BUILD similarity index 100% rename from util-inject/src/test/java/com/twitter/util/inject/BUILD rename to util-inject/src/test/java/com/twitter/inject/BUILD diff --git a/util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java b/util-inject/src/test/java/com/twitter/inject/TestBindingAnnotation.java similarity index 93% rename from util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java rename to util-inject/src/test/java/com/twitter/inject/TestBindingAnnotation.java index 588c744cc7..ba1219d18b 100644 --- a/util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java +++ b/util-inject/src/test/java/com/twitter/inject/TestBindingAnnotation.java @@ -1,4 +1,4 @@ -package com.twitter.util.inject; +package com.twitter.inject; import java.lang.annotation.Retention; import java.lang.annotation.Target; diff --git a/util-inject/src/test/scala/com/twitter/util/inject/BUILD b/util-inject/src/test/scala/com/twitter/inject/BUILD similarity index 79% rename from util-inject/src/test/scala/com/twitter/util/inject/BUILD rename to util-inject/src/test/scala/com/twitter/inject/BUILD index c13d7fd0bc..9592a332d1 100644 --- a/util-inject/src/test/scala/com/twitter/util/inject/BUILD +++ b/util-inject/src/test/scala/com/twitter/inject/BUILD @@ -10,7 +10,7 @@ junit_tests( "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/scalatestplus:scalacheck", - "util/util-inject/src/main/scala/com/twitter/util/inject", - "util/util-inject/src/test/java/com/twitter/util/inject", + "util/util-inject/src/main/scala/com/twitter/inject", + "util/util-inject/src/test/java/com/twitter/inject", ], ) diff --git a/util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala b/util-inject/src/test/scala/com/twitter/inject/InjectorTest.scala similarity index 86% rename from util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala rename to util-inject/src/test/scala/com/twitter/inject/InjectorTest.scala index 982cb753d2..b58bb9ee6b 100644 --- a/util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala +++ b/util-inject/src/test/scala/com/twitter/inject/InjectorTest.scala @@ -1,48 +1,16 @@ -package com.twitter.util.inject +package com.twitter.inject import com.google.inject._ import com.google.inject.name.Names -import com.twitter.util.inject.classes.{TypeProvider, _} -import com.twitter.util.inject.module.TestModule -import javax.inject.{Inject, Named, Singleton} +import com.twitter.inject.classes.{TypeProvider, _} +import com.twitter.inject.module.{SeqAbstractModule, TestModule} +import javax.inject.{Named, Singleton} import net.codingwell.scalaguice.ScalaModule import org.junit.runner.RunWith import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatestplus.junit.JUnitRunner -object SeqAbstractModule extends AbstractModule with ScalaModule { - @Singleton - @Provides - def provideSeq: Seq[Array[Byte]] = - Seq.empty[Array[Byte]] -} - -trait MyServiceInterface { - - def add2(int: Int): Int -} - -class MyServiceImpl extends MyServiceInterface { - override def add2(int: Int): Int = { - int + 2 - } -} - -class SomeClient { - protected val underlying: String = "Hello, Alice." - def get: String = underlying -} - -trait Augmentation { self: SomeClient => - override def get: String = s"${self.underlying} Welcome to Wonderland." - def onlyHere: String = "Only on augmented classes." -} - -class Baz @Inject() (client: SomeClient with Augmentation) { - client.onlyHere -} - object InjectorTest { type AugmentedSomeClient = SomeClient with Augmentation } diff --git a/util-inject/src/test/scala/com/twitter/inject/TwitterModuleTest.scala b/util-inject/src/test/scala/com/twitter/inject/TwitterModuleTest.scala new file mode 100644 index 0000000000..ef3159ee37 --- /dev/null +++ b/util-inject/src/test/scala/com/twitter/inject/TwitterModuleTest.scala @@ -0,0 +1,543 @@ +package com.twitter.inject + +import com.google.inject._ +import com.google.inject.name.Names +import com.twitter.inject.servicefactory.ComplexServiceFactory +import com.twitter.inject.classes._ +import com.twitter.inject.module.{ClassToConvert, SeqAbstractModule, TestTwitterModule} +import java.lang.annotation.Annotation +import javax.inject.{Named, Provider, Singleton} +import net.codingwell.scalaguice.ScalaModule +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import scala.collection.immutable +import scala.util.Random + +object TwitterModuleTest { + type AugmentedSomeClient = SomeClient with Augmentation +} + +class TwitterModuleTest + extends AnyFunSuite + with Matchers + with BeforeAndAfterAll + with BeforeAndAfterEach { + import TwitterModuleTest._ + + private case class W[T](t: T) + private val annotation = Names.named(Random.alphanumeric.take(12).mkString.toLowerCase()) + + // Users should prefer the `c.t.inject.app.TestInjector` but it is not in scope here. + private val injector: Injector = Injector(Guice.createInjector(TestTwitterModule)) + + override def beforeAll(): Unit = { + super.beforeAll() + TestTwitterModule.singletonStartup(injector) + } + + override def afterAll(): Unit = { + TestTwitterModule.singletonShutdown(injector) + super.afterAll() + } + + test("byte array generics") { + import net.codingwell.scalaguice.InjectorExtensions._ + val i = Guice.createInjector(SeqAbstractModule) + + i.instance[Seq[Array[Byte]]] should equal(Seq.empty[Array[Byte]]) + } + + test("can't bind class with trait and class") { + intercept[CreationException] { + Guice.createInjector( + new AbstractModule with ScalaModule { + @Provides + @Singleton + def provideSomeClient: SomeClient with Augmentation = { + new SomeClient with Augmentation + } + }, + new AbstractModule with ScalaModule { + @Provides + @Singleton + def provideSomeClient: SomeClient = { + new SomeClient + } + } + ) + } + } + + test("class with trait type") { + import net.codingwell.scalaguice.InjectorExtensions._ + val i = Guice.createInjector( + Stage.PRODUCTION, + new AbstractModule with ScalaModule { + @Provides + @Singleton + def provideSomeClient: SomeClient with Augmentation = { + new SomeClient with Augmentation + } + }) + + val r1 = i.instance[SomeClient with Augmentation] + r1.get should equal("Hello, Alice. Welcome to Wonderland.") + + val r2 = i.instance[AugmentedSomeClient] + r2.get should equal("Hello, Alice. Welcome to Wonderland.") + } + + test("class only type") { + import net.codingwell.scalaguice.InjectorExtensions._ + val i = Guice.createInjector( + Stage.PRODUCTION, + new AbstractModule with ScalaModule { + @Provides + @Singleton + def provideSomeClient: SomeClient = { + new SomeClient + } + }) + + // fails since only SomeClient is bound + val e = intercept[ProvisionException] { + i.instance[Baz] + } + e.getCause.isInstanceOf[ClassCastException] should be(true) + } + + test("get assisted factory instance from injector") { + assertServiceFactory(injector.instance[ComplexServiceFactory]) + assertServiceFactory(injector.instance(classOf[ComplexServiceFactory])) + assertServiceFactory(injector.instance(Key.get(classOf[ComplexServiceFactory]))) + } + + test("get additional instances from injector") { + injector.instance[String](Names.named("str1")) should be("string1") + injector.instance[String, TestBindingAnnotation] should be("prod string") + } + + test("type conversion") { + injector.instance[ClassToConvert](Names.named("name")) should be(ClassToConvert("Steve")) + } + + test("allow binding target type using a type parameter") { + val module = new TwitterModule { + override protected def configure(): Unit = { + bind[X].to[Y] + } + } + + Injector(Guice.createInjector(module)).instance[X] + } + + test("allow binding target provider type using a type parameter") { + val module = new TwitterModule { + override protected def configure(): Unit = { + bind[X].toProvider[YProvider] + } + } + + Injector(Guice.createInjector(module)).instance[X] + } + + test("allow binding to provider of subtype using type parameter") { + val module = new TwitterModule { + override protected def configure(): Unit = { + bind[Gen[String]].toProvider[ZProvider] + } + } + + Injector(Guice.createInjector(module)).instance(new Key[Gen[String]] {}) + } + + test("allow binding to provider with injected type literal") { + val module = new TwitterModule { + override protected def configure(): Unit = { + bind[String].toProvider[TypeProvider[B]] + } + } + + Injector(Guice.createInjector(module)).instance(new Key[String] {}) + } + + test("allow binding in scope using a type parameter") { + val module = new TwitterModule { + override protected def configure(): Unit = { + bind[X].to[Y].in[Singleton] + } + } + + Injector(Guice.createInjector(module)).instance[X] + } + + test("allow binding a container with a generic singleton type") { + val module = new TwitterModule { + override protected def configure(): Unit = { + bind[SealedTraitContainer[FinalSealedTrait.type]] + .toProvider[SealedTraitContainerFinalSealedTraitProvider] + } + } + + Injector(Guice.createInjector(module)) + .instance(new Key[SealedTraitContainer[FinalSealedTrait.type]] {}) + } + + test("allow binding with annotation using a type parameter") { + val module = new TwitterModule { + override protected def configure(): Unit = { + bind[X].annotatedWith[Named].to[Y] + } + } + + Injector(Guice.createInjector(module)).instance(Key.get(classOf[X], classOf[Named])) + } + + test("allow use provider form javax.inject.Provider") { + val module = new TwitterModule { + override def configure(): Unit = { + bind[Foo].toProvider[FooProviderWithJavax] + } + } + + Injector(Guice.createInjector(module)).instance[Foo] + } + + test("provide helpful error when bound to self") { + val module = new TwitterModule { + override def configure(): Unit = { + bind[X].to[X] + } + } + val e = intercept[CreationException] { + Injector(Guice.createInjector(module)).instance[X] + } + val messages = e.getErrorMessages + messages.size should equal(1) + e.getStackTrace.exists(_.getClassName == this.getClass.getName) + val sources = messages.iterator.next.getSource + sources.contains(this.getClass.getName) should be(true) + } + + test("multibinder#bind empty [T]") { + val module = new TwitterModule { + override def configure(): Unit = { + bindMultiple[String] + } + } + validate(Injector(Guice.createInjector(module)).instance[immutable.Set[String]]) + } + + test("multibinder#bind [T]") { + val module = new TwitterModule { + override def configure(): Unit = { + bindMultiple[String].addBinding.toInstance("A") + bindMultiple[String].addBinding.toInstance("B") + } + } + validate(Injector(Guice.createInjector(module)).instance[immutable.Set[String]], "A", "B") + } + + test("multibinder#bind [T, Ann]") { + val module = new TwitterModule { + override def configure(): Unit = { + val multi = bindMultiple[String, TestBindingAnnotation] + multi.addBinding.toInstance("A") + multi.addBinding.toInstance("B") + } + } + validate( + Injector(Guice.createInjector(module)).instance[immutable.Set[String], TestBindingAnnotation], + "A", + "B") + } + + test("multibinder#bind [T, Annotation]") { + val module = new TwitterModule { + override def configure(): Unit = { + val multi = bindMultiple[String](annotation) + multi.addBinding.toInstance("A") + multi.addBinding.toInstance("B") + } + } + validate( + Injector(Guice.createInjector(module)).instance[immutable.Set[String]](annotation), + "A", + "B") + } + + test("multibinder#deduplicate") { + val module = new TwitterModule { + override def configure(): Unit = { + bindMultiple[Symbol].addBinding.toInstance('A) + bindMultiple[Symbol].addBinding.toInstance('A) + } + } + validate(Injector(Guice.createInjector(module)).instance[immutable.Set[Symbol]], 'A) + } + + test("multibinder#permit duplicates") { + val module = new TwitterModule { + override def configure(): Unit = { + val multi = bindMultiple[Symbol].permitDuplicates() // silently discards duplicates + multi.addBinding.toInstance('A) + multi.addBinding.toInstance('A) + } + } + validate(Injector(Guice.createInjector(module)).instance[immutable.Set[Symbol]], 'A) + } + + test("multibinder#bind from multiple modules") { + def newModule(i: Int) = new TwitterModule { + override def configure(): Unit = { + val multi = bindMultiple[Int] + multi.addBinding.toInstance(i) + } + } + + validate( + Injector(Guice.createInjector(newModule(1), newModule(2))).instance[immutable.Set[Int]], + 1, + 2) + } + + test("multibinder#bind parameterized [T]") { + val module = new TwitterModule { + override def configure(): Unit = { + val mbStrings = bindMultiple[W[String]] + mbStrings.addBinding.toInstance(W("A")) + val mbInts = bindMultiple[W[Int]] + mbInts.addBinding.toInstance(W(1)) + } + } + + val injector = Injector(Guice.createInjector(module)) + validate(injector.instance[immutable.Set[W[String]]], W("A")) + validate(injector.instance[immutable.Set[W[Int]]], W(1)) + } + + test("multibinder#bind parameterized [T, Ann]") { + val module = new TwitterModule { + override def configure(): Unit = { + val mbStrings = bindMultiple[W[String], TestBindingAnnotation] + mbStrings.addBinding.toInstance(W("A")) + val mbInts = bindMultiple[W[Int], TestBindingAnnotation] + mbInts.addBinding.toInstance(W(1)) + } + } + + val injector = Injector(Guice.createInjector(module)) + validate(injector.instance[immutable.Set[W[String]], TestBindingAnnotation], W("A")) + validate(injector.instance[immutable.Set[W[Int]], TestBindingAnnotation], W(1)) + } + + test("multibinder#bind parameterized [T, Annotation]") { + val module = new TwitterModule { + override def configure(): Unit = { + val mbStrings = bindMultiple[W[String]](annotation) + mbStrings.addBinding.toInstance(W("A")) + val mbInts = bindMultiple[W[Int]](annotation) + mbInts.addBinding.toInstance(W(1)) + } + } + + val injector = Injector(Guice.createInjector(module)) + validate(injector.instance[immutable.Set[W[String]]](annotation), W("A")) + validate(injector.instance[immutable.Set[W[Int]]](annotation), W(1)) + } + + test("optionalbinder#bind [T]") { + val module = new TwitterModule { + override def configure(): Unit = { + bindOption[String].setDefault.toInstance("A") + } + } + + validate(module) + } + + test("optionalbinder#bind [T, Ann]") { + val module = new TwitterModule { + override def configure(): Unit = { + bindOption[String, Named].setDefault.toInstance("A") + } + } + + validateWithAnn[String, Named](module) + } + + test("optionalbinder#bind [T, Annotation]") { + val module = new TwitterModule { + override def configure(): Unit = { + val opt = bindOption[String](annotation) + opt.setDefault.toInstance("A") + } + } + + validateWithAnnotation(module, annotation, expected = "A") + } + + test("optionalbinder#bind optional without default") { + val module = new TwitterModule { + override def configure(): Unit = { + bindOption[String] + } + } + + validateAbsent(module) + } + + test("optionalbinder#override default value") { + val module = new TwitterModule { + override def configure(): Unit = { + val opt = bindOption[String] + opt.setDefault.toInstance("B") + opt.setBinding.toInstance("A") + } + } + + validate(module) + } + + test("optionalbinder#override default value with second binder") { + val module = new TwitterModule { + override def configure(): Unit = { + bindOption[String].setDefault.toInstance("B") + bindOption[String].setBinding.toInstance("A") + } + } + + validate(module) + } + + test("optionalbinder#allow separately annotated bindings for the same [T]") { + val module = new TwitterModule { + override def configure(): Unit = { + val aOpt = bindOption[String](Names.named("A")) + aOpt.setBinding.toInstance("A") + val bOpt = bindOption[String](Names.named("B")) + bOpt.setDefault.toInstance("B") + } + } + + validateWithAnnotation(module, Names.named("A"), expected = "A") + validateWithAnnotation(module, Names.named("B"), expected = "B") + } + + test("optionalbinder#bind parameterized [T]") { + val module = new TwitterModule { + override def configure(): Unit = { + val sOpt = bindOption[W[String]] + sOpt.setDefault.toInstance(W("A")) + val nOpt = bindOption[W[Int]] + nOpt.setDefault.toInstance(W(1)) + } + } + + validate(module, expected = W("A")) + validate(module, expected = W(1)) + } + + test("optionalbinder#bind parameterized [T, Ann]") { + val module = new TwitterModule { + override def configure(): Unit = { + val sOpt = bindOption[W[String], TestBindingAnnotation] + sOpt.setDefault.toInstance(W("A")) + val nOpt = bindOption[W[Int], TestBindingAnnotation] + nOpt.setDefault.toInstance(W(1)) + } + } + + validateWithAnn[W[String], TestBindingAnnotation](module, expected = W("A")) + validateWithAnn[W[Int], TestBindingAnnotation](module, expected = W(1)) + } + + test("optionalbinder#bind parameterized [T, Annotation]") { + val module = new TwitterModule { + override def configure(): Unit = { + val sOpt = bindOption[W[String]](annotation) + sOpt.setDefault.toInstance(W("A")) + val nOpt = bindOption[W[Int]](annotation) + nOpt.setDefault.toInstance(W(1)) + } + } + + validateWithAnnotation(module, annotation, expected = W("A")) + validateWithAnnotation(module, annotation, expected = W(1)) + } + + private[this] def assertServiceFactory(complexServiceFactory: ComplexServiceFactory): Unit = { + complexServiceFactory.create("foo").execute should equal("done foo") + } + + private[this] def validate[T](set: Set[T], expected: T*): Unit = { + set should have size expected.length + for (e <- expected) { + set should contain(e) + } + } + + private[this] def validate[T: Manifest](module: Module, expected: T = "A"): Unit = { + val injector = Injector(Guice.createInjector(module)) + + // Check Option + injector.instance[Option[T]] should contain(expected) + injector.instance[Option[Provider[T]]].get.get() should equal(expected) + injector.instance[Option[javax.inject.Provider[T]]].get.get() should equal(expected) + + // Check Option + injector.instance[Option[T]].get should equal(expected) + injector.instance[Option[Provider[T]]].get.get() should equal(expected) + injector.instance[Option[javax.inject.Provider[T]]].get.get() should equal(expected) + } + + private def validateWithAnn[T: Manifest, Ann <: Annotation: Manifest]( + module: Module, + expected: T = "A" + ): Unit = { + val injector = Injector(Guice.createInjector(module)) + + // Check Option + injector.instance[Option[T], Ann] should contain(expected) + injector.instance[Option[Provider[T]], Ann].get.get() should equal(expected) + injector.instance[Option[javax.inject.Provider[T]], Ann].get.get() should equal(expected) + + // Check Option + injector.instance[Option[T], Ann].get should equal(expected) + injector.instance[Option[Provider[T]], Ann].get.get() should equal(expected) + injector.instance[Option[javax.inject.Provider[T]], Ann].get.get() should equal(expected) + } + + private def validateWithAnnotation[T: Manifest]( + module: Module, + annotation: Annotation, + expected: T + ): Unit = { + val injector = Injector(Guice.createInjector(module)) + + // Check Option + injector.instance[Option[T]](annotation) should contain(expected) + injector.instance[Option[Provider[T]]](annotation).get.get() should equal(expected) + injector.instance[Option[javax.inject.Provider[T]]](annotation).get.get() should equal(expected) + + // Check Option + injector.instance[Option[T]](annotation).get should equal(expected) + injector.instance[Option[Provider[T]]](annotation).get.get() should equal(expected) + injector.instance[Option[javax.inject.Provider[T]]](annotation).get.get() should equal(expected) + } + + private def validateAbsent[T: Manifest](module: Module, expected: T = "A"): Unit = { + val injector = Injector(Guice.createInjector(module)) + + // Check Option + injector.instance[Option[T]] should be(None) + injector.instance[Option[Provider[T]]] should be(None) + injector.instance[Option[javax.inject.Provider[T]]] should be(None) + + // Check Option + injector.instance[Option[T]].isDefined should be(false) + injector.instance[Option[Provider[T]]].isDefined should be(false) + injector.instance[Option[javax.inject.Provider[T]]].isDefined should be(false) + } +} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala b/util-inject/src/test/scala/com/twitter/inject/TypeUtilsTest.scala similarity index 98% rename from util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala rename to util-inject/src/test/scala/com/twitter/inject/TypeUtilsTest.scala index 1c7cb6bbb4..b2567b9b06 100644 --- a/util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala +++ b/util-inject/src/test/scala/com/twitter/inject/TypeUtilsTest.scala @@ -1,4 +1,4 @@ -package com.twitter.util.inject +package com.twitter.inject import org.scalacheck.Arbitrary import org.scalatest.funsuite.AnyFunSuite diff --git a/util-inject/src/test/scala/com/twitter/util/inject/classes.scala b/util-inject/src/test/scala/com/twitter/inject/classes.scala similarity index 65% rename from util-inject/src/test/scala/com/twitter/util/inject/classes.scala rename to util-inject/src/test/scala/com/twitter/inject/classes.scala index 86373267f2..059bdac24a 100644 --- a/util-inject/src/test/scala/com/twitter/util/inject/classes.scala +++ b/util-inject/src/test/scala/com/twitter/inject/classes.scala @@ -1,8 +1,42 @@ -package com.twitter.util.inject +package com.twitter.inject import com.google.inject.{Provider, TypeLiteral} import javax.inject.{Inject, Named} +class SomeClient { + protected val underlying: String = "Hello, Alice." + def get: String = underlying +} + +trait Augmentation { self: SomeClient => + override def get: String = s"${self.underlying} Welcome to Wonderland." + def onlyHere: String = "Only on augmented classes." +} + +class Baz @Inject() (client: SomeClient with Augmentation) { + client.onlyHere +} +trait MyServiceInterface { + + def add2(int: Int): Int +} + +class MyServiceImpl extends MyServiceInterface { + override def add2(int: Int): Int = { + int + 2 + } +} +trait MultiService { + val name: String +} + +class OneMultiService extends MultiService { + override val name = "one" +} + +class TwoMultiService extends MultiService { + override val name = "two" +} object classes { object Outer { trait X diff --git a/util-inject/src/test/scala/com/twitter/inject/module/SeqAbstractModule.scala b/util-inject/src/test/scala/com/twitter/inject/module/SeqAbstractModule.scala new file mode 100644 index 0000000000..0b9f52d7c1 --- /dev/null +++ b/util-inject/src/test/scala/com/twitter/inject/module/SeqAbstractModule.scala @@ -0,0 +1,12 @@ +package com.twitter.inject.module + +import com.google.inject.{AbstractModule, Provides} +import javax.inject.Singleton +import net.codingwell.scalaguice.ScalaModule + +object SeqAbstractModule extends AbstractModule with ScalaModule { + @Singleton + @Provides + def provideSeq: Seq[Array[Byte]] = + Seq.empty[Array[Byte]] +} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala b/util-inject/src/test/scala/com/twitter/inject/module/TestModule.scala similarity index 84% rename from util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala rename to util-inject/src/test/scala/com/twitter/inject/module/TestModule.scala index 1ed54c9181..7e13802ac6 100644 --- a/util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala +++ b/util-inject/src/test/scala/com/twitter/inject/module/TestModule.scala @@ -1,8 +1,9 @@ -package com.twitter.util.inject.module +package com.twitter.inject.module import com.google.inject.AbstractModule import com.google.inject.name.Names -import com.twitter.util.inject.{MyServiceImpl, MyServiceInterface, TestBindingAnnotation} +import com.twitter.inject.TestBindingAnnotation +import com.twitter.inject.{MyServiceImpl, MyServiceInterface} import java.util.Properties import javax.inject.Singleton import net.codingwell.scalaguice.ScalaModule diff --git a/util-inject/src/test/scala/com/twitter/inject/module/TestTwitterModule.scala b/util-inject/src/test/scala/com/twitter/inject/module/TestTwitterModule.scala new file mode 100644 index 0000000000..e195050ccc --- /dev/null +++ b/util-inject/src/test/scala/com/twitter/inject/module/TestTwitterModule.scala @@ -0,0 +1,136 @@ +package com.twitter.inject.module + +import com.google.inject.name.{Named, Names} +import com.google.inject.spi.TypeConverter +import com.google.inject.{Provides, TypeLiteral} +import com.twitter.app.Flag +import com.twitter.conversions.DurationOps._ +import com.twitter.inject.servicefactory.ComplexServiceFactory +import com.twitter.inject.{ + Injector, + MultiService, + MyServiceImpl, + MyServiceInterface, + OneMultiService, + TestBindingAnnotation, + TwitterModule, + TwoMultiService +} +import javax.inject.Singleton +import java.util.Properties +import org.scalatest.Assertions +import scala.util.control.NonFatal + +// for testing the function/API of a TwitterModule -- not for general testing +object TestTwitterModule extends TwitterModule with Assertions { + + // Note: The following flag values are not used in this module, but are @Flag injected elsewhere + flag("moduleMagicNum", "30", "Module Magic number") + flag("moduleDuration", 5.seconds, "Module duration") + + val localModuleFlag: Flag[String] = flag("moduleString", "default", "A passed string") + + override protected def configure(): Unit = { + bind[String].annotatedWith(Names.named("str1")).toInstance("string1") + bind[String].annotatedWith(Names.named("str2")).toInstance("string2") + + bind[Int].toInstance(11) + bind[String].toInstance("default string") + bind[String].annotatedWith[TestBindingAnnotation].toInstance("prod string") + + bind[MyServiceInterface].to[MyServiceImpl].in[Singleton] + + bindAssistedFactory[ComplexServiceFactory]() + getProvider[Int] + + bindMultiple[MultiService].addBinding.to[OneMultiService] + bindMultiple[MultiService].addBinding.to[TwoMultiService] + + bindOption[String](Names.named("SomeString")) + bindOption[Double](Names.named("OptionalDouble")).setDefault.toInstance(142d) + bindOption[Float].setBinding.toInstance(137f) + bindOption[String](Names.named("DoEverythingString")).setBinding.toInstance("NOTDEFAULT") + + addTypeConverter[ClassToConvert](new TypeConverter { + override def convert(s: String, typeLiteral: TypeLiteral[_]): AnyRef = { + ClassToConvert(s) + } + }) + + val properties = new Properties() + properties.setProperty("name", "Steve") + Names.bindProperties(binder(), properties) + } + + override def singletonStartup(injector: Injector): Unit = { + assert( + !localModuleFlag.isDefined + ) // isDefined is only true when a value has been parsed, will be false if the flag still has the default + try { + localModuleFlag() + fail() + } catch { + case NonFatal(_) => + // expected: // no flag parsing happens (since we are creating the injector manually) so the flag read fails + } + assert(injector.instance[String, TestBindingAnnotation] == "prod string") + assert(injector.instance[String, TestBindingAnnotation] == "prod string") + + // check that the Set was bound with two elements + val services = injector.instance[Set[MultiService]] + assert(services.size == 2) + services.foreach { + case _: OneMultiService => assert(true) + case _: TwoMultiService => assert(true) + case _ => assert(false) + } + + // binder created with no binding, lookup of Option[String] should return None + assert(injector.instance[Option[String]](Names.named("SomeString")).isEmpty) + + // binds the Option[Double] + assert(injector.instance[Option[Double]](Names.named("OptionalDouble")).isDefined) + assert(injector.instance[Option[Double]](Names.named("OptionalDouble")).get == 142d) + // should also bind the Double, since we called ".setDefault" + assert(injector.instance[Double](Names.named("OptionalDouble")) == 142d) + + assert(injector.instance[Option[Float]].isDefined) + assert(injector.instance[Option[Float]].get == 137f) + + // no binder was ever created here + try { + injector.instance[Option[String]](Names.named("MyString")) + fail() + } catch { + case NonFatal(e) => + // ensure that the DoEverythingModule is in the stack trace + assert(e.getStackTrace.exists(_.getClassName == TestTwitterModule.getClass.getName)) + } + + // binds the Option[String] + assert(injector.instance[Option[String]](Names.named("DoEverythingString")).isDefined) + assert(injector.instance[Option[String]](Names.named("DoEverythingString")).get == "NOTDEFAULT") + // should also bind the String, since we called ".setBinding" + assert(injector.instance[String](Names.named("DoEverythingString")) == "NOTDEFAULT") + + val myService = injector.instance[MyServiceInterface] + val myService2 = injector.instance[MyServiceInterface] + assert(myService.toString == myService2.toString) + } + + override def singletonPostWarmupComplete(injector: Injector): Unit = { + info("module post warmup complete") + } + + override def singletonShutdown(injector: Injector): Unit = { + info("shutdown") + } + + @Provides + @Named("example") + def provideNamedString: String = { + "named" + } +} + +case class ClassToConvert(name: String) diff --git a/util-inject/src/test/scala/com/twitter/inject/servicefactory.scala b/util-inject/src/test/scala/com/twitter/inject/servicefactory.scala new file mode 100644 index 0000000000..6d1821e21d --- /dev/null +++ b/util-inject/src/test/scala/com/twitter/inject/servicefactory.scala @@ -0,0 +1,32 @@ +package com.twitter.inject + +import com.google.inject.assistedinject.Assisted +import javax.inject.{Inject, Named, Singleton} + +object servicefactory { + + @Singleton + class DoEverythingService { + + def doit = { + "done" + } + } + + class ComplexService @Inject() ( + exampleService: DoEverythingService, + defaultString: String, + @Named("str1") string1: String, + @Named("str2") string2: String, + defaultInt: Int, + @Assisted name: String) { + + def execute = { + exampleService.doit + " " + name + } + } + + trait ComplexServiceFactory { + def create(name: String): ComplexService + } +} From ad616bad950ecd7223604e0abf5e6deb78468001 Mon Sep 17 00:00:00 2001 From: Martijn Hoekstra Date: Wed, 19 Aug 2020 18:08:24 +0000 Subject: [PATCH 373/796] util: use scalatest non-deprecated names Problem Use of many deprecated names since upgrading to scalatest 3.1. Solution Run using autofix from https://github.com/scalatest/autofix/tree/master/3.1.x Result All deprecated names replaced with their new names. Signed-off-by: Bryce Anderson JIRA Issues: CSL-10095 Differential Revision: https://phabricator.twitter.biz/D535435 --- .../scala/com/twitter/app/lifecycle/NotifierTest.scala | 4 ++-- util-app/src/test/scala/com/twitter/app/AppTest.scala | 4 ++-- .../src/test/scala/com/twitter/app/ClassPathTest.scala | 4 ++-- util-app/src/test/scala/com/twitter/app/FlagTest.scala | 4 ++-- .../src/test/scala/com/twitter/app/FlaggableTest.scala | 4 ++-- util-app/src/test/scala/com/twitter/app/FlagsTest.scala | 4 ++-- .../src/test/scala/com/twitter/app/GlobalFlagTest.scala | 4 ++-- .../src/test/scala/com/twitter/app/LoadServiceTest.scala | 4 ++-- .../scala/com/twitter/cache/AbstractFutureCacheTest.scala | 4 ++-- .../twitter/cache/AbstractLoadingFutureCacheTest.scala | 4 ++-- .../test/scala/com/twitter/cache/EvictingCacheTest.scala | 4 ++-- .../scala/com/twitter/cache/LazilyEvictingCacheTest.scala | 4 ++-- .../src/test/scala/com/twitter/cache/RefreshTest.scala | 4 ++-- .../test/scala/com/twitter/util/StringEncoderTest.scala | 8 ++++---- .../scala/com/twitter/concurrent/AsyncMeterTest.scala | 4 ++-- .../scala/com/twitter/concurrent/AsyncMutexTest.scala | 4 ++-- .../scala/com/twitter/concurrent/AsyncQueueTest.scala | 4 ++-- .../scala/com/twitter/concurrent/AsyncSemaphoreTest.scala | 4 ++-- .../scala/com/twitter/concurrent/AsyncStreamTest.scala | 4 ++-- .../test/scala/com/twitter/concurrent/BrokerTest.scala | 4 ++-- .../src/test/scala/com/twitter/concurrent/OfferTest.scala | 4 ++-- .../src/test/scala/com/twitter/concurrent/OnceTest.scala | 4 ++-- .../test/scala/com/twitter/concurrent/PeriodTest.scala | 4 ++-- .../test/scala/com/twitter/concurrent/SchedulerTest.scala | 7 ++++--- .../scala/com/twitter/concurrent/SerializedTest.scala | 4 ++-- .../scala/com/twitter/concurrent/SpoolSourceTest.scala | 4 ++-- .../src/test/scala/com/twitter/concurrent/SpoolTest.scala | 4 ++-- .../src/test/scala/com/twitter/concurrent/TxTest.scala | 4 ++-- .../scala/com/twitter/conversions/DurationOpsTest.scala | 4 ++-- .../scala/com/twitter/conversions/OpsTogetherTest.scala | 4 ++-- .../scala/com/twitter/conversions/PercentOpsTest.scala | 4 ++-- .../com/twitter/conversions/StorageUnitOpsTest.scala | 4 ++-- .../test/scala/com/twitter/conversions/U64OpsTest.scala | 4 ++-- .../test/scala/com/twitter/io/ActivitySourceTest.scala | 4 ++-- .../src/test/scala/com/twitter/io/BufByteWriterTest.scala | 4 ++-- .../test/scala/com/twitter/io/BufInputStreamTest.scala | 4 ++-- .../src/test/scala/com/twitter/io/BufReaderTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/io/BufTest.scala | 4 ++-- .../src/test/scala/com/twitter/io/ByteReaderTest.scala | 4 ++-- .../scala/com/twitter/io/CachingActivitySourceTest.scala | 4 ++-- .../com/twitter/io/ClassLoaderActivitySourceTest.scala | 4 ++-- .../com/twitter/io/FilePollingActivitySourceTest.scala | 5 +++-- util-core/src/test/scala/com/twitter/io/FilesTest.scala | 4 ++-- .../test/scala/com/twitter/io/InputStreamReaderTest.scala | 4 ++-- .../test/scala/com/twitter/io/IteratorReaderTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/io/PipeTest.scala | 5 +++-- .../test/scala/com/twitter/io/ReaderSemanticsTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/io/ReaderTest.scala | 5 +++-- .../src/test/scala/com/twitter/io/StreamIOTest.scala | 4 ++-- .../src/test/scala/com/twitter/io/TempDirectoryTest.scala | 4 ++-- .../src/test/scala/com/twitter/io/TempFileTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/io/WriterTest.scala | 5 +++-- .../src/test/scala/com/twitter/util/ActivityTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/Base64LongTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/CancellableTest.scala | 4 ++-- .../test/scala/com/twitter/util/ClosableOnceTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/ClosableTest.scala | 4 ++-- .../test/scala/com/twitter/util/CloseAwaitablyTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/ConfigTest.scala | 5 +++-- util-core/src/test/scala/com/twitter/util/DiffTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/util/EventTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/FutureOfferTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/FuturePoolTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/FutureTest.scala | 4 ++-- .../scala/com/twitter/util/LastWriteWinsQueueTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/util/LocalTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/MemoizeTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/MonitorTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/NetUtilTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/util/PoolTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/PromiseTest.scala | 4 ++-- .../scala/com/twitter/util/SlowProbeProxyTimerTest.scala | 4 ++-- .../test/scala/com/twitter/util/StateMachineTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/StorageUnitTest.scala | 5 ++--- .../scala/com/twitter/util/StringConversionsTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/ThrowablesTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/util/TimeTest.scala | 6 +++--- util-core/src/test/scala/com/twitter/util/TimerTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/TokenBucketTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/util/TryTest.scala | 4 ++-- .../scala/com/twitter/util/TwitterDateFormatTest.scala | 4 ++-- util-core/src/test/scala/com/twitter/util/VarTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/WaitQueueTest.scala | 4 ++-- .../test/scala/com/twitter/util/WindowedAdderTest.scala | 4 ++-- .../hashing/ConsistentHashingDistributorTest.scala | 4 ++-- .../test/scala/com/twitter/hashing/DiagnosticsTest.scala | 4 ++-- .../src/test/scala/com/twitter/hashing/HashableTest.scala | 4 ++-- .../test/scala/com/twitter/hashing/KeyHasherTest.scala | 4 ++-- .../src/test/scala/com/twitter/jvm/ContentionTest.scala | 4 ++-- .../src/test/scala/com/twitter/jvm/CpuProfileTest.scala | 4 ++-- .../src/test/scala/com/twitter/jvm/EstimatorTest.scala | 4 ++-- util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala | 4 ++-- .../src/test/scala/com/twitter/jvm/NumProcsTest.scala | 4 ++-- util-jvm/src/test/scala/com/twitter/jvm/OptTest.scala | 4 ++-- .../scala/com/twitter/util/lint/GlobalRulesTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/lint/RuleTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/lint/RulesTest.scala | 4 ++-- .../test/scala/com/twitter/logging/FileHandlerTest.scala | 4 ++-- .../test/scala/com/twitter/logging/FormatterTest.scala | 5 ++--- .../test/scala/com/twitter/logging/HasLogLevelTest.scala | 4 ++-- .../test/scala/com/twitter/logging/LogRecordTest.scala | 4 ++-- .../src/test/scala/com/twitter/logging/LoggerTest.scala | 5 +++-- .../src/test/scala/com/twitter/logging/PolicyTest.scala | 5 ++--- .../scala/com/twitter/logging/QueueingHandlerTest.scala | 4 ++-- .../scala/com/twitter/logging/SyslogHandlerTest.scala | 4 ++-- .../scala/com/twitter/logging/ThrottledHandlerTest.scala | 5 +++-- .../test/scala/com/twitter/util/reflect/ProxyTest.scala | 4 ++-- .../scala/com/twitter/util/registry/FormatterTest.scala | 4 ++-- .../scala/com/twitter/util/registry/LibraryTest.scala | 4 ++-- .../scala/com/twitter/util/registry/RegistryTest.scala | 4 ++-- .../test/scala/com/twitter/util/registry/RosterTest.scala | 4 ++-- .../util/routing/PolymorphicRouterCompilationTest.scala | 4 ++-- .../com/twitter/util/routing/RouterBuilderTest.scala | 4 ++-- .../test/scala/com/twitter/util/routing/RouterTest.scala | 4 ++-- .../scala/com/twitter/util/security/CredentialsTest.scala | 4 ++-- .../com/twitter/util/security/NullPrincipalTest.scala | 4 ++-- .../twitter/util/security/NullSslSessionContextTest.scala | 4 ++-- .../com/twitter/util/security/NullSslSessionTest.scala | 4 ++-- .../scala/com/twitter/util/security/PemFileTest.scala | 4 ++-- .../util/security/Pkcs8EncodedKeySpecFileTest.scala | 4 ++-- .../util/security/Pkcs8KeyManagerFactoryTest.scala | 4 ++-- .../com/twitter/util/security/PrivateKeyFileTest.scala | 4 ++-- .../com/twitter/util/security/X500PrincipalInfoTest.scala | 4 ++-- .../twitter/util/security/X509CertificateFileTest.scala | 4 ++-- .../scala/com/twitter/util/security/X509CrlFileTest.scala | 4 ++-- .../util/security/X509TrustManagerFactoryTest.scala | 4 ++-- .../test/scala/com/twitter/util/logging/LoggingTest.scala | 5 +++-- .../com/twitter/util/logging/MarkerLoggingTest.scala | 5 +++-- .../twitter/util/logging/SerializableLoggingTest.scala | 4 ++-- .../finagle/stats/BroadcastStatsReceiverTest.scala | 5 +++-- .../scala/com/twitter/finagle/stats/CachedRegexTest.scala | 4 ++-- .../stats/CategorizingExceptionStatsHandlerTest.scala | 4 ++-- .../com/twitter/finagle/stats/CumulativeGaugeTest.scala | 4 ++-- .../finagle/stats/DelegatingStatsReceiverTest.scala | 4 ++-- .../twitter/finagle/stats/DenylistStatsReceiverTest.scala | 4 ++-- .../twitter/finagle/stats/InMemoryStatsReceiverTest.scala | 4 ++-- .../com/twitter/finagle/stats/LazyStatsReceiverTest.scala | 4 ++-- .../MultiCategorizingExceptionStatsHandlerTest.scala | 4 ++-- .../com/twitter/finagle/stats/StatsReceiverTest.scala | 4 ++-- .../stats/VerbosityAdjustingStatsReceiverTest.scala | 5 +++-- .../twitter/finagle/stats/WithHistogramDetailsTest.scala | 4 ++-- .../src/main/scala/com/twitter/logging/TestLogging.scala | 5 +++-- .../com/twitter/util/testing/ArgumentCaptureTest.scala | 4 ++-- .../src/test/scala/com/twitter/util/ThriftCodecTest.scala | 4 ++-- .../scala/com/twitter/util/ThriftSerializerTest.scala | 4 ++-- .../com/twitter/util/tunable/JsonTunableMapperTest.scala | 4 ++-- .../util/tunable/ServiceLoadedTunableMapTest.scala | 4 ++-- .../scala/com/twitter/util/tunable/TunableMapTest.scala | 6 +++--- .../test/scala/com/twitter/util/tunable/TunableTest.scala | 4 ++-- .../test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala | 5 +++-- util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala | 4 ++-- util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala | 4 ++-- util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala | 4 ++-- .../twitter/zk/coordination/ShardCoordinatorTest.scala | 4 ++-- .../twitter/zk/coordination/ZkAsyncSemaphoreTest.scala | 4 ++-- 155 files changed, 329 insertions(+), 318 deletions(-) diff --git a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/NotifierTest.scala b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/NotifierTest.scala index eef5439bfb..4b07284520 100644 --- a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/NotifierTest.scala +++ b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/NotifierTest.scala @@ -1,9 +1,9 @@ package com.twitter.app.lifecycle -import org.scalatest.FunSuite import Event._ +import org.scalatest.funsuite.AnyFunSuite -class NotifierTest extends FunSuite { +class NotifierTest extends AnyFunSuite { test("emits success") { var success = false diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 4d2d541486..85bf6690ff 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -6,9 +6,9 @@ import com.twitter.app.LoadService.Binding import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.util.concurrent.ConcurrentLinkedQueue -import org.scalatest.FunSuite import scala.jdk.CollectionConverters._ import scala.language.reflectiveCalls +import org.scalatest.funsuite.AnyFunSuite class TestApp(f: () => Unit) extends App { var reason: Option[String] = None @@ -60,7 +60,7 @@ trait ErrorOnExitApp extends App { } } -class AppTest extends FunSuite { +class AppTest extends AnyFunSuite { test("App: make sure system.exit called on exception from main") { val test1 = new TestApp(() => throw new RuntimeException("simulate main failing")) diff --git a/util-app/src/test/scala/com/twitter/app/ClassPathTest.scala b/util-app/src/test/scala/com/twitter/app/ClassPathTest.scala index a7b525ea15..d2752fbb85 100644 --- a/util-app/src/test/scala/com/twitter/app/ClassPathTest.scala +++ b/util-app/src/test/scala/com/twitter/app/ClassPathTest.scala @@ -2,10 +2,10 @@ package com.twitter.app import java.net.URLClassLoader import org.mockito.Mockito._ -import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite -class ClassPathTest extends FunSuite with MockitoSugar { +class ClassPathTest extends AnyFunSuite with MockitoSugar { test("Null URL[] URLClassloader") { val classLoader = mock[URLClassLoader] diff --git a/util-app/src/test/scala/com/twitter/app/FlagTest.scala b/util-app/src/test/scala/com/twitter/app/FlagTest.scala index c714ec3117..b40eab67e5 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagTest.scala @@ -3,10 +3,10 @@ package com.twitter.app import com.twitter.app.SolarSystemPlanets._ import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Awaitable} -import org.scalatest.FunSuite import scala.collection.mutable +import org.scalatest.funsuite.AnyFunSuite -class FlagTest extends FunSuite { +class FlagTest extends AnyFunSuite { class Ctx(failFastUntilParsed: Boolean = false) { val flag = new Flags("test", includeGlobal = false, failFastUntilParsed) diff --git a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala index 43b6616828..2000d8eb1a 100644 --- a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala @@ -1,10 +1,10 @@ package com.twitter.app -import org.scalatest.FunSuite import com.twitter.util.{Duration, Time} import java.time.LocalTime +import org.scalatest.funsuite.AnyFunSuite -class FlaggableTest extends FunSuite { +class FlaggableTest extends AnyFunSuite { test("Flaggable: parse booleans") { assert(Flaggable.ofBoolean.parse("true")) diff --git a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala index efdc97a460..2d48d45302 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala @@ -1,9 +1,9 @@ package com.twitter.app import com.twitter.util.registry.{Entry, GlobalRegistry, SimpleRegistry} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class FlagsTest extends FunSuite { +class FlagsTest extends AnyFunSuite { class Ctx(failFastUntilParsed: Boolean = false) { val flag = new Flags("test", includeGlobal = false, failFastUntilParsed) diff --git a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala index 0f96629fa0..f17cb34d8e 100644 --- a/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/GlobalFlagTest.scala @@ -1,12 +1,12 @@ package com.twitter.app -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite object MyGlobalFlag extends GlobalFlag[String]("a test flag", "a global test flag") object MyGlobalFlagNoDefault extends GlobalFlag[Int]("a global test flag with no default") object MyGlobalBooleanFlag extends GlobalFlag[Boolean](false, "a boolean flag") -class GlobalFlagTest extends FunSuite { +class GlobalFlagTest extends AnyFunSuite { val flagSet = Set(MyGlobalFlag, MyGlobalBooleanFlag, MyGlobalFlagNoDefault) test("GlobalFlag.get") { diff --git a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala index b0c34d393f..5495e9bea5 100644 --- a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala +++ b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala @@ -11,8 +11,8 @@ import java.util import java.util.concurrent.{Callable, CountDownLatch, ExecutorService, Executors} import java.util.concurrent.{Future => JFuture} import java.util.Random -import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite // These traits correspond to files in: // util-app/src/test/resources/META-INF/services @@ -51,7 +51,7 @@ class LoadServiceDeadlockImpl extends LoadServiceDeadlock { UsesLoadService.lsd } -class LoadServiceTest extends FunSuite with MockitoSugar { +class LoadServiceTest extends AnyFunSuite with MockitoSugar { test("LoadService should apply[T] and return a set of instances of T") { assert(LoadService[LoadServiceRandomInterface]().nonEmpty) diff --git a/util-cache/src/test/scala/com/twitter/cache/AbstractFutureCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/AbstractFutureCacheTest.scala index d3b3e9fed2..07b3036984 100644 --- a/util-cache/src/test/scala/com/twitter/cache/AbstractFutureCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/AbstractFutureCacheTest.scala @@ -1,10 +1,10 @@ package com.twitter.cache import com.twitter.util.{Future, Promise} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite // TODO: should also check for races -abstract class AbstractFutureCacheTest extends FunSuite { +abstract class AbstractFutureCacheTest extends AnyFunSuite { def name: String diff --git a/util-cache/src/test/scala/com/twitter/cache/AbstractLoadingFutureCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/AbstractLoadingFutureCacheTest.scala index 5367d2bfb0..27fd175b1a 100644 --- a/util-cache/src/test/scala/com/twitter/cache/AbstractLoadingFutureCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/AbstractLoadingFutureCacheTest.scala @@ -2,9 +2,9 @@ package com.twitter.cache import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -abstract class AbstractLoadingFutureCacheTest extends FunSuite { +abstract class AbstractLoadingFutureCacheTest extends AnyFunSuite { // NB we can't reuse AbstractFutureCacheTest since // loading cache semantics are sufficiently unique // to merit distinct tests. diff --git a/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala index a08c598def..1868542aed 100644 --- a/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala @@ -3,10 +3,10 @@ package com.twitter.cache import com.twitter.util.{Promise, Future} import java.util.concurrent.ConcurrentHashMap import org.mockito.Mockito.{verify, never} -import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite -class EvictingCacheTest extends FunSuite with MockitoSugar { +class EvictingCacheTest extends AnyFunSuite with MockitoSugar { test("EvictingCache should evict on failed futures for set") { val cache = mock[FutureCache[String, String]] val fCache = new EvictingCache(cache) diff --git a/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala index 6947b61c0d..8cbce28c25 100644 --- a/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/LazilyEvictingCacheTest.scala @@ -5,10 +5,10 @@ import com.twitter.cache.caffeine.LoadingFutureCache import com.twitter.util.{Await, Future, Promise} import org.mockito.ArgumentMatchers._ import org.mockito.Mockito.{never, verify, when} -import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite -class LazilyEvictingCacheTest extends FunSuite with MockitoSugar { +class LazilyEvictingCacheTest extends AnyFunSuite with MockitoSugar { private val explodingCacheLoader = new CacheLoader[String, Future[String]] { def load(k: String): Future[String] = diff --git a/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala b/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala index aba8a3393d..bf872583dd 100644 --- a/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/RefreshTest.scala @@ -3,10 +3,10 @@ package com.twitter.cache import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, Promise, Time} import org.mockito.Mockito._ -import org.scalatest._ import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite -class RefreshTest extends FunSuite with MockitoSugar { +class RefreshTest extends AnyFunSuite with MockitoSugar { class Ctx { val provider = mock[() => Future[Int]] diff --git a/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala b/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala index c653281214..774155b0d0 100644 --- a/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala +++ b/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala @@ -18,9 +18,9 @@ package com.twitter.util import java.lang.StringBuilder -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class StringEncoderTest extends WordSpec { +class StringEncoderTest extends AnyWordSpec { val longString = "A string that is really really really really really really long and has more than 76 characters" val result = @@ -35,7 +35,7 @@ class StringEncoderTest extends WordSpec { } } -class Base64StringEncoderTest extends WordSpec { +class Base64StringEncoderTest extends AnyWordSpec { val urlUnsafeBytes = Array(-1.toByte, -32.toByte) val resultUnsafe = "/+A" val resultSafe = "_-A" @@ -55,7 +55,7 @@ class Base64StringEncoderTest extends WordSpec { } } -class GZIPStringEncoderTest extends WordSpec { +class GZIPStringEncoderTest extends AnyWordSpec { "a gzip string encoder" should { val gse = new GZIPStringEncoder {} "properly encode and decode strings" in { diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala index 869ef46bef..625d62ac13 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala @@ -3,9 +3,9 @@ package com.twitter.concurrent import com.twitter.util._ import com.twitter.conversions.DurationOps._ import java.util.concurrent.{RejectedExecutionException, CancellationException} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class AsyncMeterTest extends FunSuite { +class AsyncMeterTest extends AnyFunSuite { import AsyncMeter._ // Workaround methods for dealing with Scala compiler warnings: diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncMutexTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncMutexTest.scala index db42ec9c38..d01281401e 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncMutexTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncMutexTest.scala @@ -1,9 +1,9 @@ package com.twitter.concurrent -import org.scalatest.FlatSpec import com.twitter.util.Await +import org.scalatest.flatspec.AnyFlatSpec -class AsyncMutexTest extends FlatSpec { +class AsyncMutexTest extends AnyFlatSpec { "AsyncMutex" should "admit only one operation at a time" in { val m = new AsyncMutex diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncQueueTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncQueueTest.scala index 71a2d4669b..38658bb7f1 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncQueueTest.scala @@ -1,10 +1,10 @@ package com.twitter.concurrent import com.twitter.util.{Await, Awaitable, Duration, Return, Throw} -import org.scalatest.FunSuite import scala.collection.immutable.Queue +import org.scalatest.funsuite.AnyFunSuite -class AsyncQueueTest extends FunSuite { +class AsyncQueueTest extends AnyFunSuite { private def await[T](t: Awaitable[T]): T = Await.result(t, Duration.fromSeconds(5)) diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index cadbc2cf1f..d7d30ee8cc 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -3,11 +3,11 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.util.concurrent.{ConcurrentLinkedQueue, RejectedExecutionException, CountDownLatch} -import org.scalatest.fixture.FunSpec import scala.collection.mutable import org.scalatest.Outcome +import org.scalatest.funspec.FixtureAnyFunSpec -class AsyncSemaphoreTest extends FunSpec { +class AsyncSemaphoreTest extends FixtureAnyFunSpec { class AsyncSemaphoreHelper( val sem: AsyncSemaphore, var count: Int, diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 0ac81b6e04..60b394610b 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -3,10 +3,10 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ import com.twitter.util._ import org.scalacheck.{Arbitrary, Gen} -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class AsyncStreamTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class AsyncStreamTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { import AsyncStream.{mk, of} import AsyncStreamTest._ diff --git a/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala b/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala index a3cf23567e..34ea24cb50 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala @@ -1,9 +1,9 @@ package com.twitter.concurrent import com.twitter.util.{Await, Return, ObjectSizeCalculator} -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class BrokerTest extends WordSpec { +class BrokerTest extends AnyWordSpec { "Broker" should { "send data (send, recv)" in { val br = new Broker[Int] diff --git a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala index 971ed45051..277404c8cd 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala @@ -3,10 +3,10 @@ package com.twitter.concurrent import scala.util.Random import org.mockito.Mockito._ -import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, MockTimer, Promise, Return, Time} +import org.scalatest.wordspec.AnyWordSpec class SimpleOffer[T](var futures: Stream[Future[Tx[T]]]) extends Offer[T] { def this(fut: Future[Tx[T]]) = this(Stream.continually(fut)) @@ -19,7 +19,7 @@ class SimpleOffer[T](var futures: Stream[Future[Tx[T]]]) extends Offer[T] { } } -class OfferTest extends WordSpec with MockitoSugar { +class OfferTest extends AnyWordSpec with MockitoSugar { def await[A](f: Future[A]): A = Await.result(f, 2.seconds) diff --git a/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala index 868ba58903..afca463483 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OnceTest.scala @@ -2,9 +2,9 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ import com.twitter.util.{Promise, Await} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class OnceTest extends FunSuite { +class OnceTest extends AnyFunSuite { test("Once.apply should only be applied once") { var x = 0 val once = Once { x += 1 } diff --git a/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala b/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala index ee45366c0c..2e0822040e 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/PeriodTest.scala @@ -1,9 +1,9 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class PeriodTest extends FunSuite { +class PeriodTest extends AnyFunSuite { test("Period#numPeriods should behave reasonably") { val period = new Period(1.second) val dur = 1.second diff --git a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala index e2502824b4..86b7e638e1 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala @@ -4,10 +4,11 @@ import com.twitter.conversions.DurationOps._ import com.twitter.util._ import java.util.concurrent.{Executors, TimeUnit} import java.util.logging.{Handler, Level, LogRecord} -import org.scalatest.{FunSuite, Matchers} import org.scalatest.concurrent.{Eventually, IntegrationPatience} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers -abstract class LocalSchedulerTest(lifo: Boolean) extends FunSuite with Matchers { +abstract class LocalSchedulerTest(lifo: Boolean) extends AnyFunSuite with Matchers { private val scheduler = new LocalScheduler(lifo) def submit(f: => Unit): Unit = scheduler.submit(new Runnable { @@ -156,7 +157,7 @@ class LocalSchedulerFifoTest extends LocalSchedulerTest(false) class LocalSchedulerLifoTest extends LocalSchedulerTest(true) -class ThreadPoolSchedulerTest extends FunSuite with Eventually with IntegrationPatience { +class ThreadPoolSchedulerTest extends AnyFunSuite with Eventually with IntegrationPatience { test("works") { val p = new Promise[Unit] val scheduler = new ThreadPoolScheduler("test") diff --git a/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala index 7411cd1336..578129fb82 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala @@ -1,9 +1,9 @@ package com.twitter.concurrent import java.util.concurrent.CountDownLatch -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class SerializedTest extends WordSpec with Serialized { +class SerializedTest extends AnyWordSpec with Serialized { "Serialized" should { "runs blocks, one at a time, in the order received" in { val t1CallsSerializedFirst = new CountDownLatch(1) diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala index 7b2c590a41..792835f201 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala @@ -2,9 +2,9 @@ package com.twitter.concurrent import com.twitter.util.{Return, Await} import java.util.concurrent.atomic.AtomicInteger -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class SpoolSourceTest extends WordSpec { +class SpoolSourceTest extends AnyWordSpec { "SpoolSource" should { class SpoolSourceHelper { val source = new SpoolSource[Int] diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala index ae4824f8fb..443a224a51 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala @@ -5,11 +5,11 @@ import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, Promise, Return, Throw} import java.io.EOFException import org.scalacheck.Arbitrary -import org.scalatest.WordSpec import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable.ArrayBuffer +import org.scalatest.wordspec.AnyWordSpec -class SpoolTest extends WordSpec with ScalaCheckDrivenPropertyChecks { +class SpoolTest extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { def await[A](f: Future[A]): A = Await.result(f, 2.seconds) diff --git a/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala b/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala index 6abc0bcd30..ab75e55db5 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala @@ -1,9 +1,9 @@ package com.twitter.concurrent -import org.scalatest.WordSpec import com.twitter.util.Return +import org.scalatest.wordspec.AnyWordSpec -class TxTest extends WordSpec { +class TxTest extends AnyWordSpec { "Tx.twoParty" should { "commit when everything goes dandy" in { val (stx, rtx) = Tx.twoParty(123) diff --git a/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala index 4a196e4431..bc2143bc76 100644 --- a/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala @@ -1,10 +1,10 @@ package com.twitter.conversions import com.twitter.util.Duration -import org.scalatest.FunSuite import com.twitter.conversions.DurationOps._ +import org.scalatest.funsuite.AnyFunSuite -class DurationOpsTest extends FunSuite { +class DurationOpsTest extends AnyFunSuite { test("converts Duration.Zero") { assert(0.seconds eq Duration.Zero) assert(0.milliseconds eq Duration.Zero) diff --git a/util-core/src/test/scala/com/twitter/conversions/OpsTogetherTest.scala b/util-core/src/test/scala/com/twitter/conversions/OpsTogetherTest.scala index b0e24303c9..8d689d9c08 100644 --- a/util-core/src/test/scala/com/twitter/conversions/OpsTogetherTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/OpsTogetherTest.scala @@ -1,9 +1,9 @@ package com.twitter.conversions import com.twitter.util.{Duration, StorageUnit} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class OpsTogetherTest extends FunSuite { +class OpsTogetherTest extends AnyFunSuite { test("multiple wildcard imports") { import com.twitter.conversions.DurationOps._ import com.twitter.conversions.PercentOps._ diff --git a/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala index 10073e2d7a..fbce685289 100644 --- a/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/PercentOpsTest.scala @@ -3,10 +3,10 @@ package com.twitter.conversions import com.twitter.conversions.PercentOps._ import org.scalacheck.Arbitrary.arbitrary import org.scalacheck.Gen -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class PercentOpsTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class PercentOpsTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { private[this] val Precision = 0.0000000001 diff --git a/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala index 2e46ba726b..c46bfc7093 100644 --- a/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala @@ -1,10 +1,10 @@ package com.twitter.conversions import com.twitter.util.StorageUnit -import org.scalatest.FunSuite import com.twitter.conversions.StorageUnitOps._ +import org.scalatest.funsuite.AnyFunSuite -class StorageUnitOpsTest extends FunSuite { +class StorageUnitOpsTest extends AnyFunSuite { test("converts") { assert(StorageUnit.fromBytes(1) == 1.byte) diff --git a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala index 25afa098c0..a25f4cf076 100644 --- a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala @@ -1,9 +1,9 @@ package com.twitter.conversions -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class U64OpsTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class U64OpsTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { import com.twitter.conversions.U64Ops._ test("toU64HextString") { diff --git a/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala index 171dffad16..32199c9fa5 100644 --- a/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ActivitySourceTest.scala @@ -1,9 +1,9 @@ package com.twitter.io import com.twitter.util.Activity -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ActivitySourceTest extends FunSuite { +class ActivitySourceTest extends AnyFunSuite { val ok: ActivitySource[String] = new ActivitySource[String] { def get(varName: String) = Activity.value(varName) diff --git a/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala b/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala index 544253c112..c6bc755597 100644 --- a/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala @@ -2,10 +2,10 @@ package com.twitter.io import java.lang.{Double => JDouble, Float => JFloat} import java.nio.charset.StandardCharsets -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -final class BufByteWriterTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { import ByteWriter.OverflowException private[this] def assertIndex(bw: ByteWriter, index: Int): Unit = bw match { diff --git a/util-core/src/test/scala/com/twitter/io/BufInputStreamTest.scala b/util-core/src/test/scala/com/twitter/io/BufInputStreamTest.scala index 47795ec9ad..ffc103071f 100644 --- a/util-core/src/test/scala/com/twitter/io/BufInputStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufInputStreamTest.scala @@ -1,8 +1,8 @@ package com.twitter.io -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class BufInputStreamTest extends FunSuite { +class BufInputStreamTest extends AnyFunSuite { private[this] val fileString = "Test_All_Tests\nTest_java_io_BufferedInputStream\nTest_java_io_BufferedOutputStream\nTest_ByteArrayInputStream\nTest_java_io_ByteArrayOutputStream\nTest_java_io_DataInputStream\n" private[this] val fileBuf = Buf.ByteArray.Owned(fileString.getBytes) diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index be11594823..c7bb9a4393 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -4,11 +4,11 @@ import com.twitter.conversions.DurationOps._ import com.twitter.conversions.StorageUnitOps._ import com.twitter.util.{Await, Future} import org.scalacheck.{Arbitrary, Gen} -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.annotation.tailrec +import org.scalatest.funsuite.AnyFunSuite -class BufReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class BufReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index dfa05de28f..41c42582ab 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -5,16 +5,16 @@ import java.nio.{ByteBuffer, CharBuffer} import java.nio.charset.{StandardCharsets => JChar} import java.util.Arrays import org.scalacheck.{Arbitrary, Gen, Prop} -import org.scalatest.FunSuite import org.scalatestplus.junit.AssertionsForJUnit import org.scalatestplus.mockito.MockitoSugar import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.scalatestplus.scalacheck.Checkers import scala.collection.mutable import java.nio.charset.Charset +import org.scalatest.funsuite.AnyFunSuite class BufTest - extends FunSuite + extends AnyFunSuite with MockitoSugar with ScalaCheckDrivenPropertyChecks with Checkers diff --git a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala index 6785a60473..1239a160df 100644 --- a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala @@ -3,10 +3,10 @@ package com.twitter.io import java.lang.{Double => JDouble, Float => JFloat} import java.nio.charset.StandardCharsets import org.scalacheck.Gen -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class ByteReaderTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { import ByteReader._ def readerWith(bytes: Byte*): ByteReader = ByteReader(Buf.ByteArray.Owned(bytes.toArray)) diff --git a/util-core/src/test/scala/com/twitter/io/CachingActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/CachingActivitySourceTest.scala index 816f068af1..226070cd63 100644 --- a/util-core/src/test/scala/com/twitter/io/CachingActivitySourceTest.scala +++ b/util-core/src/test/scala/com/twitter/io/CachingActivitySourceTest.scala @@ -1,10 +1,10 @@ package com.twitter.io import com.twitter.util.Activity -import org.scalatest.FunSuite import scala.util.Random +import org.scalatest.funsuite.AnyFunSuite -class CachingActivitySourceTest extends FunSuite { +class CachingActivitySourceTest extends AnyFunSuite { test("CachingActivitySource") { val cache = new CachingActivitySource[String](new ActivitySource[String] { diff --git a/util-core/src/test/scala/com/twitter/io/ClassLoaderActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/ClassLoaderActivitySourceTest.scala index 1853714f4c..e59c1d1687 100644 --- a/util-core/src/test/scala/com/twitter/io/ClassLoaderActivitySourceTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ClassLoaderActivitySourceTest.scala @@ -2,9 +2,9 @@ package com.twitter.io import com.twitter.util.FuturePool import java.io.ByteArrayInputStream -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ClassLoaderActivitySourceTest extends FunSuite { +class ClassLoaderActivitySourceTest extends AnyFunSuite { test("ClassLoaderActivitySource") { val classLoader = new ClassLoader() { diff --git a/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala b/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala index 85a0e6f7bd..7c3279625e 100644 --- a/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala +++ b/util-core/src/test/scala/com/twitter/io/FilePollingActivitySourceTest.scala @@ -2,10 +2,11 @@ package com.twitter.io import com.twitter.util.{Activity, FuturePool, MockTimer, Time} import java.io.File -import org.scalatest.{BeforeAndAfter, FunSuite} +import org.scalatest.BeforeAndAfter import scala.util.Random +import org.scalatest.funsuite.AnyFunSuite -class FilePollingActivitySourceTest extends FunSuite with BeforeAndAfter { +class FilePollingActivitySourceTest extends AnyFunSuite with BeforeAndAfter { def writeToTempFile(s: String): Unit = { val printer = new java.io.PrintWriter(new File(tempFile)) diff --git a/util-core/src/test/scala/com/twitter/io/FilesTest.scala b/util-core/src/test/scala/com/twitter/io/FilesTest.scala index 03922201c7..e2023cba7b 100644 --- a/util-core/src/test/scala/com/twitter/io/FilesTest.scala +++ b/util-core/src/test/scala/com/twitter/io/FilesTest.scala @@ -1,9 +1,9 @@ package com.twitter.io import java.io.File -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class FilesTest extends WordSpec with TempFolder { +class FilesTest extends AnyWordSpec with TempFolder { "Files" should { "delete" in withTempFolder { diff --git a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala index 40bd4e1102..b085ae166b 100644 --- a/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/InputStreamReaderTest.scala @@ -4,9 +4,9 @@ import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, FuturePool} import java.io.ByteArrayInputStream import org.mockito.Mockito._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class InputStreamReaderTest extends FunSuite { +class InputStreamReaderTest extends AnyFunSuite { def arr(i: Int, j: Int): Array[Byte] = Array.range(i, j).map(_.toByte) def buf(i: Int, j: Int): Buf = Buf.ByteArray.Owned(arr(i, j)) diff --git a/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala b/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala index 7e898389e0..c40715fbc4 100644 --- a/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala @@ -2,10 +2,10 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.Checkers +import org.scalatest.funsuite.AnyFunSuite -class IteratorReaderTest extends FunSuite with Checkers { +class IteratorReaderTest extends AnyFunSuite with Checkers { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index 8d1cb8afdc..6b4b6cbb2a 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -3,10 +3,11 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, MockTimer, Return, Time} import java.io.ByteArrayOutputStream -import org.scalatest.{FunSuite, Matchers} import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers -class PipeTest extends FunSuite with Matchers with ScalaCheckDrivenPropertyChecks { +class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyChecks { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) diff --git a/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala index 80f72e2127..211bc9d02a 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderSemanticsTest.scala @@ -5,10 +5,10 @@ import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ import com.twitter.util.{Future, Return, Await, Throw} import java.io.ByteArrayInputStream -import org.scalatest.FunSuite import scala.util.Random +import org.scalatest.funsuite.AnyFunSuite -abstract class ReaderSemanticsTest[A] extends FunSuite { +abstract class ReaderSemanticsTest[A] extends AnyFunSuite { def createReader(): Reader[A] def loopAndCheck(reader: Reader[A], onClose: Future[StreamTermination]): Future[Unit] = { diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 85e3944bcb..1773df41cd 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -10,12 +10,13 @@ import java.util.concurrent.atomic.AtomicBoolean import org.mockito.Mockito._ import org.scalacheck.{Arbitrary, Gen} import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import org.scalatest.{FunSuite, Matchers} import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable.ArrayBuffer +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers class ReaderTest - extends FunSuite + extends AnyFunSuite with ScalaCheckDrivenPropertyChecks with Matchers with Eventually diff --git a/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala b/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala index cbffb0c3a2..87f2822b1e 100644 --- a/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala +++ b/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala @@ -2,9 +2,9 @@ package com.twitter.io import scala.util.Random -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class StreamIOTest extends WordSpec { +class StreamIOTest extends AnyWordSpec { "StreamIO.copy" should { "copy the entire stream" in { val buf = new Array[Byte](2048) diff --git a/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala b/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala index 49366cdb68..0c13d2af34 100644 --- a/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala +++ b/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala @@ -1,8 +1,8 @@ package com.twitter.io -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class TempDirectoryTest extends WordSpec { +class TempDirectoryTest extends AnyWordSpec { "TempDirectory" should { diff --git a/util-core/src/test/scala/com/twitter/io/TempFileTest.scala b/util-core/src/test/scala/com/twitter/io/TempFileTest.scala index bf792caf79..e365ff3f10 100644 --- a/util-core/src/test/scala/com/twitter/io/TempFileTest.scala +++ b/util-core/src/test/scala/com/twitter/io/TempFileTest.scala @@ -2,9 +2,9 @@ package com.twitter.io import java.io.{ByteArrayInputStream, DataInputStream} import java.util.Arrays -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class TempFileTest extends WordSpec { +class TempFileTest extends AnyWordSpec { "TempFile" should { diff --git a/util-core/src/test/scala/com/twitter/io/WriterTest.scala b/util-core/src/test/scala/com/twitter/io/WriterTest.scala index 5ac95ae375..71d216be0f 100644 --- a/util-core/src/test/scala/com/twitter/io/WriterTest.scala +++ b/util-core/src/test/scala/com/twitter/io/WriterTest.scala @@ -3,9 +3,10 @@ package com.twitter.io import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, Promise} import java.io.{ByteArrayOutputStream, OutputStream} -import org.scalatest.{FunSuite, Matchers} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers -class WriterTest extends FunSuite with Matchers { +class WriterTest extends AnyFunSuite with Matchers { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) diff --git a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala index 7c53e9f9b8..57808afc26 100644 --- a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala @@ -1,9 +1,9 @@ package com.twitter.util import java.util.concurrent.atomic.AtomicReference -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ActivityTest extends FunSuite { +class ActivityTest extends AnyFunSuite { test("Activity#flatMap") { val v = Var(Activity.Pending: Activity.State[Int]) val ref = new AtomicReference[Seq[Activity.State[Int]]] diff --git a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala index 785e73c01b..e90007845a 100644 --- a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala +++ b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala @@ -4,8 +4,8 @@ import com.twitter.util.Base64Long.{fromBase64, StandardBase64Alphabet, toBase64 import org.scalacheck.Arbitrary import org.scalacheck.Arbitrary.arbitrary import org.scalacheck.Gen -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite object Base64LongTest { private val BoundaryValues = @@ -52,7 +52,7 @@ object Base64LongTest { } } -class Base64LongTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class Base64LongTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { import Base64LongTest._ private[this] def forAllLongs(f: (Alphabet, Long) => Unit): Unit = { diff --git a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala index 299fb9d7a0..14d150218d 100644 --- a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala @@ -1,8 +1,8 @@ package com.twitter.util -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class CancellableTest extends FunSuite { +class CancellableTest extends AnyFunSuite { test("Cancellable.nil is always cancelled") { assert(!Cancellable.nil.isCancelled) diff --git a/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala index 2a9ad140dc..2d234bf8bf 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableOnceTest.scala @@ -1,9 +1,9 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ClosableOnceTest extends FunSuite { +class ClosableOnceTest extends AnyFunSuite { private def ready[T <: Awaitable[_]](awaitable: T): T = Await.ready(awaitable, 10.seconds) diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index 5c5372958e..d2f07ba1a5 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -1,11 +1,11 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} import scala.language.reflectiveCalls +import org.scalatest.funsuite.AnyFunSuite -class ClosableTest extends FunSuite with Eventually with IntegrationPatience { +class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience { // Workaround methods for dealing with Scala compiler warnings: // diff --git a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala index a2d3bb6ea2..0aea96b7e6 100644 --- a/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CloseAwaitablyTest.scala @@ -1,9 +1,9 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class CloseAwaitablyTest extends FunSuite { +class CloseAwaitablyTest extends AnyFunSuite { class Context extends Closable with CloseAwaitably { val p: Promise[Unit] = new Promise[Unit] var n: Int = 0 diff --git a/util-core/src/test/scala/com/twitter/util/ConfigTest.scala b/util-core/src/test/scala/com/twitter/util/ConfigTest.scala index 6bb9f670d4..60f4f1040c 100644 --- a/util-core/src/test/scala/com/twitter/util/ConfigTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ConfigTest.scala @@ -1,8 +1,9 @@ package com.twitter.util -import org.scalatest.{Matchers, WordSpec} +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec -class ConfigTest extends WordSpec with Matchers { +class ConfigTest extends AnyWordSpec with Matchers { import Config._ "Config" should { diff --git a/util-core/src/test/scala/com/twitter/util/DiffTest.scala b/util-core/src/test/scala/com/twitter/util/DiffTest.scala index 2b12ac0697..197c078436 100644 --- a/util-core/src/test/scala/com/twitter/util/DiffTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DiffTest.scala @@ -1,10 +1,10 @@ package com.twitter.util -import org.scalatest.FunSuite import org.scalacheck.Arbitrary.arbitrary import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class DiffTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class DiffTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { val f: Int => String = _.toString test("Diffable.ofSet") { diff --git a/util-core/src/test/scala/com/twitter/util/EventTest.scala b/util-core/src/test/scala/com/twitter/util/EventTest.scala index 07830a83eb..da9bc69077 100644 --- a/util-core/src/test/scala/com/twitter/util/EventTest.scala +++ b/util-core/src/test/scala/com/twitter/util/EventTest.scala @@ -2,10 +2,10 @@ package com.twitter.util import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import java.util.concurrent.{CountDownLatch, Executors} -import org.scalatest.FunSuite import scala.collection.mutable +import org.scalatest.funsuite.AnyFunSuite -class EventTest extends FunSuite { +class EventTest extends AnyFunSuite { test("pub/sub while active") { val e = Event[Int]() diff --git a/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala b/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala index d02eda29bd..1aeef90299 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala @@ -1,10 +1,10 @@ package com.twitter.concurrent -import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar import com.twitter.util.{Promise, Return} +import org.scalatest.wordspec.AnyWordSpec -class FutureOfferTest extends WordSpec with MockitoSugar { +class FutureOfferTest extends AnyWordSpec with MockitoSugar { "Future.toOffer" should { "activate when future is satisfied (poll)" in { val p = new Promise[Int] diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index 44191ee195..a7ccc7cc7d 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -2,13 +2,13 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import java.util.concurrent.{Future => JFuture, _} -import org.scalatest.FunSuite import org.scalatest.concurrent.Eventually import org.scalatest.time.{Millis, Seconds, Span} import scala.runtime.NonLocalReturnControl import scala.util.control.NonFatal +import org.scalatest.funsuite.AnyFunSuite -class FuturePoolTest extends FunSuite with Eventually { +class FuturePoolTest extends AnyFunSuite with Eventually { implicit override val patienceConfig: PatienceConfig = PatienceConfig(timeout = scaled(Span(15, Seconds)), interval = scaled(Span(5, Millis))) diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index c0b29829b9..954ee649d1 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -8,7 +8,6 @@ import org.mockito.Mockito.{never, times, verify, when} import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer import org.scalacheck.{Arbitrary, Gen} -import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.jdk.CollectionConverters._ @@ -16,6 +15,7 @@ import scala.runtime.NonLocalReturnControl import scala.util.Random import scala.util.control.ControlThrowable import scala.util.control.NonFatal +import org.scalatest.wordspec.AnyWordSpec private object FutureTest { def await[A](f: Future[A], timeout: Duration = 5.seconds): A = @@ -38,7 +38,7 @@ private object FutureTest { } } -class FutureTest extends WordSpec with MockitoSugar with ScalaCheckDrivenPropertyChecks { +class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenPropertyChecks { import FutureTest._ implicit class FutureMatcher[A](future: Future[A]) { diff --git a/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala b/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala index b00894dcb9..73fab075e8 100644 --- a/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala @@ -1,8 +1,8 @@ package com.twitter.util -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class LastWriteWinsQueueTest extends WordSpec { +class LastWriteWinsQueueTest extends AnyWordSpec { "LastWriteWinsQueue" should { val queue = new LastWriteWinsQueue[String] diff --git a/util-core/src/test/scala/com/twitter/util/LocalTest.scala b/util-core/src/test/scala/com/twitter/util/LocalTest.scala index 803bd0e3b3..91cd396a7d 100644 --- a/util-core/src/test/scala/com/twitter/util/LocalTest.scala +++ b/util-core/src/test/scala/com/twitter/util/LocalTest.scala @@ -2,10 +2,10 @@ package com.twitter.util import org.scalacheck.Arbitrary import org.scalacheck.Arbitrary.arbitrary -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class LocalTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class LocalTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { test("Local: should be undefined by default") { val local = new Local[Int] assert(local() == None) diff --git a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala index 4722544fbc..2294a8fbab 100644 --- a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala @@ -4,9 +4,9 @@ import com.twitter.conversions.DurationOps._ import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.{TimeUnit, CountDownLatch => JavaCountDownLatch} import org.mockito.Mockito._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class MemoizeTest extends FunSuite { +class MemoizeTest extends AnyFunSuite { test("Memoize.apply: only runs the function once for the same input") { // mockito can't spy anonymous classes, // and this was the simplest approach i could come up with. diff --git a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala index e6d6b0913a..9f17351274 100644 --- a/util-core/src/test/scala/com/twitter/util/MonitorTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MonitorTest.scala @@ -2,8 +2,8 @@ package com.twitter.util import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ -import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite object MonitorTest { class MockMonitor extends Monitor { @@ -11,7 +11,7 @@ object MonitorTest { } } -class MonitorTest extends FunSuite with MockitoSugar { +class MonitorTest extends AnyFunSuite with MockitoSugar { import MonitorTest._ class MonitorOrElseHelper { diff --git a/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala b/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala index 3d5542e0b1..ccabff1ef4 100644 --- a/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala +++ b/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala @@ -2,9 +2,9 @@ package com.twitter.util import java.net.InetAddress -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class NetUtilTest extends WordSpec { +class NetUtilTest extends AnyWordSpec { "NetUtil" should { "isIpv4Address" in { for (i <- 0.to(255)) { diff --git a/util-core/src/test/scala/com/twitter/util/PoolTest.scala b/util-core/src/test/scala/com/twitter/util/PoolTest.scala index da58d91fc3..11d15401f6 100644 --- a/util-core/src/test/scala/com/twitter/util/PoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PoolTest.scala @@ -1,10 +1,10 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.scalatest.WordSpec import scala.collection.mutable +import org.scalatest.wordspec.AnyWordSpec -class PoolTest extends WordSpec { +class PoolTest extends AnyWordSpec { private[this] def await[T](f: Awaitable[T]): T = Await.result(f, 2.seconds) diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index d0dc983bb5..e310bbcc25 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -1,9 +1,9 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class PromiseTest extends FunSuite { +class PromiseTest extends AnyFunSuite { test("Promise.detach should not detach other attached promises") { val p = new Promise[Unit] diff --git a/util-core/src/test/scala/com/twitter/util/SlowProbeProxyTimerTest.scala b/util-core/src/test/scala/com/twitter/util/SlowProbeProxyTimerTest.scala index 71808d0be9..770c3f5bc8 100644 --- a/util-core/src/test/scala/com/twitter/util/SlowProbeProxyTimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/SlowProbeProxyTimerTest.scala @@ -1,10 +1,10 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.scalatest.FunSuite import scala.collection.mutable +import org.scalatest.funsuite.AnyFunSuite -class SlowProbeProxyTimerTest extends FunSuite { +class SlowProbeProxyTimerTest extends AnyFunSuite { private type Task = () => Unit private val NullTask = new TimerTask { def cancel(): Unit = () } diff --git a/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala b/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala index c472fee792..dcb037c786 100644 --- a/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala @@ -1,8 +1,8 @@ package com.twitter.util -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class StateMachineTest extends WordSpec { +class StateMachineTest extends AnyWordSpec { "StateMachine" should { class StateMachineHelper { class Machine extends StateMachine { diff --git a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala index 8b77193035..3a42c01e27 100644 --- a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala @@ -1,10 +1,9 @@ package com.twitter.util -import org.scalatest.FunSuite - import com.twitter.conversions.StorageUnitOps._ +import org.scalatest.funsuite.AnyFunSuite -class StorageUnitTest extends FunSuite { +class StorageUnitTest extends AnyFunSuite { test("StorageUnit: should convert whole numbers into storage units (back and forth)") { assert(1.byte.inBytes == 1) assert(1.kilobyte.inBytes == 1024) diff --git a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala b/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala index 09f90c6776..eaaa43e682 100644 --- a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala @@ -17,9 +17,9 @@ package com.twitter.util import com.twitter.conversions.StringOps._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class StringConversionsTest extends FunSuite { +class StringConversionsTest extends AnyFunSuite { test("string#quoteC") { assert("nothing".quoteC == "nothing") diff --git a/util-core/src/test/scala/com/twitter/util/ThrowablesTest.scala b/util-core/src/test/scala/com/twitter/util/ThrowablesTest.scala index a9d1d55368..0661094553 100644 --- a/util-core/src/test/scala/com/twitter/util/ThrowablesTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ThrowablesTest.scala @@ -1,8 +1,8 @@ package com.twitter.util -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ThrowablesTest extends FunSuite { +class ThrowablesTest extends AnyFunSuite { test("Throwables.mkString: flatten non-nested exception") { val t = new Throwable assert(Throwables.mkString(t) == Seq(t.getClass.getName)) diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index fba5f60652..62a3d3a237 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -4,13 +4,13 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, import java.util.{Locale, TimeZone} import java.util.concurrent.TimeUnit -import org.scalatest.WordSpec import org.scalatest.concurrent.{Eventually, IntegrationPatience} import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import com.twitter.conversions.DurationOps._ +import org.scalatest.wordspec.AnyWordSpec -trait TimeLikeSpec[T <: TimeLike[T]] extends WordSpec with ScalaCheckDrivenPropertyChecks { +trait TimeLikeSpec[T <: TimeLike[T]] extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { val ops: TimeLikeOps[T] import ops._ @@ -371,7 +371,7 @@ trait TimeLikeSpec[T <: TimeLike[T]] extends WordSpec with ScalaCheckDrivenPrope } } -class TimeFormatTest extends WordSpec { +class TimeFormatTest extends AnyWordSpec { "TimeFormat" should { "format correctly with non US locale" in { val locale = Locale.GERMAN diff --git a/util-core/src/test/scala/com/twitter/util/TimerTest.scala b/util-core/src/test/scala/com/twitter/util/TimerTest.scala index 7776d57fe5..aeb97c0026 100644 --- a/util-core/src/test/scala/com/twitter/util/TimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimerTest.scala @@ -6,11 +6,11 @@ import java.util.concurrent.atomic.AtomicInteger import org.mockito.ArgumentCaptor import org.mockito.ArgumentMatchers.any import org.mockito.Mockito.{never, verify, when} -import org.scalatest.FunSuite import org.scalatest.concurrent.{IntegrationPatience, Eventually} import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite -class TimerTest extends FunSuite with MockitoSugar with Eventually with IntegrationPatience { +class TimerTest extends AnyFunSuite with MockitoSugar with Eventually with IntegrationPatience { private def testTimerRunsWithLocals(timer: Timer): Unit = { val timerLocal = new AtomicInteger(0) diff --git a/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala b/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala index a20f09e11a..ecfaf91f54 100644 --- a/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TokenBucketTest.scala @@ -1,9 +1,9 @@ package com.twitter.util -import org.scalatest.FunSuite import com.twitter.conversions.DurationOps._ +import org.scalatest.funsuite.AnyFunSuite -class TokenBucketTest extends FunSuite { +class TokenBucketTest extends AnyFunSuite { test("a leaky bucket is leaky") { Time.withCurrentTimeFrozen { tc => val b = TokenBucket.newLeakyBucket(3.seconds, 0, Stopwatch.timeMillis) diff --git a/util-core/src/test/scala/com/twitter/util/TryTest.scala b/util-core/src/test/scala/com/twitter/util/TryTest.scala index f40a490109..fa0ed84882 100644 --- a/util-core/src/test/scala/com/twitter/util/TryTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TryTest.scala @@ -1,8 +1,8 @@ package com.twitter.util -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class TryTest extends FunSuite { +class TryTest extends AnyFunSuite { class MyException extends Exception val e: Exception = new Exception("this is an exception") diff --git a/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala b/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala index d86073335a..9e17fef3ff 100644 --- a/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala @@ -2,9 +2,9 @@ package com.twitter.util import java.util.Locale -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class TwitterDateFormatTest extends WordSpec { +class TwitterDateFormatTest extends AnyWordSpec { "TwitterDateFormat" should { "disallow Y without w" in { intercept[IllegalArgumentException] { diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 99f08bbfe0..06603bb8d7 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -2,11 +2,11 @@ package com.twitter.util import java.util.concurrent.atomic.AtomicReference import org.scalacheck.Arbitrary.arbitrary -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable +import org.scalatest.funsuite.AnyFunSuite -class VarTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { // Workaround methods for dealing with Scala compiler warnings: // diff --git a/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala b/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala index 1e1436384a..afb75749a1 100644 --- a/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala @@ -1,10 +1,10 @@ package com.twitter.util import org.scalacheck.Gen -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class WaitQueueTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class WaitQueueTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { private class EmptyK[A] extends Promise.K[A] { protected[util] def depth: Short = 0 diff --git a/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala b/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala index ffea53ecc2..c397ca5865 100644 --- a/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WindowedAdderTest.scala @@ -1,9 +1,9 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class WindowedAdderTest extends FunSuite { +class WindowedAdderTest extends AnyFunSuite { private def newAdder() = WindowedAdder(3 * 1000, 3, Stopwatch.timeMillis) test("sums things up when time stands still") { diff --git a/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala index 013a14b8c7..3d4868aa5a 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala @@ -1,14 +1,14 @@ package com.twitter.hashing import org.scalacheck.Gen -import org.scalatest.WordSpec import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable import java.io.{BufferedReader, InputStreamReader} import java.nio.{ByteBuffer, ByteOrder} import java.security.MessageDigest +import org.scalatest.wordspec.AnyWordSpec -class ConsistentHashingDistributorTest extends WordSpec with ScalaCheckDrivenPropertyChecks { +class ConsistentHashingDistributorTest extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { "KetamaDistributor" should { val nodes = Seq( HashNode("10.0.1.1", 600, 1), diff --git a/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala index 7f806b4913..2c679128a6 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala @@ -1,8 +1,8 @@ package com.twitter.hashing -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class DiagnosticsTest extends WordSpec { +class DiagnosticsTest extends AnyWordSpec { "Diagnostics" should { "print distribution" in { val hosts = 1 until 500 map { "10.1.1." + _ + ":11211:4" } diff --git a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala index de3ca360d7..75af619d4c 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala @@ -1,9 +1,9 @@ package com.twitter.hashing -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class HashableTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class HashableTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { private[this] val algorithms = Seq( Hashable.CRC32_ITU, diff --git a/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala index b628b39abc..f334302983 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala @@ -2,11 +2,11 @@ package com.twitter.hashing import com.twitter.io.TempFile import java.util.Base64 -import org.scalatest.WordSpec import scala.collection.mutable.ListBuffer import java.nio.charset.StandardCharsets.UTF_8 +import org.scalatest.wordspec.AnyWordSpec -class KeyHasherTest extends WordSpec { +class KeyHasherTest extends AnyWordSpec { def readResource(name: String) = { var lines = new ListBuffer[String]() val src = scala.io.Source.fromFile(TempFile.fromResourcePath(getClass, "/" + name)) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala index a4e283d02e..fe2305efd0 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/ContentionTest.scala @@ -2,9 +2,9 @@ package com.twitter.jvm import com.twitter.util.{Await, Promise} import java.util.concurrent.locks.ReentrantLock -import org.scalatest.FunSuite import org.scalatest.concurrent.Eventually import org.scalatest.time.{Millis, Seconds, Span} +import org.scalatest.funsuite.AnyFunSuite class Philosopher { val ready = new Promise[Unit] @@ -18,7 +18,7 @@ class Philosopher { } } -class ContentionTest extends FunSuite with Eventually { +class ContentionTest extends AnyFunSuite with Eventually { implicit override val patienceConfig = PatienceConfig(timeout = scaled(Span(15, Seconds)), interval = scaled(Span(5, Millis))) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala index 4e754ebe27..a87a1f150e 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala @@ -3,9 +3,9 @@ package com.twitter.jvm import com.twitter.conversions.DurationOps._ import com.twitter.util.Time import java.io.ByteArrayOutputStream -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class CpuProfileTest extends FunSuite { +class CpuProfileTest extends AnyFunSuite { test("record") { // record() calls Time.now 3 times initially, and then 3 times on every loop iteration. diff --git a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala index ff28e078a2..95c8b20e02 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorTest.scala @@ -1,8 +1,8 @@ package com.twitter.jvm -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class EstimatorTest extends FunSuite { +class EstimatorTest extends AnyFunSuite { test("LoadAverage") { // This makes LoadAverage.a = 1/2 for easy testing. val interval = -1d / math.log(0.5) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala index e050f19d45..c0eb0da0d8 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala @@ -5,11 +5,11 @@ import com.twitter.util.Time import java.util.logging.{Level, Logger} import org.mockito.ArgumentMatchers.contains import org.mockito.Mockito.{times, verify, when} -import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar import scala.collection.mutable +import org.scalatest.wordspec.AnyWordSpec -class JvmTest extends WordSpec with MockitoSugar { +class JvmTest extends AnyWordSpec with MockitoSugar { "Jvm" should { class JvmHelper(suppliedLogger: Option[Logger] = None) extends Jvm { protected override def logger: Logger = suppliedLogger match { diff --git a/util-jvm/src/test/scala/com/twitter/jvm/NumProcsTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/NumProcsTest.scala index cffb9d0c5c..74b9ea17a4 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/NumProcsTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/NumProcsTest.scala @@ -1,8 +1,8 @@ package com.twitter.jvm -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class NumProcsTest extends FunSuite { +class NumProcsTest extends AnyFunSuite { test("return the number of available processors according to the runtime by default") { assert(System.getProperty("com.twitter.jvm.numProcs") == null) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/OptTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/OptTest.scala index ba4933dde0..c9ba8578e6 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/OptTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/OptTest.scala @@ -3,9 +3,9 @@ package com.twitter.jvm import java.lang.{Boolean => JBool} import java.lang.management.ManagementFactory import javax.management.ObjectName -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class OptTest extends FunSuite { +class OptTest extends AnyFunSuite { test("Opts") { if (System.getProperty("java.vm.name").contains("HotSpot")) { val DiagnosticName = diff --git a/util-lint/src/test/scala/com/twitter/util/lint/GlobalRulesTest.scala b/util-lint/src/test/scala/com/twitter/util/lint/GlobalRulesTest.scala index 4f048422ae..4ff541ff0f 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/GlobalRulesTest.scala +++ b/util-lint/src/test/scala/com/twitter/util/lint/GlobalRulesTest.scala @@ -1,8 +1,8 @@ package com.twitter.util.lint -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class GlobalRulesTest extends FunSuite { +class GlobalRulesTest extends AnyFunSuite { private val neverRule = Rule.apply(Category.Performance, "R2", "Good") { Nil } diff --git a/util-lint/src/test/scala/com/twitter/util/lint/RuleTest.scala b/util-lint/src/test/scala/com/twitter/util/lint/RuleTest.scala index 3cf47ccf69..cbb9e9c43f 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/RuleTest.scala +++ b/util-lint/src/test/scala/com/twitter/util/lint/RuleTest.scala @@ -1,8 +1,8 @@ package com.twitter.util.lint -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class RuleTest extends FunSuite { +class RuleTest extends AnyFunSuite { private def withName(name: String): Rule = Rule(Category.Performance, name, "descriptive description") { Nil } diff --git a/util-lint/src/test/scala/com/twitter/util/lint/RulesTest.scala b/util-lint/src/test/scala/com/twitter/util/lint/RulesTest.scala index 802b8522e0..2d1669dcf6 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/RulesTest.scala +++ b/util-lint/src/test/scala/com/twitter/util/lint/RulesTest.scala @@ -1,8 +1,8 @@ package com.twitter.util.lint -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class RulesTest extends FunSuite { +class RulesTest extends AnyFunSuite { private var flag = false diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index 68ae4c1083..74bbe2af00 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -18,7 +18,6 @@ package com.twitter.logging import java.io._ import java.util.{Calendar, Date, UUID, logging => javalog} -import org.scalatest.WordSpec import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder @@ -26,8 +25,9 @@ import com.twitter.util.Time import java.nio.file.attribute.BasicFileAttributes import java.nio.file.{Files, Path, Paths} import java.util.function.BiPredicate +import org.scalatest.wordspec.AnyWordSpec -class FileHandlerTest extends WordSpec with TempFolder { +class FileHandlerTest extends AnyWordSpec with TempFolder { def reader(filename: String): BufferedReader = { new BufferedReader(new InputStreamReader(new FileInputStream(new File(folderName, filename)))) } diff --git a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala index ace293b2b0..53f5ada361 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala @@ -18,11 +18,10 @@ package com.twitter.logging import java.util.{logging => javalog} -import org.scalatest.WordSpec - import com.twitter.conversions.StringOps._ +import org.scalatest.wordspec.AnyWordSpec -class FormatterTest extends WordSpec { +class FormatterTest extends AnyWordSpec { val basicFormatter = new Formatter val utcFormatter = new Formatter( diff --git a/util-logging/src/test/scala/com/twitter/logging/HasLogLevelTest.scala b/util-logging/src/test/scala/com/twitter/logging/HasLogLevelTest.scala index 6191cd7464..f7eecd2534 100644 --- a/util-logging/src/test/scala/com/twitter/logging/HasLogLevelTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/HasLogLevelTest.scala @@ -1,8 +1,8 @@ package com.twitter.logging -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class HasLogLevelTest extends FunSuite { +class HasLogLevelTest extends AnyFunSuite { private class WithLogLevel(val logLevel: Level, cause: Throwable = null) extends Exception(cause) diff --git a/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala b/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala index 5b6d6b1e80..3d16a8e372 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala @@ -1,9 +1,9 @@ package com.twitter.logging import java.util.logging.{LogRecord => JRecord} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class LogRecordTest extends FunSuite { +class LogRecordTest extends AnyFunSuite { test("LogRecord should getMethod properly") { Logger.withLoggers(Nil) { new LogRecordTestHelper({ r: JRecord => r.getSourceMethodName() }) { diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index 159d07f838..975c3f9db4 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -21,10 +21,11 @@ import com.twitter.io.TempFolder import java.net.InetSocketAddress import java.util.concurrent.{Callable, CountDownLatch, Executors, Future, TimeUnit} import java.util.{logging => javalog} -import org.scalatest.{BeforeAndAfter, WordSpec} +import org.scalatest.BeforeAndAfter import scala.collection.mutable +import org.scalatest.wordspec.AnyWordSpec -class LoggerTest extends WordSpec with TempFolder with BeforeAndAfter { +class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { val logLevel = Logger.levelNames(Option[String](System.getenv("log")).getOrElse("FATAL").toUpperCase) diff --git a/util-logging/src/test/scala/com/twitter/logging/PolicyTest.scala b/util-logging/src/test/scala/com/twitter/logging/PolicyTest.scala index 9663ff61af..ec64873ed3 100644 --- a/util-logging/src/test/scala/com/twitter/logging/PolicyTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/PolicyTest.scala @@ -1,10 +1,9 @@ package com.twitter.logging -import org.scalatest.FunSuite - import com.twitter.util.StorageUnit +import org.scalatest.funsuite.AnyFunSuite -class PolicyTest extends FunSuite { +class PolicyTest extends AnyFunSuite { import Policy._ test("Policy.parse: never") { diff --git a/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala index 52c36680de..7763c147f0 100644 --- a/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala @@ -18,10 +18,10 @@ package com.twitter.logging import com.twitter.util.Local import java.util.{logging => javalog} -import org.scalatest.WordSpec import org.scalatest.concurrent.{IntegrationPatience, Eventually} +import org.scalatest.wordspec.AnyWordSpec -class QueueingHandlerTest extends WordSpec with Eventually with IntegrationPatience { +class QueueingHandlerTest extends AnyWordSpec with Eventually with IntegrationPatience { class MockHandler extends Handler(BareFormatter, None) { def publish(record: javalog.LogRecord) = () diff --git a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala index 7be3894240..b4c84882b1 100644 --- a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala @@ -19,9 +19,9 @@ package com.twitter.logging import java.net.{DatagramPacket, DatagramSocket} import java.util.{logging => javalog} -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class SyslogHandlerTest extends WordSpec { +class SyslogHandlerTest extends AnyWordSpec { val record1 = new javalog.LogRecord(Level.FATAL, "fatal message!") record1.setLoggerName("net.lag.whiskey.Train") record1.setMillis(1206769996722L) diff --git a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala index 18d09eea38..166b878e40 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala @@ -19,9 +19,10 @@ package com.twitter.logging import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import com.twitter.util.Time -import org.scalatest.{BeforeAndAfter, WordSpec} +import org.scalatest.BeforeAndAfter +import org.scalatest.wordspec.AnyWordSpec -class ThrottledHandlerTest extends WordSpec with BeforeAndAfter with TempFolder { +class ThrottledHandlerTest extends AnyWordSpec with BeforeAndAfter with TempFolder { private var handler: StringHandler = _ "ThrottledHandler" should { diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala index 9dff6d0099..cb5ce488c5 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala @@ -1,7 +1,7 @@ package com.twitter.util.reflect import com.twitter.util.{Future, Promise, Stopwatch} -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec object ProxySpec { trait TestInterface { @@ -32,7 +32,7 @@ object ProxySpec { } } -class ProxyTest extends WordSpec { +class ProxyTest extends AnyWordSpec { import ProxySpec._ diff --git a/util-registry/src/test/scala/com/twitter/util/registry/FormatterTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/FormatterTest.scala index 9fdb617d34..05e044d18b 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/FormatterTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/FormatterTest.scala @@ -1,9 +1,9 @@ package com.twitter.util.registry -import org.scalatest.FunSuite import scala.util.control.NoStackTrace +import org.scalatest.funsuite.AnyFunSuite -class FormatterTest extends FunSuite { +class FormatterTest extends AnyFunSuite { test("asMap generates reasonable Maps") { val registry = new SimpleRegistry registry.put(Seq("foo", "bar"), "baz") diff --git a/util-registry/src/test/scala/com/twitter/util/registry/LibraryTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/LibraryTest.scala index a5f58a6afd..c6d66b4366 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/LibraryTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/LibraryTest.scala @@ -1,8 +1,8 @@ package com.twitter.util.registry -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class LibraryTest extends FunSuite { +class LibraryTest extends AnyFunSuite { test("Library.register registers libraries") { val simple = new SimpleRegistry GlobalRegistry.withRegistry(simple) { diff --git a/util-registry/src/test/scala/com/twitter/util/registry/RegistryTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/RegistryTest.scala index 1b8d15b388..bd97759b56 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/RegistryTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/RegistryTest.scala @@ -1,9 +1,9 @@ package com.twitter.util.registry import java.lang.{Character => JCharacter} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -abstract class RegistryTest extends FunSuite { +abstract class RegistryTest extends AnyFunSuite { def mkRegistry(): Registry def name: String diff --git a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala index be11ae916e..46a864bc3d 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala @@ -3,10 +3,10 @@ package com.twitter.util.registry import java.util.logging.Logger import org.mockito.Mockito.{never, verify} import org.mockito.ArgumentMatchers.anyObject -import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite -class RosterTest extends FunSuite with MockitoSugar { +class RosterTest extends AnyFunSuite with MockitoSugar { def withRoster(fn: (Roster, Logger) => Unit): Unit = { val simple = new SimpleRegistry simple.put(Seq("foo", "bar"), "fantastic") diff --git a/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala index f5bb2a48f5..96e2db4214 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala @@ -1,6 +1,6 @@ package com.twitter.util.routing -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite private object PolymorphicRouterCompilationTest { @@ -80,7 +80,7 @@ private object PolymorphicRouterCompilationTest { } // these tests ensure that we properly support polymorphic Router and RouterBuilder types -class PolymorphicRouterCompilationTest extends FunSuite { +class PolymorphicRouterCompilationTest extends AnyFunSuite { import PolymorphicRouterCompilationTest._ test("Supports routing with polymorphic destinations") { diff --git a/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala index 151705f01f..d975ef2ea3 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala @@ -1,9 +1,9 @@ package com.twitter.util.routing import com.twitter.util.routing.simple.{SimpleRoute, SimpleRouter} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class RouterBuilderTest extends FunSuite { +class RouterBuilderTest extends AnyFunSuite { private object TestRouter { def newBuilder(): RouterBuilder[String, SimpleRoute, SimpleRouter] = diff --git a/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala index 130b74be52..9d85f1ddb0 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala @@ -5,9 +5,9 @@ import com.twitter.util.routing.dynamic.{DynamicRouter, DynamicRoute, Method, Re import com.twitter.util.routing.simple.{SimpleRoute, SimpleRouter} import com.twitter.util.{Await, Awaitable, Closable, Future, Time} import java.util.concurrent.atomic.AtomicBoolean -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class RouterTest extends FunSuite { +class RouterTest extends AnyFunSuite { private def await[T](awaitable: Awaitable[T]): T = Await.result(awaitable, 2.seconds) diff --git a/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala b/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala index 0476866c23..32360795cd 100644 --- a/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala @@ -16,10 +16,10 @@ package com.twitter.util.security -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.Checkers +import org.scalatest.funsuite.AnyFunSuite -class CredentialsTest extends FunSuite with Checkers { +class CredentialsTest extends AnyFunSuite with Checkers { test("parse a simple auth file") { val content = "username: root\npassword: hellokitty\n" val result = Credentials(content) diff --git a/util-security/src/test/scala/com/twitter/util/security/NullPrincipalTest.scala b/util-security/src/test/scala/com/twitter/util/security/NullPrincipalTest.scala index 8ea6c82e72..ba208b8f06 100644 --- a/util-security/src/test/scala/com/twitter/util/security/NullPrincipalTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/NullPrincipalTest.scala @@ -1,9 +1,9 @@ package com.twitter.util.security import java.security.Principal -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class NullPrincipalTest extends FunSuite { +class NullPrincipalTest extends AnyFunSuite { test("NullPrincipal is a Principal") { val principal: Principal = NullPrincipal diff --git a/util-security/src/test/scala/com/twitter/util/security/NullSslSessionContextTest.scala b/util-security/src/test/scala/com/twitter/util/security/NullSslSessionContextTest.scala index c86d64d892..0a22dab61d 100644 --- a/util-security/src/test/scala/com/twitter/util/security/NullSslSessionContextTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/NullSslSessionContextTest.scala @@ -1,9 +1,9 @@ package com.twitter.util.security import javax.net.ssl.SSLSessionContext -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class NullSslSessionContextTest extends FunSuite { +class NullSslSessionContextTest extends AnyFunSuite { test("NullSslSessionContext is an SSLSessionContext") { val sslSessionContext: SSLSessionContext = NullSslSessionContext diff --git a/util-security/src/test/scala/com/twitter/util/security/NullSslSessionTest.scala b/util-security/src/test/scala/com/twitter/util/security/NullSslSessionTest.scala index 342deb740b..bc2d5027fb 100644 --- a/util-security/src/test/scala/com/twitter/util/security/NullSslSessionTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/NullSslSessionTest.scala @@ -1,9 +1,9 @@ package com.twitter.util.security import javax.net.ssl.SSLSession -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class NullSslSessionTest extends FunSuite { +class NullSslSessionTest extends AnyFunSuite { test("NullSslSession is an SSLSession") { val sslSession: SSLSession = NullSslSession diff --git a/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala index cd0367060c..8cf1cc794f 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala @@ -3,9 +3,9 @@ package com.twitter.util.security import com.twitter.io.TempFile import com.twitter.util.Try import java.io.File -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class PemFileTest extends FunSuite { +class PemFileTest extends AnyFunSuite { private[this] def readPemFileMessage(messageType: String)(tempFile: File): Try[String] = { val pemFile = new PemFile(tempFile) diff --git a/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala index fd60f05281..edc0241232 100644 --- a/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala @@ -4,9 +4,9 @@ import com.twitter.io.TempFile import com.twitter.util.Try import java.io.File import java.security.spec.PKCS8EncodedKeySpec -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class Pkcs8EncodedKeySpecFileTest extends FunSuite { +class Pkcs8EncodedKeySpecFileTest extends AnyFunSuite { private[this] val readKeySpecFromFile: File => Try[PKCS8EncodedKeySpec] = (tempFile) => { diff --git a/util-security/src/test/scala/com/twitter/util/security/Pkcs8KeyManagerFactoryTest.scala b/util-security/src/test/scala/com/twitter/util/security/Pkcs8KeyManagerFactoryTest.scala index f4d02818c0..5b5f3726fa 100644 --- a/util-security/src/test/scala/com/twitter/util/security/Pkcs8KeyManagerFactoryTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/Pkcs8KeyManagerFactoryTest.scala @@ -3,9 +3,9 @@ package com.twitter.util.security import com.twitter.io.TempFile import java.io.File import javax.net.ssl.{KeyManager, X509KeyManager} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class PKCS8KeyManagerFactoryTest extends FunSuite { +class PKCS8KeyManagerFactoryTest extends AnyFunSuite { private[this] def assertContainsCertsAndKey(keyManager: KeyManager, numCerts: Int = 1): Unit = { assert(keyManager.isInstanceOf[X509KeyManager]) diff --git a/util-security/src/test/scala/com/twitter/util/security/PrivateKeyFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/PrivateKeyFileTest.scala index f7e5e25f0c..48f7f1ac4b 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PrivateKeyFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PrivateKeyFileTest.scala @@ -2,9 +2,9 @@ package com.twitter.util.security import com.twitter.io.TempFile import java.security.spec.InvalidKeySpecException -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class PrivateKeyFileTest extends FunSuite { +class PrivateKeyFileTest extends AnyFunSuite { test("File is garbage") { // Lines were manually deleted from a real pkcs 8 pem file diff --git a/util-security/src/test/scala/com/twitter/util/security/X500PrincipalInfoTest.scala b/util-security/src/test/scala/com/twitter/util/security/X500PrincipalInfoTest.scala index d28100ab05..530aa6bf23 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X500PrincipalInfoTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X500PrincipalInfoTest.scala @@ -1,9 +1,9 @@ package com.twitter.util.security import javax.security.auth.x500.X500Principal -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class X500PrincipalInfoTest extends FunSuite { +class X500PrincipalInfoTest extends AnyFunSuite { test("Empty principal") { val principal = new X500Principal("") diff --git a/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala index 1aa3845a1d..8f1dea9fcb 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala @@ -4,9 +4,9 @@ import com.twitter.io.TempFile import com.twitter.util.Try import java.io.File import java.security.cert.{CertificateException, X509Certificate} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class X509CertificateFileTest extends FunSuite { +class X509CertificateFileTest extends AnyFunSuite { private[this] def assertIsCslCert(cert: X509Certificate): Unit = { val subjectData: String = cert.getSubjectX500Principal.getName() diff --git a/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala index 251eb746b5..ba536cecf6 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala @@ -4,9 +4,9 @@ import com.twitter.io.TempFile import com.twitter.util.Try import java.io.File import java.security.cert.{CRLException, X509CRL} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class X509CrlFileTest extends FunSuite { +class X509CrlFileTest extends AnyFunSuite { private[this] def assertCrlException(tryCrl: Try[X509CRL]): Unit = PemFileTestUtils.assertException[CRLException, X509CRL](tryCrl) diff --git a/util-security/src/test/scala/com/twitter/util/security/X509TrustManagerFactoryTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509TrustManagerFactoryTest.scala index 5a3b8819cb..3e5b4d09fb 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509TrustManagerFactoryTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509TrustManagerFactoryTest.scala @@ -4,9 +4,9 @@ import com.twitter.io.TempFile import java.io.File import java.security.cert.X509Certificate import javax.net.ssl.{TrustManager, X509TrustManager} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class X509TrustManagerFactoryTest extends FunSuite { +class X509TrustManagerFactoryTest extends AnyFunSuite { private[this] def loadCertFromResource(resourcePath: String): X509Certificate = { val tempFile = TempFile.fromResourcePath(resourcePath) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala index 28ccedbcfe..986e965fd2 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala @@ -3,11 +3,12 @@ package com.twitter.util.logging import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ import org.scalatestplus.mockito.MockitoSugar -import org.scalatest.{FunSuite, Matchers} import org.slf4j import scala.language.reflectiveCalls +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers -class LoggingTest extends FunSuite with Matchers with MockitoSugar { +class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { /* Trace */ diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala index c897aed22b..287556ffda 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala @@ -3,12 +3,13 @@ package com.twitter.util.logging import org.mockito.ArgumentMatchers._ import org.mockito.Mockito._ import org.scalatestplus.mockito.MockitoSugar -import org.scalatest.{FunSuite, Matchers} import org.slf4j import org.slf4j.Marker import scala.language.reflectiveCalls +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers -class MarkerLoggingTest extends FunSuite with Matchers with MockitoSugar { +class MarkerLoggingTest extends AnyFunSuite with Matchers with MockitoSugar { test("Marker Logging#trace enabled") { val f = fixture(_.isTraceEnabled(anyObject[Marker]), isEnabled = true) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/SerializableLoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/SerializableLoggingTest.scala index 32460dbba3..2d06d59a67 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/SerializableLoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/SerializableLoggingTest.scala @@ -2,9 +2,9 @@ package com.twitter.util.logging import com.twitter.io.TempFolder import java.io._ -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class SerializableLoggingTest extends FunSuite with TempFolder { +class SerializableLoggingTest extends AnyFunSuite with TempFolder { def read[T](filename: String): T = { new ObjectInputStream(new FileInputStream(new File(folderName, filename))) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala index e440bc35ef..b3d07e9866 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/BroadcastStatsReceiverTest.scala @@ -2,9 +2,10 @@ package com.twitter.finagle.stats import com.twitter.util.{Future, Await} -import org.scalatest.{Matchers, FunSuite} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers -class BroadcastStatsReceiverTest extends FunSuite with Matchers { +class BroadcastStatsReceiverTest extends AnyFunSuite with Matchers { test("counter") { val recv1 = new InMemoryStatsReceiver diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CachedRegexTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CachedRegexTest.scala index b7aecd4628..bef0e3ced9 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/CachedRegexTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CachedRegexTest.scala @@ -1,9 +1,9 @@ package com.twitter.finagle.stats -import org.scalatest.FunSuite import scala.jdk.CollectionConverters._ +import org.scalatest.funsuite.AnyFunSuite -class CachedRegexTest extends FunSuite { +class CachedRegexTest extends AnyFunSuite { test("Lets through things it should let through") { val cachedRegex = new CachedRegex("foo".r) val original: Map[String, Number] = Map("bar" -> 3) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala index 4c03a81529..bde55f1085 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala @@ -1,8 +1,8 @@ package com.twitter.finagle.stats -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class CategorizingExceptionStatsHandlerTest extends FunSuite { +class CategorizingExceptionStatsHandlerTest extends AnyFunSuite { val categorizer = (t: Throwable) => { "clienterrors" } test("uses category, source, exception chains and rollup") { diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala index 70f257bbca..e5d8a3f258 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala @@ -2,10 +2,10 @@ package com.twitter.finagle.stats import java.util.concurrent.Executor import java.util.concurrent.atomic.AtomicInteger -import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} +import org.scalatest.funsuite.AnyFunSuite -class CumulativeGaugeTest extends FunSuite with Eventually with IntegrationPatience { +class CumulativeGaugeTest extends AnyFunSuite with Eventually with IntegrationPatience { private[this] val sameThreadExecutor = new Executor { def execute(command: Runnable): Unit = command.run() diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala index c18b08f375..bc7aa742c5 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala @@ -1,10 +1,10 @@ package com.twitter.finagle.stats import org.scalacheck.{Arbitrary, Gen} -import org.scalatest.FunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import org.scalatest.funsuite.AnyFunSuite -class DelegatingStatsReceiverTest extends FunSuite with ScalaCheckDrivenPropertyChecks { +class DelegatingStatsReceiverTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { class Ctx { var leafCounter = 0 diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala index 1dcc9941bd..173efcaf8b 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala @@ -1,8 +1,8 @@ package com.twitter.finagle.stats -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class DenylistStatsReceiverTest extends FunSuite { +class DenylistStatsReceiverTest extends AnyFunSuite { // scalafix:off StoreGaugesAsMemberVariables test("DenylistStatsReceiver denylists properly") { val inmemory = new InMemoryStatsReceiver() diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index fde38e8111..d5cf7ba331 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -2,11 +2,11 @@ package com.twitter.finagle.stats import java.io.{ByteArrayOutputStream, PrintStream} import java.nio.charset.StandardCharsets -import org.scalatest.FunSuite import org.scalatest.concurrent.{Eventually, IntegrationPatience} import scala.collection.parallel.immutable.ParRange +import org.scalatest.funsuite.AnyFunSuite -class InMemoryStatsReceiverTest extends FunSuite with Eventually with IntegrationPatience { +class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with IntegrationPatience { // scalafix:off StoreGaugesAsMemberVariables test("clear") { diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/LazyStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/LazyStatsReceiverTest.scala index d59d89d84b..67dea43e02 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/LazyStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/LazyStatsReceiverTest.scala @@ -1,8 +1,8 @@ package com.twitter.finagle.stats -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class LazyStatsReceiverTest extends FunSuite { +class LazyStatsReceiverTest extends AnyFunSuite { test("Doesn't eagerly initialize counters") { val underlying = new InMemoryStatsReceiver() val wrapped = new LazyStatsReceiver(underlying) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala index acb386c95e..4a4cf7dccf 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala @@ -1,8 +1,8 @@ package com.twitter.finagle.stats -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class MultiCategorizingExceptionStatsHandlerTest extends FunSuite { +class MultiCategorizingExceptionStatsHandlerTest extends AnyFunSuite { test("uses label, flags, source, exception chain and rolls up") { val receiver = new InMemoryStatsReceiver diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 372b5a5c03..946ba3492f 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -4,10 +4,10 @@ import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future} import java.util.concurrent.TimeUnit import org.mockito.Mockito._ -import org.scalatest.FunSuite import scala.collection.mutable.ArrayBuffer +import org.scalatest.funsuite.AnyFunSuite -class StatsReceiverTest extends FunSuite { +class StatsReceiverTest extends AnyFunSuite { test("RollupStatsReceiver counter/stats") { val mem = new InMemoryStatsReceiver val receiver = new RollupStatsReceiver(mem) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiverTest.scala index b85b7203a3..8d73923946 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiverTest.scala @@ -1,8 +1,9 @@ package com.twitter.finagle.stats -import org.scalatest.{FunSuite, OneInstancePerTest} +import org.scalatest.OneInstancePerTest +import org.scalatest.funsuite.AnyFunSuite -class VerbosityAdjustingStatsReceiverTest extends FunSuite with OneInstancePerTest { +class VerbosityAdjustingStatsReceiverTest extends AnyFunSuite with OneInstancePerTest { val inMemory = new InMemoryStatsReceiver() val verbose = new VerbosityAdjustingStatsReceiver(inMemory, Verbosity.Debug) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/WithHistogramDetailsTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/WithHistogramDetailsTest.scala index 564065ce03..89261f65e6 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/WithHistogramDetailsTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/WithHistogramDetailsTest.scala @@ -1,8 +1,8 @@ package com.twitter.finagle.stats -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class WithHistogramDetailsTest extends FunSuite { +class WithHistogramDetailsTest extends AnyFunSuite { test("AggregateWithHistogramDetails rejects empties") { intercept[IllegalArgumentException] { AggregateWithHistogramDetails(Nil) diff --git a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala index 80a3affb58..53cb73414d 100644 --- a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala +++ b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala @@ -18,13 +18,14 @@ package com.twitter.logging import java.util.{logging => jlogging} -import org.scalatest.{BeforeAndAfter, WordSpec} +import org.scalatest.BeforeAndAfter +import org.scalatest.wordspec.AnyWordSpec /** * Specify logging during unit tests via system property, defaulting to FATAL only. */ @deprecated("Prefer using slf4j for logging", "2019-11-20") -trait TestLogging extends BeforeAndAfter { self: WordSpec => +trait TestLogging extends BeforeAndAfter { self: AnyWordSpec => val logLevel: Level = Logger.levelNames(Option[String](System.getenv("log")).getOrElse("FATAL").toUpperCase) diff --git a/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala b/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala index 86484cfc4a..502350d755 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala +++ b/util-test/src/test/scala/com/twitter/util/testing/ArgumentCaptureTest.scala @@ -2,10 +2,10 @@ package com.twitter.util.testing import org.mockito.Matchers._ import org.mockito.Mockito._ -import org.scalatest.FunSuite import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.funsuite.AnyFunSuite -class ArgumentCaptureTest extends FunSuite with MockitoSugar with ArgumentCapture { +class ArgumentCaptureTest extends AnyFunSuite with MockitoSugar with ArgumentCapture { class MockSubject { def method1(arg: String): Long = 0L def method2(arg1: String, arg2: String): Long = 0L diff --git a/util-thrift/src/test/scala/com/twitter/util/ThriftCodecTest.scala b/util-thrift/src/test/scala/com/twitter/util/ThriftCodecTest.scala index c440c4ee16..6e1534f37f 100644 --- a/util-thrift/src/test/scala/com/twitter/util/ThriftCodecTest.scala +++ b/util-thrift/src/test/scala/com/twitter/util/ThriftCodecTest.scala @@ -1,8 +1,8 @@ package com.twitter.util -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class ThriftCodecTest extends FunSuite { +class ThriftCodecTest extends AnyFunSuite { private def roundTrip(codec: ThriftCodec[TestThriftStructure, _]): Unit = { val struct = new TestThriftStructure("aString", 5) diff --git a/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala b/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala index 4890935641..31fabad810 100644 --- a/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala +++ b/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala @@ -16,9 +16,9 @@ package com.twitter.util -import org.scalatest.WordSpec +import org.scalatest.wordspec.AnyWordSpec -class ThriftSerializerTest extends WordSpec { +class ThriftSerializerTest extends AnyWordSpec { val aString = "me gustan los tacos y los burritos" val aNumber = 42 val original = new TestThriftStructure(aString, aNumber) diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala index fc3cbff309..f8e87a2515 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala @@ -6,13 +6,13 @@ import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ import com.twitter.util.{Duration, Return, StorageUnit, Throw} -import org.scalatest.FunSuite import scala.jdk.CollectionConverters._ +import org.scalatest.funsuite.AnyFunSuite // Used for veryifying custom deserialization case class Foo(number: Double) -class JsonTunableMapperTest extends FunSuite { +class JsonTunableMapperTest extends AnyFunSuite { test("returns a Throw if json is empty") { JsonTunableMapper().parse("") match { diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/ServiceLoadedTunableMapTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/ServiceLoadedTunableMapTest.scala index e0486392ac..ddba96abb2 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/ServiceLoadedTunableMapTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/ServiceLoadedTunableMapTest.scala @@ -1,6 +1,6 @@ package com.twitter.util.tunable -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite class ServiceLoadedTunableTestClient1 extends ServiceLoadedTunableMap with TunableMap.Proxy { @@ -23,7 +23,7 @@ class ServiceLoadedTunableTestClient2Dup extends ServiceLoadedTunableMap with Tu def id: String = "client2" } -class ServiceLoadedTunableMapTest extends FunSuite { +class ServiceLoadedTunableMapTest extends AnyFunSuite { test( "IllegalArgumentException thrown when there is more than one ServiceLoadedTunableMap " + diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/TunableMapTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/TunableMapTest.scala index 78cdfedf43..12a8c52597 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/TunableMapTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/TunableMapTest.scala @@ -1,8 +1,8 @@ package com.twitter.util.tunable -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class MutableTest extends FunSuite { +class MutableTest extends AnyFunSuite { test("map is initially empty") { assert(TunableMap.newMutable().entries.size == 0) @@ -384,7 +384,7 @@ class MutableTest extends FunSuite { } } -class NullTunableMapTest extends FunSuite { +class NullTunableMapTest extends AnyFunSuite { test("NullTunableMap returns Tunable.None") { val nullTunableMap = NullTunableMap diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/TunableTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/TunableTest.scala index 9a5b875ac0..0a7c7ed32c 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/TunableTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/TunableTest.scala @@ -1,8 +1,8 @@ package com.twitter.util.tunable -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class TunableTest extends FunSuite { +class TunableTest extends AnyFunSuite { test("Tunable cannot have empty id") { intercept[IllegalArgumentException] { diff --git a/util-zk-test/src/test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala b/util-zk-test/src/test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala index fe56e51681..a6f9991aa9 100644 --- a/util-zk-test/src/test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala +++ b/util-zk-test/src/test/scala/com/twitter/zk/ServerCnxnFactoryTest.scala @@ -4,9 +4,10 @@ import com.twitter.io.TempDirectory import java.io.File import java.net.InetAddress import org.apache.zookeeper.server.ZooKeeperServer -import org.scalatest.{BeforeAndAfter, FunSuite} +import org.scalatest.BeforeAndAfter +import org.scalatest.funsuite.AnyFunSuite -class ServerCnxnFactoryTest extends FunSuite with BeforeAndAfter { +class ServerCnxnFactoryTest extends AnyFunSuite with BeforeAndAfter { val addr = InetAddress.getLocalHost var testServer: ZooKeeperServer = null diff --git a/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala b/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala index e9f4badef7..5890f19ee0 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ConnectorTest.scala @@ -1,12 +1,12 @@ package com.twitter.zk import org.mockito.Mockito._ -import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar import com.twitter.util.Future +import org.scalatest.wordspec.AnyWordSpec -class ConnectorTest extends WordSpec with MockitoSugar { +class ConnectorTest extends AnyWordSpec with MockitoSugar { "Connector.RoundRobin" should { "require underlying connections" in { intercept[Exception] { diff --git a/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala index 1e31581e33..84c89beb75 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZNodeTest.scala @@ -3,10 +3,10 @@ package com.twitter.zk /** * @author ver@twitter.com */ -import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.wordspec.AnyWordSpec -class ZNodeTest extends WordSpec with MockitoSugar { +class ZNodeTest extends AnyWordSpec with MockitoSugar { "ZNode" should { class ZNodeSpecHelper { val zk = mock[ZkClient] diff --git a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala index 5196a1e6a6..7915d044fc 100644 --- a/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/ZkClientTest.scala @@ -10,14 +10,14 @@ import org.mockito.Mockito._ import org.mockito._ import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer -import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ import com.twitter.logging.{Level, Logger} import com.twitter.util._ +import org.scalatest.wordspec.AnyWordSpec -class ZkClientTest extends WordSpec with MockitoSugar { +class ZkClientTest extends AnyWordSpec with MockitoSugar { Logger.get("").setLevel(Level.FATAL) implicit val javaTimer = new JavaTimer(true) diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala index 648af965ec..d275ccd629 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ShardCoordinatorTest.scala @@ -3,14 +3,14 @@ package com.twitter.zk.coordination import scala.jdk.CollectionConverters._ import org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE -import org.scalatest.WordSpec import org.scalatestplus.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, JavaTimer} import com.twitter.zk.{NativeConnector, RetryPolicy, ZkClient} +import org.scalatest.wordspec.AnyWordSpec -class ShardCoordinatorTest extends WordSpec with MockitoSugar { +class ShardCoordinatorTest extends AnyWordSpec with MockitoSugar { "ShardCoordinator" should { diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala index 591dccbbaf..806545e6c1 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/ZkAsyncSemaphoreTest.scala @@ -7,14 +7,14 @@ import com.twitter.zk.{NativeConnector, RetryPolicy, ZkClient, ZNode} import java.util.concurrent.ConcurrentLinkedQueue import org.apache.zookeeper.KeeperException.NoNodeException import org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE -import org.scalatest.WordSpec import org.scalatest.concurrent.Waiters import org.scalatest.concurrent.Eventually._ import org.scalatest.time.{Millis, Seconds, Span} import org.scalatestplus.mockito.MockitoSugar import scala.jdk.CollectionConverters._ +import org.scalatest.wordspec.AnyWordSpec -class ZkAsyncSemaphoreTest extends WordSpec with MockitoSugar with Waiters { +class ZkAsyncSemaphoreTest extends AnyWordSpec with MockitoSugar with Waiters { "ZkAsyncSemaphore" should { From 6623204fae6cb0b4585b861a27922494f3159a09 Mon Sep 17 00:00:00 2001 From: Marcy Reyes Date: Fri, 21 Aug 2020 16:47:57 +0000 Subject: [PATCH 374/796] Corrected CSL team in PROJECT Files This is a follow up to the PROJECT/PROJECT.yml migration in source (D528770). The csl team was incorrectly included as csl@twitter.com. Updated all PROJECT files containing the CSL team to reflect them as csl-team:ldap. Differential Revision: https://phabricator.twitter.biz/D536529 --- PROJECT.yml | 2 +- util-zk/PROJECT | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PROJECT.yml b/PROJECT.yml index 0b31cc64f9..bc21dcd32a 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -8,4 +8,4 @@ owners: - ryano - vkostyukov watchers: - - csl@twitter.com + - csl-team:ldap diff --git a/util-zk/PROJECT b/util-zk/PROJECT index 646a189266..e930d5fcfc 100644 --- a/util-zk/PROJECT +++ b/util-zk/PROJECT @@ -8,5 +8,5 @@ owners: - vkostyukov watchers: - coord-team@twitter.com - - csl@twitter.com + - csl-team:ldap - observability@twitter.com From 4c971ba9ef5fa11fb3837746205fbab4a7e24f04 Mon Sep 17 00:00:00 2001 From: Bonnie Eisenman Date: Tue, 25 Aug 2020 20:09:26 +0000 Subject: [PATCH 375/796] Twitter-oss: Prepare OSS librares for patch release 20.8.1 Problem: We want to release the next versions of our Twitter OSS libraries 20.8.1 - util - scrooge - finagle - twitter-server - finatra Solution: Prepare libraries for their next release. Differential Revision: https://phabricator.twitter.biz/D538583 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dfc2818633..448a7d0818 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.8.1 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index d55c2e55a8..e5d48edd3d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.8.0" +val utilCore = "com.twitter" %% "util-core" % "20.8.1" ``` # Units diff --git a/build.sbt b/build.sbt index a7fee36057..5973b7c6c1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.9.0-SNAPSHOT" +val releaseVersion = "20.8.1" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.0" From df3c5599cb8365bf194e36b69fcfa5d9b36cd936 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 26 Aug 2020 01:48:28 +0000 Subject: [PATCH 376/796] util/util-mock: Move to com.twitter.util.mock package Problem/Solution We've agreed that source in util/ should be ideally be namespaced under `com.twitter.util` where possible. Let's update util/util-mock before it is released to be in package `com.twitter.util.mock`. JIRA Issues: CSL-10106 Differential Revision: https://phabricator.twitter.biz/D538625 --- util-mock/src/main/scala/com/twitter/{ => util}/mock/BUILD | 0 .../src/main/scala/com/twitter/{ => util}/mock/Mockito.scala | 4 ++-- util-mock/src/test/java/com/twitter/{ => util}/mock/BUILD | 0 .../src/test/java/com/twitter/{ => util}/mock/JavaFoo.java | 2 +- util-mock/src/test/scala/com/twitter/{ => util}/mock/BUILD | 4 ++-- .../scala/com/twitter/{ => util}/mock/MockitoTraitTest.scala | 2 +- .../twitter/{ => util}/mock/MockitoTraitWithFixtureTest.scala | 2 +- .../{ => util}/mock/ResetMocksAfterEachTestTraitTest.scala | 4 ++-- 8 files changed, 9 insertions(+), 9 deletions(-) rename util-mock/src/main/scala/com/twitter/{ => util}/mock/BUILD (100%) rename util-mock/src/main/scala/com/twitter/{ => util}/mock/Mockito.scala (98%) rename util-mock/src/test/java/com/twitter/{ => util}/mock/BUILD (100%) rename util-mock/src/test/java/com/twitter/{ => util}/mock/JavaFoo.java (69%) rename util-mock/src/test/scala/com/twitter/{ => util}/mock/BUILD (76%) rename util-mock/src/test/scala/com/twitter/{ => util}/mock/MockitoTraitTest.scala (99%) rename util-mock/src/test/scala/com/twitter/{ => util}/mock/MockitoTraitWithFixtureTest.scala (97%) rename util-mock/src/test/scala/com/twitter/{ => util}/mock/ResetMocksAfterEachTestTraitTest.scala (94%) diff --git a/util-mock/src/main/scala/com/twitter/mock/BUILD b/util-mock/src/main/scala/com/twitter/util/mock/BUILD similarity index 100% rename from util-mock/src/main/scala/com/twitter/mock/BUILD rename to util-mock/src/main/scala/com/twitter/util/mock/BUILD diff --git a/util-mock/src/main/scala/com/twitter/mock/Mockito.scala b/util-mock/src/main/scala/com/twitter/util/mock/Mockito.scala similarity index 98% rename from util-mock/src/main/scala/com/twitter/mock/Mockito.scala rename to util-mock/src/main/scala/com/twitter/util/mock/Mockito.scala index 1ff639b676..8f6f3da407 100644 --- a/util-mock/src/main/scala/com/twitter/mock/Mockito.scala +++ b/util-mock/src/main/scala/com/twitter/util/mock/Mockito.scala @@ -1,4 +1,4 @@ -package com.twitter.mock +package com.twitter.util.mock import org.mockito.{ArgumentMatchersSugar, IdiomaticMockito} @@ -13,7 +13,7 @@ import org.mockito.{ArgumentMatchersSugar, IdiomaticMockito} * * This trait uses [[org.mockito.IdiomaticMockito]] which is heavily influenced by ScalaTest Matchers. * - * To use, mix in the [[com.twitter.mock.Mockito]] trait where desired. + * To use, mix in the [[com.twitter.util.mock.Mockito]] trait where desired. * * ==Create a new mock== * diff --git a/util-mock/src/test/java/com/twitter/mock/BUILD b/util-mock/src/test/java/com/twitter/util/mock/BUILD similarity index 100% rename from util-mock/src/test/java/com/twitter/mock/BUILD rename to util-mock/src/test/java/com/twitter/util/mock/BUILD diff --git a/util-mock/src/test/java/com/twitter/mock/JavaFoo.java b/util-mock/src/test/java/com/twitter/util/mock/JavaFoo.java similarity index 69% rename from util-mock/src/test/java/com/twitter/mock/JavaFoo.java rename to util-mock/src/test/java/com/twitter/util/mock/JavaFoo.java index 3eae8a1c32..012851460d 100644 --- a/util-mock/src/test/java/com/twitter/mock/JavaFoo.java +++ b/util-mock/src/test/java/com/twitter/util/mock/JavaFoo.java @@ -1,4 +1,4 @@ -package com.twitter.mock; +package com.twitter.util.mock; public interface JavaFoo { Integer varargMethod(Integer... arg); diff --git a/util-mock/src/test/scala/com/twitter/mock/BUILD b/util-mock/src/test/scala/com/twitter/util/mock/BUILD similarity index 76% rename from util-mock/src/test/scala/com/twitter/mock/BUILD rename to util-mock/src/test/scala/com/twitter/util/mock/BUILD index 3a88794a25..32dffaed56 100644 --- a/util-mock/src/test/scala/com/twitter/mock/BUILD +++ b/util-mock/src/test/scala/com/twitter/util/mock/BUILD @@ -8,7 +8,7 @@ junit_tests( "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:scala", - "util/util-mock/src/main/scala/com/twitter/mock", - "util/util-mock/src/test/java/com/twitter/mock", + "util/util-mock/src/main/scala/com/twitter/util/mock", + "util/util-mock/src/test/java/com/twitter/util/mock", ], ) diff --git a/util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala b/util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitTest.scala similarity index 99% rename from util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala rename to util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitTest.scala index 1ee36d241e..a5a3f48361 100644 --- a/util-mock/src/test/scala/com/twitter/mock/MockitoTraitTest.scala +++ b/util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitTest.scala @@ -1,4 +1,4 @@ -package com.twitter.mock +package com.twitter.util.mock import com.twitter.conversions.DurationOps._ import com.twitter.util._ diff --git a/util-mock/src/test/scala/com/twitter/mock/MockitoTraitWithFixtureTest.scala b/util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitWithFixtureTest.scala similarity index 97% rename from util-mock/src/test/scala/com/twitter/mock/MockitoTraitWithFixtureTest.scala rename to util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitWithFixtureTest.scala index 6eeaa325a3..fdaaa2fee9 100644 --- a/util-mock/src/test/scala/com/twitter/mock/MockitoTraitWithFixtureTest.scala +++ b/util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitWithFixtureTest.scala @@ -1,4 +1,4 @@ -package com.twitter.mock +package com.twitter.util.mock import org.junit.runner.RunWith import org.mockito.exceptions.verification.{NeverWantedButInvoked, NoInteractionsWanted} diff --git a/util-mock/src/test/scala/com/twitter/mock/ResetMocksAfterEachTestTraitTest.scala b/util-mock/src/test/scala/com/twitter/util/mock/ResetMocksAfterEachTestTraitTest.scala similarity index 94% rename from util-mock/src/test/scala/com/twitter/mock/ResetMocksAfterEachTestTraitTest.scala rename to util-mock/src/test/scala/com/twitter/util/mock/ResetMocksAfterEachTestTraitTest.scala index 43d7af5e99..d62851065e 100644 --- a/util-mock/src/test/scala/com/twitter/mock/ResetMocksAfterEachTestTraitTest.scala +++ b/util-mock/src/test/scala/com/twitter/util/mock/ResetMocksAfterEachTestTraitTest.scala @@ -1,4 +1,4 @@ -package com.twitter.mock +package com.twitter.util.mock import org.junit.runner.RunWith import org.mockito.MockitoSugar @@ -24,7 +24,7 @@ object ResetMocksAfterEachTestTraitTest { } /** - * Ensure [[ResetMocksAfterEachTest]] works with [[com.twitter.mock.Mockito]] + * Ensure [[ResetMocksAfterEachTest]] works with [[com.twitter.util.mock.Mockito]] */ @RunWith(classOf[JUnitRunner]) class ResetMocksAfterEachTestTraitTest From 9cbac2182c153b2284bf8bf266edfde052007a95 Mon Sep 17 00:00:00 2001 From: Bonnie Eisenman Date: Wed, 26 Aug 2020 14:29:14 +0000 Subject: [PATCH 377/796] Twitter-oss: update OSS libraries post-release to 20.9.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D538818 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e5d48edd3d..eae78e8fd1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 5973b7c6c1..a7fee36057 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.8.1" +val releaseVersion = "20.9.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.0" From 5d8877dba7cdf278c105ab4747c7886d633dc38b Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 27 Aug 2020 22:00:21 +0000 Subject: [PATCH 378/796] 3rdparty: Bump the jackson versions to 2.11.2 Problem We're on an old version of jackson Solution Upgrade!! JIRA Issues: CSL-10108 Differential Revision: https://phabricator.twitter.biz/D538440 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 448a7d0818..240d48b0bf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Bump version of Jackson to 2.11.2. ``PHAB_ID=D538440`` + 20.8.1 ------ diff --git a/build.sbt b/build.sbt index a7fee36057..0bd6fc0351 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ import scoverage.ScoverageKeys val releaseVersion = "20.9.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.11.0" +val jacksonVersion = "2.11.2" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" From bcc3b70d008dc2b7da30831f46164b7f244079bb Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 31 Aug 2020 21:52:17 +0000 Subject: [PATCH 379/796] util-inject: Move to com.twitter.util.inject packaging Problem/Solution Update package namespace to `com.twitter.util.inject`. JIRA Issues: CSL-10107 Differential Revision: https://phabricator.twitter.biz/D542040 --- util-inject/README.md | 7 +++++++ .../src/main/java/com/twitter/{ => util}/inject/flag/BUILD | 0 .../java/com/twitter/{ => util}/inject/flag/Flagged.java | 2 +- .../com/twitter/{ => util}/inject/flag/FlaggedImpl.java | 2 +- .../java/com/twitter/{ => util}/inject/flag/Flags.java | 2 +- .../src/main/scala/com/twitter/{ => util}/inject/BUILD | 0 .../main/scala/com/twitter/{ => util}/inject/Flags.scala | 0 .../scala/com/twitter/{ => util}/inject/Injector.scala | 0 .../com/twitter/{ => util}/inject/InjectorModule.scala | 0 .../com/twitter/{ => util}/inject/TwitterBaseModule.scala | 0 .../com/twitter/{ => util}/inject/TwitterModule.scala | 0 .../com/twitter/{ => util}/inject/TwitterModuleFlags.scala | 0 .../twitter/{ => util}/inject/TwitterModuleLifecycle.scala | 0 .../scala/com/twitter/{ => util}/inject/TypeUtils.scala | 0 .../scala/com/twitter/{ => util}/inject/internal/BUILD | 6 +++--- .../twitter/{ => util}/inject/internal/FlagsModule.scala | 2 +- .../{ => util}/inject/internal/InstalledModules.scala | 0 .../com/twitter/{ => util}/inject/internal/Modules.scala | 0 .../inject/internal/TwitterTypeConvertersModule.scala | 0 .../com/twitter/{ => util}/inject/internal/iterable.scala | 0 .../com/twitter/{ => util}/inject/internal/package.scala | 0 .../src/test/java/com/twitter/{ => util}/inject/BUILD | 0 .../twitter/{ => util}/inject/TestBindingAnnotation.java | 0 .../src/test/scala/com/twitter/{ => util}/inject/BUILD | 4 ++-- .../scala/com/twitter/{ => util}/inject/InjectorTest.scala | 0 .../com/twitter/{ => util}/inject/TwitterModuleTest.scala | 0 .../com/twitter/{ => util}/inject/TypeUtilsTest.scala | 0 .../test/scala/com/twitter/{ => util}/inject/classes.scala | 0 .../{ => util}/inject/module/SeqAbstractModule.scala | 0 .../com/twitter/{ => util}/inject/module/TestModule.scala | 0 .../{ => util}/inject/module/TestTwitterModule.scala | 0 .../com/twitter/{ => util}/inject/servicefactory.scala | 0 32 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 util-inject/README.md rename util-inject/src/main/java/com/twitter/{ => util}/inject/flag/BUILD (100%) rename util-inject/src/main/java/com/twitter/{ => util}/inject/flag/Flagged.java (94%) rename util-inject/src/main/java/com/twitter/{ => util}/inject/flag/FlaggedImpl.java (96%) rename util-inject/src/main/java/com/twitter/{ => util}/inject/flag/Flags.java (95%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/BUILD (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/Flags.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/Injector.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/InjectorModule.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/TwitterBaseModule.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/TwitterModule.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/TwitterModuleFlags.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/TwitterModuleLifecycle.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/TypeUtils.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/internal/BUILD (74%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/internal/FlagsModule.scala (98%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/internal/InstalledModules.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/internal/Modules.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/internal/TwitterTypeConvertersModule.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/internal/iterable.scala (100%) rename util-inject/src/main/scala/com/twitter/{ => util}/inject/internal/package.scala (100%) rename util-inject/src/test/java/com/twitter/{ => util}/inject/BUILD (100%) rename util-inject/src/test/java/com/twitter/{ => util}/inject/TestBindingAnnotation.java (100%) rename util-inject/src/test/scala/com/twitter/{ => util}/inject/BUILD (79%) rename util-inject/src/test/scala/com/twitter/{ => util}/inject/InjectorTest.scala (100%) rename util-inject/src/test/scala/com/twitter/{ => util}/inject/TwitterModuleTest.scala (100%) rename util-inject/src/test/scala/com/twitter/{ => util}/inject/TypeUtilsTest.scala (100%) rename util-inject/src/test/scala/com/twitter/{ => util}/inject/classes.scala (100%) rename util-inject/src/test/scala/com/twitter/{ => util}/inject/module/SeqAbstractModule.scala (100%) rename util-inject/src/test/scala/com/twitter/{ => util}/inject/module/TestModule.scala (100%) rename util-inject/src/test/scala/com/twitter/{ => util}/inject/module/TestTwitterModule.scala (100%) rename util-inject/src/test/scala/com/twitter/{ => util}/inject/servicefactory.scala (100%) diff --git a/util-inject/README.md b/util-inject/README.md new file mode 100644 index 0000000000..c62f2a3072 --- /dev/null +++ b/util-inject/README.md @@ -0,0 +1,7 @@ +util-inject +=========== + +2020-08-31 + +This library is currently in development and experimental. We do +not recommend users attempt to adopt this code at this time. diff --git a/util-inject/src/main/java/com/twitter/inject/flag/BUILD b/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD similarity index 100% rename from util-inject/src/main/java/com/twitter/inject/flag/BUILD rename to util-inject/src/main/java/com/twitter/util/inject/flag/BUILD diff --git a/util-inject/src/main/java/com/twitter/inject/flag/Flagged.java b/util-inject/src/main/java/com/twitter/util/inject/flag/Flagged.java similarity index 94% rename from util-inject/src/main/java/com/twitter/inject/flag/Flagged.java rename to util-inject/src/main/java/com/twitter/util/inject/flag/Flagged.java index 9dcb73e668..0c2c596e01 100644 --- a/util-inject/src/main/java/com/twitter/inject/flag/Flagged.java +++ b/util-inject/src/main/java/com/twitter/util/inject/flag/Flagged.java @@ -1,4 +1,4 @@ -package com.twitter.inject.flag; +package com.twitter.util.inject.flag; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/util-inject/src/main/java/com/twitter/inject/flag/FlaggedImpl.java b/util-inject/src/main/java/com/twitter/util/inject/flag/FlaggedImpl.java similarity index 96% rename from util-inject/src/main/java/com/twitter/inject/flag/FlaggedImpl.java rename to util-inject/src/main/java/com/twitter/util/inject/flag/FlaggedImpl.java index c5fc11c566..591629148b 100644 --- a/util-inject/src/main/java/com/twitter/inject/flag/FlaggedImpl.java +++ b/util-inject/src/main/java/com/twitter/util/inject/flag/FlaggedImpl.java @@ -1,4 +1,4 @@ -package com.twitter.inject.flag; +package com.twitter.util.inject.flag; import java.io.Serializable; import java.lang.annotation.Annotation; diff --git a/util-inject/src/main/java/com/twitter/inject/flag/Flags.java b/util-inject/src/main/java/com/twitter/util/inject/flag/Flags.java similarity index 95% rename from util-inject/src/main/java/com/twitter/inject/flag/Flags.java rename to util-inject/src/main/java/com/twitter/util/inject/flag/Flags.java index 6b7c3c94a3..0695caa89f 100644 --- a/util-inject/src/main/java/com/twitter/inject/flag/Flags.java +++ b/util-inject/src/main/java/com/twitter/util/inject/flag/Flags.java @@ -1,4 +1,4 @@ -package com.twitter.inject.flag; +package com.twitter.util.inject.flag; import com.google.inject.Key; diff --git a/util-inject/src/main/scala/com/twitter/inject/BUILD b/util-inject/src/main/scala/com/twitter/util/inject/BUILD similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/BUILD rename to util-inject/src/main/scala/com/twitter/util/inject/BUILD diff --git a/util-inject/src/main/scala/com/twitter/inject/Flags.scala b/util-inject/src/main/scala/com/twitter/util/inject/Flags.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/Flags.scala rename to util-inject/src/main/scala/com/twitter/util/inject/Flags.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/Injector.scala b/util-inject/src/main/scala/com/twitter/util/inject/Injector.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/Injector.scala rename to util-inject/src/main/scala/com/twitter/util/inject/Injector.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/InjectorModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/InjectorModule.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/InjectorModule.scala rename to util-inject/src/main/scala/com/twitter/util/inject/InjectorModule.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/TwitterBaseModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/TwitterBaseModule.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/TwitterBaseModule.scala rename to util-inject/src/main/scala/com/twitter/util/inject/TwitterBaseModule.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/TwitterModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/TwitterModule.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/TwitterModule.scala rename to util-inject/src/main/scala/com/twitter/util/inject/TwitterModule.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/TwitterModuleFlags.scala b/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/TwitterModuleFlags.scala rename to util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/TwitterModuleLifecycle.scala b/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleLifecycle.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/TwitterModuleLifecycle.scala rename to util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleLifecycle.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/TypeUtils.scala b/util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/TypeUtils.scala rename to util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/BUILD b/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD similarity index 74% rename from util-inject/src/main/scala/com/twitter/inject/internal/BUILD rename to util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD index 81cf77bb49..22537bef23 100644 --- a/util-inject/src/main/scala/com/twitter/inject/internal/BUILD +++ b/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD @@ -12,13 +12,13 @@ scala_library( "3rdparty/jvm/javax/inject:javax.inject", "util/util-app/src/main/scala", "util/util-core:util-core-util", - "util/util-inject/src/main/java/com/twitter/inject/flag", - "util/util-inject/src/main/scala/com/twitter/inject", + "util/util-inject/src/main/java/com/twitter/util/inject/flag", + "util/util-inject/src/main/scala/com/twitter/util/inject", "util/util-slf4j-api/src/main/scala", ], exports = [ "util/util-app/src/main/scala", - "util/util-inject/src/main/java/com/twitter/inject/flag", + "util/util-inject/src/main/java/com/twitter/util/inject/flag", "util/util-slf4j-api/src/main/scala", ], ) diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/FlagsModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/FlagsModule.scala similarity index 98% rename from util-inject/src/main/scala/com/twitter/inject/internal/FlagsModule.scala rename to util-inject/src/main/scala/com/twitter/util/inject/internal/FlagsModule.scala index d98c2a6423..e7d2057883 100644 --- a/util-inject/src/main/scala/com/twitter/inject/internal/FlagsModule.scala +++ b/util-inject/src/main/scala/com/twitter/util/inject/internal/FlagsModule.scala @@ -1,7 +1,7 @@ package com.twitter.inject.internal import com.google.inject.{AbstractModule, Key} -import com.twitter.inject.flag.Flags +import com.twitter.util.inject.flag.Flags import com.twitter.util.logging.Logging import javax.inject.Provider diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/InstalledModules.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/InstalledModules.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/internal/InstalledModules.scala rename to util-inject/src/main/scala/com/twitter/util/inject/internal/InstalledModules.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/Modules.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/Modules.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/internal/Modules.scala rename to util-inject/src/main/scala/com/twitter/util/inject/internal/Modules.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/TwitterTypeConvertersModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/TwitterTypeConvertersModule.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/internal/TwitterTypeConvertersModule.scala rename to util-inject/src/main/scala/com/twitter/util/inject/internal/TwitterTypeConvertersModule.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/iterable.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/iterable.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/internal/iterable.scala rename to util-inject/src/main/scala/com/twitter/util/inject/internal/iterable.scala diff --git a/util-inject/src/main/scala/com/twitter/inject/internal/package.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/package.scala similarity index 100% rename from util-inject/src/main/scala/com/twitter/inject/internal/package.scala rename to util-inject/src/main/scala/com/twitter/util/inject/internal/package.scala diff --git a/util-inject/src/test/java/com/twitter/inject/BUILD b/util-inject/src/test/java/com/twitter/util/inject/BUILD similarity index 100% rename from util-inject/src/test/java/com/twitter/inject/BUILD rename to util-inject/src/test/java/com/twitter/util/inject/BUILD diff --git a/util-inject/src/test/java/com/twitter/inject/TestBindingAnnotation.java b/util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java similarity index 100% rename from util-inject/src/test/java/com/twitter/inject/TestBindingAnnotation.java rename to util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java diff --git a/util-inject/src/test/scala/com/twitter/inject/BUILD b/util-inject/src/test/scala/com/twitter/util/inject/BUILD similarity index 79% rename from util-inject/src/test/scala/com/twitter/inject/BUILD rename to util-inject/src/test/scala/com/twitter/util/inject/BUILD index 9592a332d1..c13d7fd0bc 100644 --- a/util-inject/src/test/scala/com/twitter/inject/BUILD +++ b/util-inject/src/test/scala/com/twitter/util/inject/BUILD @@ -10,7 +10,7 @@ junit_tests( "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/scalatestplus:scalacheck", - "util/util-inject/src/main/scala/com/twitter/inject", - "util/util-inject/src/test/java/com/twitter/inject", + "util/util-inject/src/main/scala/com/twitter/util/inject", + "util/util-inject/src/test/java/com/twitter/util/inject", ], ) diff --git a/util-inject/src/test/scala/com/twitter/inject/InjectorTest.scala b/util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala similarity index 100% rename from util-inject/src/test/scala/com/twitter/inject/InjectorTest.scala rename to util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala diff --git a/util-inject/src/test/scala/com/twitter/inject/TwitterModuleTest.scala b/util-inject/src/test/scala/com/twitter/util/inject/TwitterModuleTest.scala similarity index 100% rename from util-inject/src/test/scala/com/twitter/inject/TwitterModuleTest.scala rename to util-inject/src/test/scala/com/twitter/util/inject/TwitterModuleTest.scala diff --git a/util-inject/src/test/scala/com/twitter/inject/TypeUtilsTest.scala b/util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala similarity index 100% rename from util-inject/src/test/scala/com/twitter/inject/TypeUtilsTest.scala rename to util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala diff --git a/util-inject/src/test/scala/com/twitter/inject/classes.scala b/util-inject/src/test/scala/com/twitter/util/inject/classes.scala similarity index 100% rename from util-inject/src/test/scala/com/twitter/inject/classes.scala rename to util-inject/src/test/scala/com/twitter/util/inject/classes.scala diff --git a/util-inject/src/test/scala/com/twitter/inject/module/SeqAbstractModule.scala b/util-inject/src/test/scala/com/twitter/util/inject/module/SeqAbstractModule.scala similarity index 100% rename from util-inject/src/test/scala/com/twitter/inject/module/SeqAbstractModule.scala rename to util-inject/src/test/scala/com/twitter/util/inject/module/SeqAbstractModule.scala diff --git a/util-inject/src/test/scala/com/twitter/inject/module/TestModule.scala b/util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala similarity index 100% rename from util-inject/src/test/scala/com/twitter/inject/module/TestModule.scala rename to util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala diff --git a/util-inject/src/test/scala/com/twitter/inject/module/TestTwitterModule.scala b/util-inject/src/test/scala/com/twitter/util/inject/module/TestTwitterModule.scala similarity index 100% rename from util-inject/src/test/scala/com/twitter/inject/module/TestTwitterModule.scala rename to util-inject/src/test/scala/com/twitter/util/inject/module/TestTwitterModule.scala diff --git a/util-inject/src/test/scala/com/twitter/inject/servicefactory.scala b/util-inject/src/test/scala/com/twitter/util/inject/servicefactory.scala similarity index 100% rename from util-inject/src/test/scala/com/twitter/inject/servicefactory.scala rename to util-inject/src/test/scala/com/twitter/util/inject/servicefactory.scala From b08a02ec68ca2e987ecec0cb5f7b177d9dccb831 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 1 Sep 2020 23:04:37 +0000 Subject: [PATCH 380/796] util-app: Lay out groundwork for injectable flags Problem Application frameworks (Finatra, for example) might want to take advantage of type information available for Flags in runtime. This may enable the framework's middleware to inject flag values via DI. Solution Expose type information on each Flaggable such that application frameworks relying on DI can use it to leverage Flag injection. JIRA Issues: CSL-10061 Differential Revision: https://phabricator.twitter.biz/D542135 --- CHANGELOG.rst | 7 ++ .../src/main/scala/com/twitter/app/Flag.scala | 7 +- .../scala/com/twitter/app/Flaggable.scala | 98 ++++++++++++++----- .../java/com/twitter/app/JavaFlagTest.java | 4 +- .../test/scala/com/twitter/app/FlagTest.scala | 6 ++ .../scala/com/twitter/app/FlaggableTest.scala | 8 ++ .../util/inject/TwitterModuleFlags.scala | 12 +-- 7 files changed, 109 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 240d48b0bf..6177a85454 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-app: `Flaggable.mandatory` now takes implicit `ClassTag[T]` as an argument. This is change is + source-compatible in Scala but requires Java users to pass argument explicitly via + `ClassTag$.MODULE$.apply(clazz)`. ``PHAB_ID=D542135`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-app/src/main/scala/com/twitter/app/Flag.scala b/util-app/src/main/scala/com/twitter/app/Flag.scala index 2f61c94a86..e33bf59dfb 100644 --- a/util-app/src/main/scala/com/twitter/app/Flag.scala +++ b/util-app/src/main/scala/com/twitter/app/Flag.scala @@ -82,7 +82,7 @@ class Flag[T: Flaggable] private[app] ( private[twitter] def this(name: String, help: String, usage: String) = this(name, help, usage, false) - protected val flaggable: Flaggable[T] = implicitly[Flaggable[T]] + private[twitter] val flaggable: Flaggable[T] = implicitly[Flaggable[T]] private[this] def localValue: Option[T] = localFlagValues() match { case None => None @@ -250,6 +250,11 @@ class Flag[T: Flaggable] private[app] ( */ def getWithDefault: Option[T] = valueOrDefault + /** + * Get the default value for this flag. This will return `None` if no default value is specified. + */ + private[twitter] def getDefault: Option[T] = default + /** * Get the *unparsed* (as string) value if it has been set or if there is a default value supplied. * diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index b350ad397e..ce6327ba8a 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -2,6 +2,7 @@ package com.twitter.app import com.twitter.util.{Duration, StorageUnit, Time, TimeFormat} import java.io.File +import java.lang.reflect.Type import java.lang.{ Boolean => JBoolean, Double => JDouble, @@ -13,6 +14,7 @@ import java.net.InetSocketAddress import java.time.LocalTime import java.util.{List => JList, Map => JMap, Set => JSet} import scala.jdk.CollectionConverters._ +import scala.reflect.ClassTag /** * A type class providing evidence for parsing type `T` as a flag value. @@ -76,26 +78,56 @@ abstract class Flaggable[T] { */ object Flaggable { + /** + * Just like [[Flaggable]] but provides the runtime with the information about the type `T`. This + * can be useful for frameworks leveraging DI for Twitter Flags. + * + * @note You should prefer this interface for new Flaggables to allow them participate in DI (if + * needed). Eventually this interface may or may not submerge into the parent [[Flaggable]]. + */ + private[twitter] abstract class Typed[T] extends Flaggable[T] { + + /** + * A raw type for this Flaggable. + */ + def rawType: Type + } + + /** + * Just like [[Flaggable.Typed]] but signals the runtime that this Flaggable type is generic so it + * takes types as arguments. For example, `List[T]` is such type and it take one type-parameter + * `T` to construct `List[T]`. + */ + private[twitter] abstract class Generic[T] extends Typed[T] { + + /** + * An [[Array]] of Flaggables that map to type-parameters for this generic Flaggable. + */ + def parameters: Array[Flaggable[_]] + } + /** * Create a `Flaggable[T]` according to a `String => T` conversion function. * * @param f Function that parses a string into an object of type `T` */ - def mandatory[T](f: String => T): Flaggable[T] = new Flaggable[T] { + def mandatory[T](f: String => T)(implicit ct: ClassTag[T]): Flaggable[T] = new Typed[T] { def parse(s: String): T = f(s) + def rawType: Type = ct.runtimeClass } implicit val ofString: Flaggable[String] = mandatory(identity) - // Pairs of Flaggable conversions for types with corresponding Java boxed types. - implicit val ofBoolean: Flaggable[Boolean] = new Flaggable[Boolean] { + implicit val ofBoolean: Flaggable[Boolean] = new Typed[Boolean] { override def default: Option[Boolean] = Some(true) def parse(s: String): Boolean = s.toBoolean + def rawType: Type = classOf[Boolean] } - implicit val ofJavaBoolean: Flaggable[JBoolean] = new Flaggable[JBoolean] { + implicit val ofJavaBoolean: Flaggable[JBoolean] = new Typed[JBoolean] { override def default: Option[JBoolean] = Some(JBoolean.TRUE) def parse(s: String): JBoolean = JBoolean.valueOf(s.toBoolean) + def rawType: Type = classOf[JBoolean] } implicit val ofInt: Flaggable[Int] = mandatory(_.toInt) @@ -124,7 +156,7 @@ object Flaggable { implicit val ofJavaLocalTime: Flaggable[LocalTime] = mandatory(LocalTime.parse) implicit val ofInetSocketAddress: Flaggable[InetSocketAddress] = - new Flaggable[InetSocketAddress] { + new Typed[InetSocketAddress] { def parse(v: String): InetSocketAddress = v.split(":") match { case Array("", p) => new InetSocketAddress(p.toInt) @@ -141,14 +173,13 @@ object Flaggable { case _ => addr.getHostName }, addr.getPort) + + def rawType: Type = classOf[InetSocketAddress] } - implicit val ofFile: Flaggable[File] = new Flaggable[File] { - override def parse(v: String): File = new File(v) - override def show(file: File): String = file.toString - } + implicit val ofFile: Flaggable[File] = mandatory(s => new File(s)) - implicit def ofTuple[T: Flaggable, U: Flaggable]: Flaggable[(T, U)] = new Flaggable[(T, U)] { + implicit def ofTuple[T: Flaggable, U: Flaggable]: Flaggable[(T, U)] = new Generic[(T, U)] { private val tflag = implicitly[Flaggable[T]] private val uflag = implicitly[Flaggable[U]] @@ -164,6 +195,9 @@ object Flaggable { val (t, u) = tup tflag.show(t) + "," + uflag.show(u) } + + def parameters: Array[Flaggable[_]] = Array(tflag, uflag) + def rawType: Type = classOf[Tuple2[_, _]] } /** @@ -171,8 +205,8 @@ object Flaggable { * @param clazz The [[java.lang.Enum]] class. * @note [[java.lang.Enum]] enumeration constant value look up is case insensitive. */ - def ofJavaEnum[T <: Enum[T]](clazz: Class[T]): Flaggable[T] = - mandatory { s: String => + def ofJavaEnum[T <: Enum[T]](clazz: Class[T]): Flaggable[T] = new Typed[T] { + def parse(s: String): T = clazz.getEnumConstants.find(_.name.equalsIgnoreCase(s)) match { case Some(prop) => prop case _ => @@ -180,27 +214,35 @@ object Flaggable { s"The property $s does not belong to Java Enum ${clazz.getName}, the constants defined " + s"in the class are: ${clazz.getEnumConstants.mkString(",")}.") } - } - private[app] class SetFlaggable[T: Flaggable] extends Flaggable[Set[T]] { + def rawType: Type = clazz + } + + private[app] class SetFlaggable[T: Flaggable] extends Generic[Set[T]] { private val flag = implicitly[Flaggable[T]] assert(flag.default.isEmpty) override def parse(v: String): Set[T] = if (v.isEmpty) Set.empty[T] else v.split(",").iterator.map(flag.parse(_)).toSet override def show(set: Set[T]): String = set.map(flag.show).mkString(",") + + def parameters: Array[Flaggable[_]] = Array(flag) + def rawType: Type = classOf[Set[_]] } - private[app] class SeqFlaggable[T: Flaggable] extends Flaggable[Seq[T]] { + private[app] class SeqFlaggable[T: Flaggable] extends Generic[Seq[T]] { private val flag = implicitly[Flaggable[T]] assert(flag.default.isEmpty) def parse(v: String): Seq[T] = if (v.isEmpty) Seq.empty[T] else v.split(",").toSeq.map(flag.parse) override def show(seq: Seq[T]): String = seq.map(flag.show).mkString(",") + + override def parameters: Array[Flaggable[_]] = Array(flag) + override def rawType: Type = classOf[Seq[_]] } - private[app] class MapFlaggable[K: Flaggable, V: Flaggable] extends Flaggable[Map[K, V]] { + private[app] class MapFlaggable[K: Flaggable, V: Flaggable] extends Generic[Map[K, V]] { private val kflag = implicitly[Flaggable[K]] private val vflag = implicitly[Flaggable[V]] @@ -227,30 +269,40 @@ object Flaggable { override def show(out: Map[K, V]): String = { out.toSeq.map { case (k, v) => k.toString + "=" + v.toString }.mkString(",") } + + def parameters: Array[Flaggable[_]] = Array(kflag, vflag) + def rawType: Type = classOf[Map[_, _]] } implicit def ofSet[T: Flaggable]: Flaggable[Set[T]] = new SetFlaggable[T] implicit def ofSeq[T: Flaggable]: Flaggable[Seq[T]] = new SeqFlaggable[T] implicit def ofMap[K: Flaggable, V: Flaggable]: Flaggable[Map[K, V]] = new MapFlaggable[K, V] - implicit def ofJavaSet[T: Flaggable]: Flaggable[JSet[T]] = new Flaggable[JSet[T]] { + implicit def ofJavaSet[T: Flaggable]: Flaggable[JSet[T]] = new Generic[JSet[T]] { val setFlaggable = new SetFlaggable[T] override def parse(v: String): JSet[T] = setFlaggable.parse(v).asJava override def show(set: JSet[T]): String = setFlaggable.show(set.asScala.toSet) + + def parameters: Array[Flaggable[_]] = setFlaggable.parameters + def rawType: Type = classOf[JSet[_]] } - implicit def ofJavaList[T: Flaggable]: Flaggable[JList[T]] = new Flaggable[JList[T]] { + implicit def ofJavaList[T: Flaggable]: Flaggable[JList[T]] = new Generic[JList[T]] { val seqFlaggable = new SeqFlaggable[T] override def parse(v: String): JList[T] = seqFlaggable.parse(v).asJava override def show(list: JList[T]): String = seqFlaggable.show(list.asScala.toSeq) - } - implicit def ofJavaMap[K: Flaggable, V: Flaggable]: Flaggable[JMap[K, V]] = { - val mapFlaggable = new MapFlaggable[K, V] + def parameters: Array[Flaggable[_]] = seqFlaggable.parameters + def rawType: Type = classOf[JList[_]] + } - new Flaggable[JMap[K, V]] { + implicit def ofJavaMap[K: Flaggable, V: Flaggable]: Flaggable[JMap[K, V]] = + new Generic[JMap[K, V]] { + val mapFlaggable = new MapFlaggable[K, V] def parse(in: String): JMap[K, V] = mapFlaggable.parse(in).asJava override def show(out: JMap[K, V]): String = mapFlaggable.show(out.asScala.toMap) + + def parameters: Array[Flaggable[_]] = mapFlaggable.parameters + def rawType: Type = classOf[JMap[_, _]] } - } } diff --git a/util-app/src/test/java/com/twitter/app/JavaFlagTest.java b/util-app/src/test/java/com/twitter/app/JavaFlagTest.java index bdbfc93d72..38d8fe5ac4 100644 --- a/util-app/src/test/java/com/twitter/app/JavaFlagTest.java +++ b/util-app/src/test/java/com/twitter/app/JavaFlagTest.java @@ -6,6 +6,8 @@ import java.util.List; import java.util.Map; +import scala.reflect.ClassTag$; + import org.junit.Test; import com.twitter.util.Duration; @@ -32,7 +34,7 @@ static class Named { @Override public Named apply(String n) { return new Named(n); } - }); + }, ClassTag$.MODULE$.apply(Named.class)); Named(String name) { this.name = name; diff --git a/util-app/src/test/scala/com/twitter/app/FlagTest.scala b/util-app/src/test/scala/com/twitter/app/FlagTest.scala index b40eab67e5..940c73ff3a 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagTest.scala @@ -12,6 +12,7 @@ class FlagTest extends AnyFunSuite { val flag = new Flags("test", includeGlobal = false, failFastUntilParsed) val fooFlag = flag("foo", 123, "The foo value") val barFlag = flag("bar", "okay", "The bar value") + val bazFlag = flag[String]("baz", "The baz value") } test("Flag: defaults") { @@ -21,6 +22,11 @@ class FlagTest extends AnyFunSuite { flag.finishParsing() assert(fooFlag() == 123) assert(barFlag() == "okay") + intercept[IllegalArgumentException](bazFlag()) + + assert(fooFlag.getDefault == Some(123)) + assert(barFlag.getDefault == Some("okay")) + assert(bazFlag.getDefault.isEmpty) } test("Flag: override a flag") { diff --git a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala index 2000d8eb1a..b55cb06a3e 100644 --- a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala @@ -77,4 +77,12 @@ class FlaggableTest extends AnyFunSuite { assert(Flaggable.ofTuple[Int, String].parse("1,hello") == ((1, "hello"))) intercept[IllegalArgumentException] { Flaggable.ofTuple[Int, String].parse("1") } } + + test("Flaggable: expose types and kinds") { + assert(Flaggable.ofFile.isInstanceOf[Flaggable.Typed[_]]) + assert(Flaggable.ofJavaBoolean.isInstanceOf[Flaggable.Typed[_]]) + + assert(Flaggable.ofSeq[Int].isInstanceOf[Flaggable.Generic[_]]) + assert(Flaggable.ofJavaSet[Int].isInstanceOf[Flaggable.Generic[_]]) + } } diff --git a/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala b/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala index 4e17c209fc..1511fa5556 100644 --- a/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala +++ b/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala @@ -43,10 +43,8 @@ private[inject] trait TwitterModuleFlags { help: String, flaggable: Flaggable[T] ): Flag[T] = { - implicit val implicitFlaggable: Flaggable[T] = flaggable - val flag = new Flag[T](name, help, default, failFastUntilParsed = failfastOnFlagsNotParsed) { - override protected val flaggable: Flaggable[T] = implicitFlaggable - } + val flag = + new Flag[T](name, help, default, failFastUntilParsed = failfastOnFlagsNotParsed)(flaggable) flags += flag flag } @@ -66,10 +64,8 @@ private[inject] trait TwitterModuleFlags { usage: String, flaggable: Flaggable[T] ): Flag[T] = { - implicit val implicitFlaggable: Flaggable[T] = flaggable - val flag = new Flag[T](name, help, usage, failFastUntilParsed = failfastOnFlagsNotParsed) { - override protected val flaggable: Flaggable[T] = implicitFlaggable - } + val flag = + new Flag[T](name, help, usage, failFastUntilParsed = failfastOnFlagsNotParsed)(flaggable) flags += flag flag } From a7444194918b4d5d69c099414628cf55b469300e Mon Sep 17 00:00:00 2001 From: Ekaterina Tyurina Date: Wed, 2 Sep 2020 04:06:52 +0000 Subject: [PATCH 381/796] [csl targets] explicitly list out the runtime scoped dependencies **Problem and Solution** Pants can specify the runtime scope at dep level so it is implicit at the dependee level, whereas bazel cannot, so this change makes the runtime deps explicit for bazel to understand. JIRA Issues: DPB-12421 Differential Revision: https://phabricator.twitter.biz/D529617 --- util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD | 5 ++++- util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD index 72acca2533..4b72642c0c 100644 --- a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD @@ -5,7 +5,10 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/slf4j:slf4j-api", - "3rdparty/jvm/org/slf4j:slf4j-simple", + scoped( + "3rdparty/jvm/org/slf4j:slf4j-simple", + scope = "runtime", + ), "util/util-slf4j-api/src/main/scala", ], ) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index 6a2230fad2..52a7814f83 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -27,6 +27,9 @@ junit_tests( "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "3rdparty/jvm/org/scalatestplus:mockito-3-2", - "3rdparty/jvm/org/slf4j:slf4j-simple", + scoped( + "3rdparty/jvm/org/slf4j:slf4j-simple", + scope = "runtime", + ), ], ) From d6dd4ef8069a2ff5379460a87672bfe6eb6e4dba Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Fri, 4 Sep 2020 17:03:56 +0000 Subject: [PATCH 382/796] finagle: configure SourceRole for finagle Clients and Servers Problem ------- Metric Metadata contains a field for the Role the service is acting as with respect to that metric, but nothing was populating it. Solution -------- Start populating the Client/Server role metadata in finagle. Result ------ Client and Server related metrics created by finagle now have the role metadata populated. JIRA Issues: CSL-9958 Differential Revision: https://phabricator.twitter.biz/D542596 --- .../stats/RoleConfiguredStatsReceiver.scala | 26 +++++++++++++ .../RoleConfiguredStatsReceiverTest.scala | 38 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala create mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala new file mode 100644 index 0000000000..65d9d76f52 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala @@ -0,0 +1,26 @@ +package com.twitter.finagle.stats + +/** + * A StatsReceiver proxy that configures all counter, stat, and gauge + * SourceRoles to the passed in "role". + * + * @param self The underlying StatsReceiver to which translated names are passed + * @param role the role used for SourceRole Metadata + */ +case class RoleConfiguredStatsReceiver( + protected val self: StatsReceiver, + val role: SourceRole) + extends StatsReceiverProxy { + + override def counter(counterSchema: CounterSchema): Counter = { + self.counter(CounterSchema(counterSchema.metricBuilder.withRole(role))) + } + + override def stat(histogramSchema: HistogramSchema): Stat = { + self.stat(HistogramSchema(histogramSchema.metricBuilder.withRole(role))) + } + + override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { + self.addGauge(GaugeSchema(gaugeSchema.metricBuilder.withRole(role)))(f) + } +} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala new file mode 100644 index 0000000000..b52585d438 --- /dev/null +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala @@ -0,0 +1,38 @@ +package com.twitter.finagle.stats + +import org.scalatest.FunSuite + +class RoleConfiguredStatsReceiverTest extends FunSuite { + test("RoleConfiguredSR configures metadata role correctly") { + val mem = new InMemoryStatsReceiver + val clientSR = new RoleConfiguredStatsReceiver(mem, Client) + val serverSR = new RoleConfiguredStatsReceiver(mem, Server) + + clientSR.counter("foo") + assert(mem.schemas.get(Seq("foo")).get.metricBuilder.role == Client) + clientSR.addGauge("bar")(3) + assert(mem.schemas.get(Seq("bar")).get.metricBuilder.role == Client) + clientSR.stat("baz") + assert(mem.schemas.get(Seq("baz")).get.metricBuilder.role == Client) + + serverSR.counter("fee") + assert(mem.schemas.get(Seq("fee")).get.metricBuilder.role == Server) + serverSR.addGauge("fi")(3) + assert(mem.schemas.get(Seq("fi")).get.metricBuilder.role == Server) + serverSR.stat("foe") + assert(mem.schemas.get(Seq("foe")).get.metricBuilder.role == Server) + } + + test("RoleConfiguredSR equality") { + val mem = new InMemoryStatsReceiver + val clientA = new RoleConfiguredStatsReceiver(mem, Client) + val clientB = new RoleConfiguredStatsReceiver(mem, Client) + val serverA = new RoleConfiguredStatsReceiver(mem, Server) + val serverB = new RoleConfiguredStatsReceiver(mem, Server) + + assert(clientA == clientB) + assert(clientA != serverA) + assert(clientA != serverB) + assert(serverA == serverB) + } +} From f25c097b27408be8fe47ea3d6f186242e0121742 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 10 Sep 2020 18:06:25 +0000 Subject: [PATCH 383/796] util-slf4j-api: Update test mocking Problem The util-slf4j-api tests are using some deprecated Mockito matchers. Solution Lets update away from MockitoSugar to Mockito Scala while updating the deprecated Matchers to their new versions. Result No deprecation warnings and new Mockito Scala based mocking in tests. Differential Revision: https://phabricator.twitter.biz/D547187 --- build.sbt | 5 +- .../test/scala/com/twitter/util/logging/BUILD | 4 +- .../twitter/util/logging/LoggingTest.scala | 118 ++++++----- .../util/logging/MarkerLoggingTest.scala | 194 +++++++++--------- 4 files changed, 157 insertions(+), 164 deletions(-) diff --git a/build.sbt b/build.sbt index 0bd6fc0351..b3e9bb8567 100644 --- a/build.sbt +++ b/build.sbt @@ -464,11 +464,10 @@ lazy val utilSlf4jApi = Project( name := "util-slf4j-api", libraryDependencies ++= Seq( slf4jApi, - "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test", + "org.mockito" %% "mockito-scala" % mockitoScalaVersion % "test", "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" ) - ).dependsOn(utilCore % "test") + ).dependsOn(utilCore % "test", utilMock % "test") lazy val utilSlf4jJulBridge = Project( id = "util-slf4j-jul-bridge", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index 52a7814f83..b23a6c4266 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -23,13 +23,13 @@ junit_tests( compiler_option_sets = ["fatal_warnings"], dependencies = [ ":testutil", - "3rdparty/jvm/org/mockito:mockito-core", + "3rdparty/jvm/org/mockito:mockito-scala", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", scoped( "3rdparty/jvm/org/slf4j:slf4j-simple", scope = "runtime", ), + "util/util-mock/src/main/scala/com/twitter/util/mock", ], ) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala index 986e965fd2..95fd35b4f5 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala @@ -1,14 +1,12 @@ package com.twitter.util.logging -import org.mockito.ArgumentMatchers._ -import org.mockito.Mockito._ -import org.scalatestplus.mockito.MockitoSugar -import org.slf4j -import scala.language.reflectiveCalls +import com.twitter.util.mock.Mockito +import org.mockito.Mockito.when import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import scala.language.reflectiveCalls -class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { +class LoggingTest extends AnyFunSuite with Matchers with Mockito { /* Trace */ @@ -16,50 +14,50 @@ class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { val f = fixture(_.isTraceEnabled, isEnabled = true) f.logger.trace(f.message) - verify(f.underlying).trace(f.message) + f.underlying.trace(f.message) was called } test("Logging#trace not enabled") { val f = fixture(_.isTraceEnabled, isEnabled = false) f.logger.trace(f.message) - verify(f.underlying, never).trace(anyString) + f.underlying.trace(any[String]) wasNever called } test("Logging#trace enabled with message and cause") { val f = fixture(_.isTraceEnabled, isEnabled = true) f.logger.trace(f.message, f.cause) - verify(f.underlying).trace(f.message, f.cause) + f.underlying.trace(f.message, f.cause) was called } test("Logging#trace not enabled with message and cause") { val f = fixture(_.isTraceEnabled, isEnabled = false) f.logger.trace(f.message, f.cause) - verify(f.underlying, never).trace(anyString, anyObject) + f.underlying.trace(any[String], any) wasNever called } test("Logging#trace enabled with parameters") { val f = fixture(_.isTraceEnabled, isEnabled = true) f.logger.traceWith(f.message, f.arg1) - verify(f.underlying).trace(f.message, Seq(f.arg1): _*) + f.underlying.trace(f.message, Seq(f.arg1): _*) was called f.logger.traceWith(f.message, f.arg1, f.arg2) - verify(f.underlying).trace(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.trace(f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.traceWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).trace(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.trace(f.message, f.arg1, f.arg2, f.arg3) was called } test("Logging#trace not enabled with parameters") { val f = fixture(_.isTraceEnabled, isEnabled = false) f.logger.traceWith(f.message, f.arg1) - verify(f.underlying, never).trace(f.message, Seq(f.arg1): _*) + f.underlying.trace(f.message, Seq(f.arg1): _*) wasNever called f.logger.traceWith(f.message, f.arg1, f.arg2) - verify(f.underlying, never).trace(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.trace(f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.traceWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).trace(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.trace(f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Debug */ @@ -68,50 +66,50 @@ class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { val f = fixture(_.isDebugEnabled, isEnabled = true) f.logger.debug(f.message) - verify(f.underlying).debug(f.message) + f.underlying.debug(f.message) was called } test("Logging#debug not enabled") { val f = fixture(_.isDebugEnabled, isEnabled = false) f.logger.debug(f.message) - verify(f.underlying, never).debug(anyString) + f.underlying.debug(any[String]) wasNever called } test("Logging#debug enabled with message and cause") { val f = fixture(_.isDebugEnabled, isEnabled = true) f.logger.debug(f.message, f.cause) - verify(f.underlying).debug(f.message, f.cause) + f.underlying.debug(f.message, f.cause) was called } test("Logging#debug not enabled with message and cause") { val f = fixture(_.isDebugEnabled, isEnabled = false) f.logger.debug(f.message, f.cause) - verify(f.underlying, never).debug(anyString, anyObject) + f.underlying.debug(any[String], any) wasNever called } test("Logging#debug enabled with parameters") { val f = fixture(_.isDebugEnabled, isEnabled = true) f.logger.debugWith(f.message, f.arg1) - verify(f.underlying).debug(f.message, Seq(f.arg1): _*) + f.underlying.debug(f.message, Seq(f.arg1): _*) was called f.logger.debugWith(f.message, f.arg1, f.arg2) - verify(f.underlying).debug(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.debug(f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.debugWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).debug(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.debug(f.message, f.arg1, f.arg2, f.arg3) was called } test("Logging#debug not enabled with parameters") { val f = fixture(_.isDebugEnabled, isEnabled = false) f.logger.debugWith(f.message, f.arg1) - verify(f.underlying, never).debug(f.message, Seq(f.arg1): _*) + f.underlying.debug(f.message, Seq(f.arg1): _*) wasNever called f.logger.debugWith(f.message, f.arg1, f.arg2) - verify(f.underlying, never).debug(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.debug(f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.debugWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).debug(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.debug(f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Info */ @@ -120,50 +118,50 @@ class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { val f = fixture(_.isInfoEnabled, isEnabled = true) f.logger.info(f.message) - verify(f.underlying).info(f.message) + f.underlying.info(f.message) was called } test("Logging#info not enabled") { val f = fixture(_.isInfoEnabled, isEnabled = false) f.logger.info(f.message) - verify(f.underlying, never).info(anyString) + f.underlying.info(any[String]) wasNever called } test("Logging#info enabled with message and cause") { val f = fixture(_.isInfoEnabled, isEnabled = true) f.logger.info(f.message, f.cause) - verify(f.underlying).info(f.message, f.cause) + f.underlying.info(f.message, f.cause) was called } test("Logging#info not enabled with message and cause") { val f = fixture(_.isInfoEnabled, isEnabled = false) f.logger.info(f.message, f.cause) - verify(f.underlying, never).info(anyString, anyObject) + f.underlying.info(any[String], any) wasNever called } test("Logging#info enabled with parameters") { val f = fixture(_.isInfoEnabled, isEnabled = true) f.logger.infoWith(f.message, f.arg1) - verify(f.underlying).info(f.message, Seq(f.arg1): _*) + f.underlying.info(f.message, Seq(f.arg1): _*) was called f.logger.infoWith(f.message, f.arg1, f.arg2) - verify(f.underlying).info(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.info(f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.infoWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).info(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.info(f.message, f.arg1, f.arg2, f.arg3) was called } test("Logging#info not enabled with parameters") { val f = fixture(_.isInfoEnabled, isEnabled = false) f.logger.infoWith(f.message, f.arg1) - verify(f.underlying, never).info(f.message, Seq(f.arg1): _*) + f.underlying.info(f.message, Seq(f.arg1): _*) wasNever called f.logger.infoWith(f.message, f.arg1, f.arg2) - verify(f.underlying, never).info(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.info(f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.infoWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).info(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.info(f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Warn */ @@ -172,50 +170,50 @@ class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { val f = fixture(_.isWarnEnabled, isEnabled = true) f.logger.warn(f.message) - verify(f.underlying).warn(f.message) + f.underlying.warn(f.message) was called } test("Logging#warn not enabled") { val f = fixture(_.isWarnEnabled, isEnabled = false) f.logger.warn(f.message) - verify(f.underlying, never).warn(anyString) + f.underlying.warn(any[String]) wasNever called } test("Logging#warn enabled with message and cause") { val f = fixture(_.isWarnEnabled, isEnabled = true) f.logger.warn(f.message, f.cause) - verify(f.underlying).warn(f.message, f.cause) + f.underlying.warn(f.message, f.cause) was called } test("Logging#warn not enabled with message and cause") { val f = fixture(_.isWarnEnabled, isEnabled = false) f.logger.warn(f.message, f.cause) - verify(f.underlying, never).warn(anyString, anyObject) + f.underlying.warn(any[String], any) wasNever called } test("Logging#warn enabled with parameters") { val f = fixture(_.isWarnEnabled, isEnabled = true) f.logger.warnWith(f.message, f.arg1) - verify(f.underlying).warn(f.message, Seq(f.arg1): _*) + f.underlying.warn(f.message, Seq(f.arg1): _*) was called f.logger.warnWith(f.message, f.arg1, f.arg2) - verify(f.underlying).warn(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.warn(f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.warnWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).warn(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.warn(f.message, f.arg1, f.arg2, f.arg3) was called } test("Logging#warn not enabled with parameters") { val f = fixture(_.isWarnEnabled, isEnabled = false) f.logger.warnWith(f.message, f.arg1) - verify(f.underlying, never).warn(f.message, Seq(f.arg1): _*) + f.underlying.warn(f.message, Seq(f.arg1): _*) wasNever called f.logger.warnWith(f.message, f.arg1, f.arg2) - verify(f.underlying, never).warn(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.warn(f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.warnWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).warn(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.warn(f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Error */ @@ -224,50 +222,50 @@ class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { val f = fixture(_.isErrorEnabled, isEnabled = true) f.logger.error(f.message) - verify(f.underlying).error(f.message) + f.underlying.error(f.message) was called } test("Logging#error not enabled") { val f = fixture(_.isErrorEnabled, isEnabled = false) f.logger.error(f.message) - verify(f.underlying, never).error(anyString) + f.underlying.error(any[String]) wasNever called } test("Logging#error enabled with message and cause") { val f = fixture(_.isErrorEnabled, isEnabled = true) f.logger.error(f.message, f.cause) - verify(f.underlying).error(f.message, f.cause) + f.underlying.error(f.message, f.cause) was called } test("Logging#error not enabled with message and cause") { val f = fixture(_.isErrorEnabled, isEnabled = false) f.logger.error(f.message, f.cause) - verify(f.underlying, never).error(anyString, anyObject) + f.underlying.error(any[String], any) wasNever called } test("Logging#error enabled with parameters") { val f = fixture(_.isErrorEnabled, isEnabled = true) f.logger.errorWith(f.message, f.arg1) - verify(f.underlying).error(f.message, Seq(f.arg1): _*) + f.underlying.error(f.message, Seq(f.arg1): _*) was called f.logger.errorWith(f.message, f.arg1, f.arg2) - verify(f.underlying).error(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.error(f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.errorWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).error(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.error(f.message, f.arg1, f.arg2, f.arg3) was called } test("Logging#error not enabled with parameters") { val f = fixture(_.isErrorEnabled, isEnabled = false) f.logger.errorWith(f.message, f.arg1) - verify(f.underlying, never).error(f.message, Seq(f.arg1): _*) + f.underlying.error(f.message, Seq(f.arg1): _*) wasNever called f.logger.errorWith(f.message, f.arg1, f.arg2) - verify(f.underlying, never).error(f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.error(f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.errorWith(f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).error(f.message, f.arg1, f.arg2, f.arg3) + f.underlying.error(f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Instance Logging */ @@ -318,7 +316,7 @@ class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { } test("Logging#null values") { - val logger = Logger(slf4j.LoggerFactory.getLogger(this.getClass)) + val logger = Logger(org.slf4j.LoggerFactory.getLogger(this.getClass)) logger.info(null) val list = Nil logger.info(list.toString) @@ -350,16 +348,16 @@ class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { /* Private */ - private def fixture(isEnabledFn: slf4j.Logger => Boolean, isEnabled: Boolean) = { + private def fixture(isEnabledFn: org.slf4j.Logger => Boolean, isEnabled: Boolean) = { new { val message = "msg" val cause = new RuntimeException("TEST EXCEPTION") val arg1 = "arg1" val arg2 = new Integer(1) val arg3 = "arg3" - val underlying = mock[slf4j.Logger] + val underlying: org.slf4j.Logger = mock[org.slf4j.Logger] when(isEnabledFn(underlying)).thenReturn(isEnabled) - val logger = Logger(underlying) + val logger: Logger = Logger(underlying) } } } diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala index 287556ffda..91689745e9 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala @@ -1,288 +1,284 @@ package com.twitter.util.logging -import org.mockito.ArgumentMatchers._ -import org.mockito.Mockito._ -import org.scalatestplus.mockito.MockitoSugar -import org.slf4j -import org.slf4j.Marker -import scala.language.reflectiveCalls +import com.twitter.util.mock.Mockito +import org.mockito.Mockito.when import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import org.slf4j.Marker +import scala.language.reflectiveCalls -class MarkerLoggingTest extends AnyFunSuite with Matchers with MockitoSugar { +class MarkerLoggingTest extends AnyFunSuite with Matchers with Mockito { test("Marker Logging#trace enabled") { - val f = fixture(_.isTraceEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = true) f.logger.trace(f.marker, f.message) - verify(f.underlying).trace(f.marker, f.message) + f.underlying.trace(f.marker, f.message) was called } test("Marker Logging#trace not enabled") { - val f = fixture(_.isTraceEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = false) f.logger.trace(f.marker, f.message) - verify(f.underlying, never).trace(anyObject[Marker], anyString) + f.underlying.trace(any[Marker], any[String]) wasNever called } test("Marker Logging#trace enabled with message and cause") { - val f = fixture(_.isTraceEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = true) f.logger.trace(f.marker, f.message, f.cause) - verify(f.underlying).trace(f.marker, f.message, f.cause) + f.underlying.trace(f.marker, f.message, f.cause) was called } test("Marker Logging#trace not enabled with message and cause") { - val f = fixture(_.isTraceEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = false) f.logger.trace(f.marker, f.message, f.cause) - verify(f.underlying, never).trace(anyObject[Marker], anyString, anyObject) + f.underlying.trace(any[Marker], any[String], any) wasNever called } test("Marker Logging#trace enabled with parameters") { - val f = fixture(_.isTraceEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = true) f.logger.traceWith(f.marker, f.message, f.arg1) - verify(f.underlying).trace(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.trace(f.marker, f.message, Seq(f.arg1): _*) was called f.logger.traceWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying).trace(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.trace(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.traceWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).trace(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.trace(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called } test("Marker Logging#trace not enabled with parameters") { - val f = fixture(_.isTraceEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = false) f.logger.traceWith(f.marker, f.message, f.arg1) - verify(f.underlying, never).trace(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.trace(f.marker, f.message, Seq(f.arg1): _*) wasNever called f.logger.traceWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying, never).trace(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.trace(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.traceWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).trace(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.trace(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Debug */ test("Marker Logging#debug enabled") { - val f = fixture(_.isDebugEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = true) f.logger.debug(f.marker, f.message) - verify(f.underlying).debug(f.marker, f.message) + f.underlying.debug(f.marker, f.message) was called } test("Marker Logging#debug not enabled") { - val f = fixture(_.isDebugEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = false) f.logger.debug(f.marker, f.message) - verify(f.underlying, never).debug(anyObject[Marker], anyString) + f.underlying.debug(any[Marker], any[String]) wasNever called } test("Marker Logging#debug enabled with message and cause") { - val f = fixture(_.isDebugEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = true) f.logger.debug(f.marker, f.message, f.cause) - verify(f.underlying).debug(f.marker, f.message, f.cause) + f.underlying.debug(f.marker, f.message, f.cause) was called } test("Marker Logging#debug not enabled with message and cause") { - val f = fixture(_.isDebugEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = false) f.logger.debug(f.marker, f.message, f.cause) - verify(f.underlying, never).debug(anyObject[Marker], anyString, anyObject) + f.underlying.debug(any[Marker], any[String], any) wasNever called } test("Marker Logging#debug enabled with parameters") { - val f = fixture(_.isDebugEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = true) f.logger.debugWith(f.marker, f.message, f.arg1) - verify(f.underlying).debug(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.debug(f.marker, f.message, Seq(f.arg1): _*) was called f.logger.debugWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying).debug(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.debug(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.debugWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).debug(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.debug(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called } test("Marker Logging#debug not enabled with parameters") { - val f = fixture(_.isDebugEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = false) f.logger.debugWith(f.marker, f.message, f.arg1) - verify(f.underlying, never).debug(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.debug(f.marker, f.message, Seq(f.arg1): _*) wasNever called f.logger.debugWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying, never).debug(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.debug(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.debugWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).debug(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.debug(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Info */ test("Marker Logging#info enabled") { - val f = fixture(_.isInfoEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = true) f.logger.info(f.marker, f.message) - verify(f.underlying).info(f.marker, f.message) + f.underlying.info(f.marker, f.message) was called } test("Marker Logging#info not enabled") { - val f = fixture(_.isInfoEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = false) f.logger.info(f.marker, f.message) - verify(f.underlying, never).info(anyObject[Marker], anyString) + f.underlying.info(any[Marker], any[String]) wasNever called } test("Marker Logging#info enabled with message and cause") { - val f = fixture(_.isInfoEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = true) f.logger.info(f.marker, f.message, f.cause) - verify(f.underlying).info(f.marker, f.message, f.cause) + f.underlying.info(f.marker, f.message, f.cause) was called } test("Marker Logging#info not enabled with message and cause") { - val f = fixture(_.isInfoEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = false) f.logger.info(f.marker, f.message, f.cause) - verify(f.underlying, never).info(anyObject[Marker], anyString, anyObject) + f.underlying.info(any[Marker], any[String], any) wasNever called } test("Marker Logging#info enabled with parameters") { - val f = fixture(_.isInfoEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = true) f.logger.infoWith(f.marker, f.message, f.arg1) - verify(f.underlying).info(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.info(f.marker, f.message, Seq(f.arg1): _*) was called f.logger.infoWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying).info(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.info(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.infoWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).info(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.info(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called } test("Marker Logging#info not enabled with parameters") { - val f = fixture(_.isInfoEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = false) f.logger.infoWith(f.marker, f.message, f.arg1) - verify(f.underlying, never).info(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.info(f.marker, f.message, Seq(f.arg1): _*) wasNever called f.logger.infoWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying, never).info(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.info(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.infoWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).info(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.info(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Warn */ test("Marker Logging#warn enabled") { - val f = fixture(_.isWarnEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = true) f.logger.warn(f.marker, f.message) - verify(f.underlying).warn(f.marker, f.message) + f.underlying.warn(f.marker, f.message) was called } test("Marker Logging#warn not enabled") { - val f = fixture(_.isWarnEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = false) f.logger.warn(f.marker, f.message) - verify(f.underlying, never).warn(anyObject[Marker], anyString) + f.underlying.warn(any[Marker], any[String]) wasNever called } test("Marker Logging#warn enabled with message and cause") { - val f = fixture(_.isWarnEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = true) f.logger.warn(f.marker, f.message, f.cause) - verify(f.underlying).warn(f.marker, f.message, f.cause) + f.underlying.warn(f.marker, f.message, f.cause) was called } test("Marker Logging#warn not enabled with message and cause") { - val f = fixture(_.isWarnEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = false) f.logger.warn(f.marker, f.message, f.cause) - verify(f.underlying, never).warn(anyObject[Marker], anyString, anyObject) + f.underlying.warn(any[Marker], any[String], any) wasNever called } test("Marker Logging#warn enabled with parameters") { - val f = fixture(_.isWarnEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = true) f.logger.warnWith(f.marker, f.message, f.arg1) - verify(f.underlying).warn(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.warn(f.marker, f.message, Seq(f.arg1): _*) was called f.logger.warnWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying).warn(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.warn(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.warnWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).warn(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.warn(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called } test("Marker Logging#warn not enabled with parameters") { - val f = fixture(_.isWarnEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = false) f.logger.warnWith(f.marker, f.message, f.arg1) - verify(f.underlying, never).warn(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.warn(f.marker, f.message, Seq(f.arg1): _*) wasNever called f.logger.warnWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying, never).warn(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.warn(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.warnWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).warn(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.warn(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Error */ test("Marker Logging#error enabled") { - val f = fixture(_.isErrorEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = true) f.logger.error(f.marker, f.message) - verify(f.underlying).error(f.marker, f.message) + f.underlying.error(f.marker, f.message) was called } test("Marker Logging#error not enabled") { - val f = fixture(_.isErrorEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = false) f.logger.error(f.marker, f.message) - verify(f.underlying, never).error(anyObject[Marker], anyString) + f.underlying.error(any[Marker], any[String]) wasNever called } test("Marker Logging#error enabled with message and cause") { - val f = fixture(_.isErrorEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = true) f.logger.error(f.marker, f.message, f.cause) - verify(f.underlying).error(f.marker, f.message, f.cause) + f.underlying.error(f.marker, f.message, f.cause) was called } test("Marker Logging#error not enabled with message and cause") { - val f = fixture(_.isErrorEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = false) f.logger.error(f.marker, f.message, f.cause) - verify(f.underlying, never).error(anyObject[Marker], anyString, anyObject) + f.underlying.error(any[Marker], any[String], any) wasNever called } test("Marker Logging#error enabled with parameters") { - val f = fixture(_.isErrorEnabled(anyObject[Marker]), isEnabled = true) + val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = true) f.logger.errorWith(f.marker, f.message, f.arg1) - verify(f.underlying).error(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.error(f.marker, f.message, Seq(f.arg1): _*) was called f.logger.errorWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying).error(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.error(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called f.logger.errorWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying).error(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.error(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called } test("Marker Logging#error not enabled with parameters") { - val f = fixture(_.isErrorEnabled(anyObject[Marker]), isEnabled = false) + val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = false) f.logger.errorWith(f.marker, f.message, f.arg1) - verify(f.underlying, never).error(f.marker, f.message, Seq(f.arg1): _*) + f.underlying.error(f.marker, f.message, Seq(f.arg1): _*) wasNever called f.logger.errorWith(f.marker, f.message, f.arg1, f.arg2) - verify(f.underlying, never).error(f.marker, f.message, Seq(f.arg1, f.arg2): _*) + f.underlying.error(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called f.logger.errorWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - verify(f.underlying, never).error(f.marker, f.message, f.arg1, f.arg2, f.arg3) + f.underlying.error(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called } /* Private */ - private def fixture(p: slf4j.Logger => Boolean, isEnabled: Boolean) = { - new { - val marker: Marker = TestMarker - val message = "msg" - val cause = new RuntimeException("TEST EXCEPTION") - val arg1 = "arg1" - val arg2 = new Integer(1) - val arg3 = "arg3" - val underlying: org.slf4j.Logger = mock[org.slf4j.Logger] - when(p(underlying)).thenReturn(isEnabled) - val logger = Logger(underlying) - } + private def fixture(p: org.slf4j.Logger => Boolean, isEnabled: Boolean) = new { + val marker: Marker = TestMarker + val message = "msg" + val cause = new RuntimeException("TEST EXCEPTION") + val arg1 = "arg1" + val arg2 = new Integer(1) + val arg3 = "arg3" + val underlying: org.slf4j.Logger = mock[org.slf4j.Logger] + when(p(underlying)).thenReturn(isEnabled) + val logger: Logger = Logger(underlying) } object TestMarker extends Marker { From e4f54d3c18b0ddc864a36b90ecbcbd7f1ed6b846 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Wed, 16 Sep 2020 05:26:11 +0000 Subject: [PATCH 384/796] util-app: Allow Boolean in Tuple/Seq/Map flags Problem Boolean flags are the only type of flags for which the argument value isn't required. This means, `-foo` and `-foo=true` are equivalent. Due to a historical accident, boolean flags were prohibited to participate in more complex flag types, specifically `Seq[Boolean]`, `Tuple2[Bollean, Boolean]` and `Map[Boolean, Boolean]`. Although the reasons for such a limitation are unknown, our best explanation is that boolean flags (or any other flags with optional values) might have been messing up with the legacy parsing logic within, for instance, comma-separated flags. Solution There seem to be no reason to exclude boolean flags from participating in more complex flag structures: `Seq`, `Tuple2`, and `Map`. JIRA Issues: CSL-10169 Differential Revision: https://phabricator.twitter.biz/D549196 --- CHANGELOG.rst | 7 +++++++ .../main/scala/com/twitter/app/Flaggable.scala | 10 +--------- .../scala/com/twitter/app/FlaggableTest.scala | 15 +++++++-------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6177a85454..7cd7bf2f88 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-app: `Seq`/`Tuple2`/`Map` flags can now operate on booleans. For example, + `Flag[Seq[Boolean]]` now works as expected instead of throwing an assert exception (previous + behaviour). ``PHAB_ID=D549196`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index ce6327ba8a..a1957509b1 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -183,9 +183,6 @@ object Flaggable { private val tflag = implicitly[Flaggable[T]] private val uflag = implicitly[Flaggable[U]] - assert(tflag.default.isEmpty) - assert(uflag.default.isEmpty) - def parse(v: String): (T, U) = v.split(",") match { case Array(t, u) => (tflag.parse(t), uflag.parse(u)) case _ => throw new IllegalArgumentException("not a 't,u'") @@ -220,8 +217,7 @@ object Flaggable { private[app] class SetFlaggable[T: Flaggable] extends Generic[Set[T]] { private val flag = implicitly[Flaggable[T]] - assert(flag.default.isEmpty) - override def parse(v: String): Set[T] = + def parse(v: String): Set[T] = if (v.isEmpty) Set.empty[T] else v.split(",").iterator.map(flag.parse(_)).toSet override def show(set: Set[T]): String = set.map(flag.show).mkString(",") @@ -232,7 +228,6 @@ object Flaggable { private[app] class SeqFlaggable[T: Flaggable] extends Generic[Seq[T]] { private val flag = implicitly[Flaggable[T]] - assert(flag.default.isEmpty) def parse(v: String): Seq[T] = if (v.isEmpty) Seq.empty[T] else v.split(",").toSeq.map(flag.parse) @@ -246,9 +241,6 @@ object Flaggable { private val kflag = implicitly[Flaggable[K]] private val vflag = implicitly[Flaggable[V]] - assert(kflag.default.isEmpty) - assert(vflag.default.isEmpty) - def parse(in: String): Map[K, V] = { val tuples = in.split(',').foldLeft(Seq.empty[String]) { case (acc, s) if !s.contains('=') => diff --git a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala index b55cb06a3e..417af43736 100644 --- a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala @@ -48,33 +48,32 @@ class FlaggableTest extends AnyFunSuite { test("Flaggable: parse seqs") { assert(Flaggable.ofSeq[Int].parse("1,2,3,4") == Seq(1, 2, 3, 4)) assert(Flaggable.ofSeq[Int].parse("") == Seq.empty[Int]) + assert(Flaggable.ofSeq[Boolean].parse("true,false,true") == Seq(true, false, true)) } test("Flaggable: parse sets") { assert(Flaggable.ofSet[Int].parse("1,2,3,4") == Set(1, 2, 3, 4)) assert(Flaggable.ofSet[Int].parse("") == Set.empty[Int]) + assert(Flaggable.ofSeq[Boolean].parse("true,false,true") == Seq(true, false, true)) } test("Flaggable: parse maps") { assert(Flaggable.ofMap[Int, Int].parse("1=2,3=4") == Map(1 -> 2, 3 -> 4)) + assert( + Flaggable + .ofMap[String, Boolean].parse("foo=true,bar=false") == Map("foo" -> true, "bar" -> false)) } test("Flaggable: parse maps with comma-separated values") { assert( - Flaggable.ofMap[String, Seq[Int]].parse("a=1,2,3,3,b=4,5") == + Flaggable.ofMap[String, Seq[Int]].parse("a=1,2,3,3,b=4,5") == // 1000191039 Map("a" -> Seq(1, 2, 3, 3), "b" -> Seq(4, 5)) ) } - test("Flaggable: parse maps of sets with comma-separated values") { - assert( - Flaggable.ofMap[String, Set[Int]].parse("a=1,2,3,3,b=4,5") == - Map("a" -> Set(1, 2, 3), "b" -> Set(4, 5)) - ) - } - test("Flaggable: parse tuples") { assert(Flaggable.ofTuple[Int, String].parse("1,hello") == ((1, "hello"))) + assert(Flaggable.ofTuple[String, Boolean].parse("foo,true") == (("foo", true))) intercept[IllegalArgumentException] { Flaggable.ofTuple[Int, String].parse("1") } } From d05182413aca797268fa9940a047658d5c11f7b9 Mon Sep 17 00:00:00 2001 From: Harshit Kumar Date: Fri, 18 Sep 2020 19:37:21 +0000 Subject: [PATCH 385/796] [util-security]: Moving util-security certs into util-security-test-certs Problem ------- Currently the util-security certs are part of the test resources making them difficult to re-use at other projects. Solution -------- Moving them into a new folder util-security-test-certs and publishing them through sbt to allow for re-use in other projects. Additional Context ------------------ https://phabricator.twitter.biz/D531826#inline-4076125 JIRA Issues: PSEC-9422 Differential Revision: https://phabricator.twitter.biz/D552345 --- build.sbt | 12 +++++++++++- .../src/main}/resources/BUILD | 0 .../src/main}/resources/README | 0 .../main}/resources/certs/test-rsa-chain-server.crt | 0 .../src/main}/resources/certs/test-rsa-chain.crt | 0 .../src/main}/resources/certs/test-rsa-expired.crt | 0 .../resources/certs/test-rsa-full-cert-chain.crt | 0 .../src/main}/resources/certs/test-rsa-future.crt | 0 .../src/main}/resources/certs/test-rsa-garbage.crt | 0 .../src/main}/resources/certs/test-rsa-inter.crt | 0 .../src/main}/resources/certs/test-rsa.crt | 0 .../main}/resources/crl/csl-intermediate-garbage.crl | 0 .../src/main}/resources/crl/csl-intermediate.crl | 0 .../src/main}/resources/keys/test-pkcs8-dsa.key | 0 .../src/main}/resources/keys/test-pkcs8-ec.key | 0 .../src/main}/resources/keys/test-pkcs8-garbage.key | 0 .../src/main}/resources/keys/test-pkcs8.key | 0 .../src/main}/resources/pem/test-before-after.txt | 0 .../src/main}/resources/pem/test-multiple.txt | 0 .../src/main}/resources/pem/test-no-foot.txt | 0 .../src/main}/resources/pem/test-no-head.txt | 0 .../src/main}/resources/pem/test.txt | 0 .../src/test/scala/com/twitter/util/security/BUILD | 2 +- 23 files changed, 12 insertions(+), 2 deletions(-) rename {util-security/src/test => util-security-test-certs/src/main}/resources/BUILD (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/README (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/certs/test-rsa-chain-server.crt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/certs/test-rsa-chain.crt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/certs/test-rsa-expired.crt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/certs/test-rsa-full-cert-chain.crt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/certs/test-rsa-future.crt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/certs/test-rsa-garbage.crt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/certs/test-rsa-inter.crt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/certs/test-rsa.crt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/crl/csl-intermediate-garbage.crl (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/crl/csl-intermediate.crl (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/keys/test-pkcs8-dsa.key (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/keys/test-pkcs8-ec.key (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/keys/test-pkcs8-garbage.key (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/keys/test-pkcs8.key (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/pem/test-before-after.txt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/pem/test-multiple.txt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/pem/test-no-foot.txt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/pem/test-no-head.txt (100%) rename {util-security/src/test => util-security-test-certs/src/main}/resources/pem/test.txt (100%) diff --git a/build.sbt b/build.sbt index b3e9bb8567..365cbadb09 100644 --- a/build.sbt +++ b/build.sbt @@ -206,6 +206,7 @@ lazy val util = Project( utilRegistry, utilRouting, utilSecurity, + utilSecurityTestCerts, utilSlf4jApi, utilSlf4jJulBridge, utilStats, @@ -490,7 +491,16 @@ lazy val utilSecurity = Project( scalacheckLib, "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ) - ).dependsOn(utilCore, utilLogging) +).dependsOn(utilCore, utilLogging, utilSecurityTestCerts % "test") + +lazy val utilSecurityTestCerts = Project( + id = "util-security-test-certs", + base = file("util-security-test-certs") +).settings( + sharedSettings +).settings( + name := "util-security-test-certs" +) lazy val utilStats = Project( id = "util-stats", diff --git a/util-security/src/test/resources/BUILD b/util-security-test-certs/src/main/resources/BUILD similarity index 100% rename from util-security/src/test/resources/BUILD rename to util-security-test-certs/src/main/resources/BUILD diff --git a/util-security/src/test/resources/README b/util-security-test-certs/src/main/resources/README similarity index 100% rename from util-security/src/test/resources/README rename to util-security-test-certs/src/main/resources/README diff --git a/util-security/src/test/resources/certs/test-rsa-chain-server.crt b/util-security-test-certs/src/main/resources/certs/test-rsa-chain-server.crt similarity index 100% rename from util-security/src/test/resources/certs/test-rsa-chain-server.crt rename to util-security-test-certs/src/main/resources/certs/test-rsa-chain-server.crt diff --git a/util-security/src/test/resources/certs/test-rsa-chain.crt b/util-security-test-certs/src/main/resources/certs/test-rsa-chain.crt similarity index 100% rename from util-security/src/test/resources/certs/test-rsa-chain.crt rename to util-security-test-certs/src/main/resources/certs/test-rsa-chain.crt diff --git a/util-security/src/test/resources/certs/test-rsa-expired.crt b/util-security-test-certs/src/main/resources/certs/test-rsa-expired.crt similarity index 100% rename from util-security/src/test/resources/certs/test-rsa-expired.crt rename to util-security-test-certs/src/main/resources/certs/test-rsa-expired.crt diff --git a/util-security/src/test/resources/certs/test-rsa-full-cert-chain.crt b/util-security-test-certs/src/main/resources/certs/test-rsa-full-cert-chain.crt similarity index 100% rename from util-security/src/test/resources/certs/test-rsa-full-cert-chain.crt rename to util-security-test-certs/src/main/resources/certs/test-rsa-full-cert-chain.crt diff --git a/util-security/src/test/resources/certs/test-rsa-future.crt b/util-security-test-certs/src/main/resources/certs/test-rsa-future.crt similarity index 100% rename from util-security/src/test/resources/certs/test-rsa-future.crt rename to util-security-test-certs/src/main/resources/certs/test-rsa-future.crt diff --git a/util-security/src/test/resources/certs/test-rsa-garbage.crt b/util-security-test-certs/src/main/resources/certs/test-rsa-garbage.crt similarity index 100% rename from util-security/src/test/resources/certs/test-rsa-garbage.crt rename to util-security-test-certs/src/main/resources/certs/test-rsa-garbage.crt diff --git a/util-security/src/test/resources/certs/test-rsa-inter.crt b/util-security-test-certs/src/main/resources/certs/test-rsa-inter.crt similarity index 100% rename from util-security/src/test/resources/certs/test-rsa-inter.crt rename to util-security-test-certs/src/main/resources/certs/test-rsa-inter.crt diff --git a/util-security/src/test/resources/certs/test-rsa.crt b/util-security-test-certs/src/main/resources/certs/test-rsa.crt similarity index 100% rename from util-security/src/test/resources/certs/test-rsa.crt rename to util-security-test-certs/src/main/resources/certs/test-rsa.crt diff --git a/util-security/src/test/resources/crl/csl-intermediate-garbage.crl b/util-security-test-certs/src/main/resources/crl/csl-intermediate-garbage.crl similarity index 100% rename from util-security/src/test/resources/crl/csl-intermediate-garbage.crl rename to util-security-test-certs/src/main/resources/crl/csl-intermediate-garbage.crl diff --git a/util-security/src/test/resources/crl/csl-intermediate.crl b/util-security-test-certs/src/main/resources/crl/csl-intermediate.crl similarity index 100% rename from util-security/src/test/resources/crl/csl-intermediate.crl rename to util-security-test-certs/src/main/resources/crl/csl-intermediate.crl diff --git a/util-security/src/test/resources/keys/test-pkcs8-dsa.key b/util-security-test-certs/src/main/resources/keys/test-pkcs8-dsa.key similarity index 100% rename from util-security/src/test/resources/keys/test-pkcs8-dsa.key rename to util-security-test-certs/src/main/resources/keys/test-pkcs8-dsa.key diff --git a/util-security/src/test/resources/keys/test-pkcs8-ec.key b/util-security-test-certs/src/main/resources/keys/test-pkcs8-ec.key similarity index 100% rename from util-security/src/test/resources/keys/test-pkcs8-ec.key rename to util-security-test-certs/src/main/resources/keys/test-pkcs8-ec.key diff --git a/util-security/src/test/resources/keys/test-pkcs8-garbage.key b/util-security-test-certs/src/main/resources/keys/test-pkcs8-garbage.key similarity index 100% rename from util-security/src/test/resources/keys/test-pkcs8-garbage.key rename to util-security-test-certs/src/main/resources/keys/test-pkcs8-garbage.key diff --git a/util-security/src/test/resources/keys/test-pkcs8.key b/util-security-test-certs/src/main/resources/keys/test-pkcs8.key similarity index 100% rename from util-security/src/test/resources/keys/test-pkcs8.key rename to util-security-test-certs/src/main/resources/keys/test-pkcs8.key diff --git a/util-security/src/test/resources/pem/test-before-after.txt b/util-security-test-certs/src/main/resources/pem/test-before-after.txt similarity index 100% rename from util-security/src/test/resources/pem/test-before-after.txt rename to util-security-test-certs/src/main/resources/pem/test-before-after.txt diff --git a/util-security/src/test/resources/pem/test-multiple.txt b/util-security-test-certs/src/main/resources/pem/test-multiple.txt similarity index 100% rename from util-security/src/test/resources/pem/test-multiple.txt rename to util-security-test-certs/src/main/resources/pem/test-multiple.txt diff --git a/util-security/src/test/resources/pem/test-no-foot.txt b/util-security-test-certs/src/main/resources/pem/test-no-foot.txt similarity index 100% rename from util-security/src/test/resources/pem/test-no-foot.txt rename to util-security-test-certs/src/main/resources/pem/test-no-foot.txt diff --git a/util-security/src/test/resources/pem/test-no-head.txt b/util-security-test-certs/src/main/resources/pem/test-no-head.txt similarity index 100% rename from util-security/src/test/resources/pem/test-no-head.txt rename to util-security-test-certs/src/main/resources/pem/test-no-head.txt diff --git a/util-security/src/test/resources/pem/test.txt b/util-security-test-certs/src/main/resources/pem/test.txt similarity index 100% rename from util-security/src/test/resources/pem/test.txt rename to util-security-test-certs/src/main/resources/pem/test.txt diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index 3606cd0060..559ea8b8fc 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -9,7 +9,7 @@ junit_tests( "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/io", + "util/util-security-test-certs/src/main/resources", "util/util-security/src/main/scala/com/twitter/util/security", - "util/util-security/src/test/resources", ], ) From d0da0ec4bf245e0bb74f1aeb17f4362d019bd269 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 22 Sep 2020 20:21:39 +0000 Subject: [PATCH 386/796] Twitter-oss: Prepare OSS libraries for release 20.9.0 Problem We want to release the next versions of our Twitter OSS libraries 20.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-10167 Differential Revision: https://phabricator.twitter.biz/D553327 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7cd7bf2f88..6230d2f2ae 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.9.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index eae78e8fd1..492ce5cae5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.8.1" +val utilCore = "com.twitter" %% "util-core" % "20.9.0" ``` # Units diff --git a/build.sbt b/build.sbt index 365cbadb09..f0f83ba027 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.9.0-SNAPSHOT" +val releaseVersion = "20.9.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 013978b6147679513de3c981705371d911661dd6 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 22 Sep 2020 23:02:54 +0000 Subject: [PATCH 387/796] Twitter-oss: update OSS libraries post-release to 20.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10167 Differential Revision: https://phabricator.twitter.biz/D553399 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index f0f83ba027..60578a391a 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.9.0" +val releaseVersion = "20.10.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 7c3a460aa50ad974e5b1e169831da1cc2e66eb49 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Thu, 24 Sep 2020 17:39:35 +0000 Subject: [PATCH 388/796] finagle-core: set relative_name metadata for finagle client and server metrics Problem ------- Users currently filter metrics via regexp on the metric name. This can lead to misidentifying metrics (for example, when looking for failures metrics a client may end up with both the failures metrics and the tries/failures metric, which includes retries). Solution -------- Add a relative_name field to metrics metadata which the users can filter on directly. This is enabled by a new StatsReceiver which marks the scope beyond which the relative metric name can be found. Result ------ Client and Server related metrics created by finagle now have their relative_names configured and exposed via metrics metadata. JIRA Issues: CSL-10202 Differential Revision: https://phabricator.twitter.biz/D552357 --- .../twitter/finagle/stats/MetricBuilder.scala | 23 ++++++++++-- .../RelativeNameMarkingStatsReceiver.scala | 35 +++++++++++++++++++ .../stats/InMemoryStatsReceiverTest.scala | 6 ++-- 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 1c1d58bbaa..f4f9733417 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -23,7 +23,8 @@ case object Server extends SourceRole * @param role whether the service is playing the part of client or server regarding this metric * @param verbosity see StatsReceiver for details * @param sourceClass the name of the class which generated this metric (ie, com.twitter.finagle.StatsFilter) - * @param name the relative metric name which will be appended to the scope of the StatsReceiver prior to long term storage + * @param name the full metric name + * @param relativeName the relative metric name which will be appended to the scope of the StatsReceiver prior to long term storage * @param processPath a universal coordinate for the resource * @param percentiles used to indicate buckets for histograms, to be set by the StatsReceiver * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder @@ -36,6 +37,7 @@ class MetricBuilder( val verbosity: Verbosity = Verbosity.Default, val sourceClass: Option[String] = None, val name: Seq[String] = Seq.empty, + val relativeName: Seq[String] = Seq.empty, val processPath: Option[String] = None, // Only persisted and relevant when building histograms. val percentiles: IndexedSeq[Double] = IndexedSeq.empty, @@ -54,6 +56,7 @@ class MetricBuilder( verbosity: Verbosity = this.verbosity, sourceClass: Option[String] = this.sourceClass, name: Seq[String] = this.name, + relativeName: Seq[String] = this.relativeName, processPath: Option[String] = this.processPath, percentiles: IndexedSeq[Double] = this.percentiles ): MetricBuilder = { @@ -65,6 +68,7 @@ class MetricBuilder( verbosity = verbosity, sourceClass = sourceClass, name = name, + relativeName = relativeName, processPath = processPath, percentiles = percentiles, statsReceiver = this.statsReceiver @@ -90,6 +94,9 @@ class MetricBuilder( def withName(name: Seq[String]): MetricBuilder = this.copy(name = name) + def withRelativeName(relativeName: Seq[String]): MetricBuilder = + if (this.relativeName == Seq.empty) this.copy(relativeName = relativeName) else this + def withPercentiles(percentiles: IndexedSeq[Double]) = this.copy(percentiles = percentiles) /** @@ -152,6 +159,7 @@ class MetricBuilder( verbosity == that.verbosity && sourceClass == that.sourceClass && name == that.name && + relativeName == that.relativeName && processPath == that.processPath && percentiles == that.percentiles case _ => false @@ -159,13 +167,22 @@ class MetricBuilder( override def hashCode(): Int = { val state = - Seq[AnyRef](description, units, role, verbosity, sourceClass, name, processPath, percentiles) + Seq[AnyRef]( + description, + units, + role, + verbosity, + sourceClass, + name, + relativeName, + processPath, + percentiles) val hashCodes = keyIndicator.hashCode() +: state.map(_.hashCode()) hashCodes.foldLeft(0)((a, b) => 31 * a + b) } override def toString(): String = { val nameString = name.mkString("/") - s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $nameString, $processPath, $percentiles, $statsReceiver)" + s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $nameString, $relativeName, $processPath, $percentiles, $statsReceiver)" } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala new file mode 100644 index 0000000000..c4924120f3 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala @@ -0,0 +1,35 @@ +package com.twitter.finagle.stats + +/** + * A StatsReceiver receiver proxy that populates relativeName for all counter, stat, and gauge + * schemas/metadata. + * + * @param self The underlying StatsReceiver to which modified schemas are passed + */ +class RelativeNameMarkingStatsReceiver( + protected val self: StatsReceiver) + extends StatsReceiverProxy { + + override def counter(counterSchema: CounterSchema): Counter = { + val schema = CounterSchema( + counterSchema.metricBuilder + .withRelativeName(counterSchema.metricBuilder.name)) + self.counter(schema) + } + + override def stat(histogramSchema: HistogramSchema): Stat = { + val schema = + HistogramSchema( + histogramSchema.metricBuilder + .withRelativeName(histogramSchema.metricBuilder.name)) + self.stat(schema) + } + + override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { + val schema = + GaugeSchema( + gaugeSchema.metricBuilder + .withRelativeName(gaugeSchema.metricBuilder.name)) + self.addGauge(schema)(f) + } +} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index d5cf7ba331..7f6201d8d4 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -344,13 +344,13 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(parts.length == 3) assert( parts( - 0) == "coolGauge GaugeSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, coolGauge, None, Vector(), InMemoryStatsReceiver))") + 0) == "coolGauge GaugeSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, coolGauge, List(), None, Vector(), InMemoryStatsReceiver))") assert( parts( - 1) == "rad/histo HistogramSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, rad/histo, None, Vector(), InMemoryStatsReceiver/rad))") + 1) == "rad/histo HistogramSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, rad/histo, List(), None, Vector(), InMemoryStatsReceiver/rad))") assert( parts( - 2) == "sweet/counter CounterSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, sweet/counter, None, Vector(), InMemoryStatsReceiver))") + 2) == "sweet/counter CounterSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, sweet/counter, List(), None, Vector(), InMemoryStatsReceiver))") } finally { ps.close() } From 0967823bfe6bbc3f1c2a1f756e7c16af99dc65d1 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Fri, 25 Sep 2020 19:42:05 +0000 Subject: [PATCH 389/796] util-core: Fix Scaladoc Warning about Usecase in Activity Problem / Solution The ScalaDoc 'usecase' tag has been deprecated. Let's replace its use in `Activity` with 'example'. JIRA Issues: CSL-9465 Differential Revision: https://phabricator.twitter.biz/D555522 --- util-core/src/main/scala/com/twitter/util/Activity.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Activity.scala b/util-core/src/main/scala/com/twitter/util/Activity.scala index 68a814e8af..dabc6446a7 100644 --- a/util-core/src/main/scala/com/twitter/util/Activity.scala +++ b/util-core/src/main/scala/com/twitter/util/Activity.scala @@ -184,7 +184,7 @@ object Activity { * Collect a collection of activities into an activity of a collection * of values. * - * @usecase def collect[T](activities: Coll[Activity[T]]): Activity[Coll[T]] + * @example def collect[T](activities: Coll[Activity[T]]): Activity[Coll[T]] */ def collect[T: ClassTag, CC[X] <: Iterable[X]]( acts: CC[Activity[T]] @@ -202,7 +202,7 @@ object Activity { * Like Var.collectIndependent this is a workaround and should be deprecated when a version * of Var.collect without a stack overflow issue is implemented. * - * @usecase def collectIndependent[T](activities: Coll[Activity[T]]): Activity[Coll[T]] + * @example def collectIndependent[T](activities: Coll[Activity[T]]): Activity[Coll[T]] */ def collectIndependent[T: ClassTag, CC[X] <: Iterable[X]]( acts: CC[Activity[T]] From 26e7874bfe4fdc7b338acfbd74612b414adf7118 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 1 Oct 2020 18:40:25 +0000 Subject: [PATCH 390/796] util-slf4j-api: Improved Java interop Problem There is no longer a need to have a separately named factory object to provide an ergonomic API for Java users. Java users are now able to use factory methods on a companion object without the need for the verbose `Companion$.MODULE$` notation. Solution Deprecate the `Loggers` factory object which provided Java factory constructors and move those into the `Logger` companion. Add more tests and patch scaladocs. Result More Java interop. Differential Revision: https://phabricator.twitter.biz/D558605 --- CHANGELOG.rst | 4 ++ util-slf4j-api/README.md | 68 ++++++++++++++----- .../com/twitter/util/logging/Logger.scala | 63 +++++++++++++---- .../com/twitter/util/logging/Loggers.scala | 13 ++-- .../com/twitter/util/logging/Logging.scala | 4 +- .../util/logging/LoggerCompilationTest.java | 45 ++++++++++++ .../twitter/util/logging/JavaExtension.java | 25 +++++++ .../com/twitter/util/logging/JavaItem.java | 2 +- .../twitter/util/logging/LoggingTest.scala | 13 +++- .../twitter/util/logging/TestJavaClass.java | 2 +- .../util/logging/TraitWithLogging.scala | 11 +++ 11 files changed, 209 insertions(+), 41 deletions(-) create mode 100644 util-slf4j-api/src/test/java/com/twitter/util/logging/LoggerCompilationTest.java create mode 100644 util-slf4j-api/src/test/scala/com/twitter/util/logging/JavaExtension.java create mode 100644 util-slf4j-api/src/test/scala/com/twitter/util/logging/TraitWithLogging.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6230d2f2ae..cedafd5c77 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* util-slf4j-api: Better Java interop. Deprecate `c.t.util.logging.Loggers` as Java users should be + able to use the `c.t.util.logging.Logger` companion object with less verbosity required. + ``PHAB_ID=D558605`` + 20.9.0 ------ diff --git a/util-slf4j-api/README.md b/util-slf4j-api/README.md index 1e45827a87..9e1b612e9a 100644 --- a/util-slf4j-api/README.md +++ b/util-slf4j-api/README.md @@ -1,43 +1,70 @@ # Twitter Util SLF4J API Support -This library provides a Scala-friendly logging wrapper around the [slf4j-api](https://www.slf4j.org/) [Logger](https://www.slf4j.org/apidocs/org/slf4j/Logger.html) interface. +This library provides a Scala-friendly logging wrapper around the [slf4j-api](https://www.slf4j.org/) +[Logger](https://www.slf4j.org/apidocs/org/slf4j/Logger.html) interface. ## Differences with `util-logging` -The code in `util-logging` is a scala wrapper over `java.util.logging` (JUL) which is a logging API **and** implementation. `util-slf4j-api` is an API-only and adds a thin scala wrapper over the [slf4j-api](https://www.slf4j.org/) which allows -users to choose their logging implementation. It is recommended that users move to using `util-slf4j-api` over `util-logging`. +The code in `util-logging` is a scala wrapper over `java.util.logging` (JUL) which is a logging API +**and** implementation. `util-slf4j-api` is an API-only and adds a thin scala wrapper over the +[slf4j-api](https://www.slf4j.org/) which allows users to choose their logging implementation. It is +recommended that users move to using `util-slf4j-api` over `util-logging`. -Since the [slf4j-api](https://www.slf4j.org/manual.html) is only an interface it requires an actual logging implementation. You should ensure that you do not end-up with *multiple* logging implementations on your classpath, e.g., you should not have multiple SLF4J bindings (`slf4j-nop`, `slf4j-log4j12`, `slf4j-jdk14`, etc.) and/or a `java.util.logging` implementation, etc. on your classpath as these are all competing implementations and since classpath order is non-deterministic, can lead to unexpected logging behavior. +Since the [slf4j-api](https://www.slf4j.org/manual.html) is only an interface it requires an actual +logging implementation. You should ensure that you do not end-up with *multiple* logging +implementations on your classpath, e.g., you should not have multiple SLF4J bindings (`slf4j-nop`, +`slf4j-log4j12`, `slf4j-jdk14`, etc.) and/or a `java.util.logging` implementation, etc. on your +classpath as these are all competing implementations and since classpath order is non-deterministic, +can lead to unexpected logging behavior. ## Usages -You can create a `c.t.util.logging.Logger` in different ways. You can pass a `name` to the `Logger#apply` factory method on the `c.t.util.logging.Logger` companion object: +You can create a `c.t.util.logging.Logger` in different ways. You can pass a `name` to the +`Logger#apply` factory method on the `c.t.util.logging.Logger` companion object: ```scala -val logger = Logger("name") +val logger = Logger("name") // scala +``` + +or in Java, use the `Logger#getLogger` method: + +```java +Logger log = Logger.getLogger("name"); // java ``` Or, pass in a class: ```scala -val logger = Logger(classOf[MyClass]) +val logger = Logger(classOf[MyClass]) // scala +``` + +```java +Logger log = Logger.getLogger(MyClass.class); // java ``` Or, use the runtime class wrapped by the implicit ClassTag parameter: ```scala -val logger = Logger[MyClass] +val logger = Logger[MyClass] // scala ``` Or, finally, use an SLF4J Logger instance: ```scala -val logger = Logger(LoggerFactory.getLogger("name")) +val logger = Logger(LoggerFactory.getLogger("name")) // scala +``` + +```java +Logger log = Logger.getLogger(LoggerFactory.getLogger("name")); // java ``` -There is also the `c.t.util.logging.Logging` trait which can be mixed into a class. Note, that the underlying SLF4J logger will be named according to the leaf class of any hierarchy into which the trait is mixed: +There is also the `c.t.util.logging.Logging` trait which can be mixed into a class. Note, that the +underlying SLF4J logger will be named according to the leaf class of any hierarchy into which the +trait is mixed: ```scala +package com.twitter + class Stock(symbol: String, price: BigDecimal) extends Logging { info(f"New stock with symbol = $symbol and price = $price%1.2f") ... @@ -47,11 +74,13 @@ class Stock(symbol: String, price: BigDecimal) extends Logging { This would produce a logging statement along the lines of: ``` -2017-01-05 13:05:49,349 INF Stock New stock with symbol = ASDF and price = 100.00 +2017-01-05 13:05:49,349 INF c.t.Stock New stock with symbol = ASDF and price = 100.00 ``` -If you are using Java and happen to extend a Scala class which mixes in the `c.t.util.logging.Logging` trait, you can either access the logger directly or choose to instantiate a new logger in your -Java class to log your statements. Again, note that the underlying SLF4J logger when mixed in via the trait will be named according to the leaf class into which it is mixed. +If you are using Java and happen to extend a Scala class which mixes in the `c.t.util.logging.Logging` +trait, you can either access the logger directly or choose to instantiate a new logger in your +Java class to log your statements. Again, note that the underlying SLF4J logger when mixed in via +the trait will be named according to the leaf class into which it is mixed. E.g., for a Scala class: @@ -61,7 +90,9 @@ class MyBaseScalaClass extends Logging { } ``` -You could choose to use the inherited logger directly since the inherited `c.t.util.logging.Logging` trait methods expect an anonymous function: +You could choose to use the inherited `logger` instance directly (since the inherited +`c.t.util.logging.Logging` trait methods would require using a `scala.Function0` as the trait +methods are explicitly call-by-name): ``` class MyJavaClass extends MyBaseScalaClass { @@ -73,20 +104,21 @@ class MyJavaClass extends MyBaseScalaClass { } ``` -Or simply instantiate a new logger: +Or instantiate a new logger explicitly: ``` class MyJavaClass extends MyBaseScalaClass { - private static final Logger logger = new Logger(MyJavaClass.class); + private static final Logger LOG = Logger.getLogger(MyJavaClass.class); ... public void methodThatDoesSomeLogging() { - logger.info("Accessing the superclass logger"); + LOG.info("Accessing the superclass logger"); } } ``` -For another Scala class that extends the above example Scala class with the mixed-in `c.t.util.logging.Logging` trait, you have the same options along with: +For another Scala class which extends the above example Scala class with the mixed-in +`c.t.util.logging.Logging` trait, you have the same options along with: - simply use the inherited Logging trait methods. - extend the `c.t.util.logging.Logging` trait again. diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala index fa783f7231..ad00ab5cba 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala @@ -7,14 +7,13 @@ import scala.reflect.{ClassTag, classTag} /** * Companion object for [[com.twitter.util.logging.Logger]] which provides * factory methods for creation. - * - * @note Java users, see [[com.twitter.util.logging.Loggers.getLogger]] */ object Logger { /** * Create a [[com.twitter.util.logging.Logger]] for the given name. - * @param name name of the underlying Logger. + * @param name name of the underlying `org.slf4j.Logger`. + * @see Java users see [[getLogger(name:String)]] * * {{{ * val logger = Logger("name") @@ -24,9 +23,22 @@ object Logger { new Logger(LoggerFactory.getLogger(name)) } + /** + * Create a [[Logger]] for the given name. + * @param name name of the underlying `org.slf4j.Logger`. + * @see Scala users see [[apply(name:String)]] + * + * {{{ + * Logger logger = Logger.getLogger("name"); + * }}} + */ + def getLogger(name: String): Logger = + Logger(name) + /** * Create a [[com.twitter.util.logging.Logger]] named for the given class. * @param clazz class to use for naming the underlying Logger. + * @see Java users see [[getLogger(clazz:Class[_])]] * * {{{ * val logger = Logger(classOf[MyClass]) @@ -36,6 +48,18 @@ object Logger { new Logger(LoggerFactory.getLogger(clazz)) } + /** + * Create a [[Logger]] named for the given class. + * @param clazz class to use for naming the underlying `org.slf4j.Logger`. + * @see Scala users see [[apply(clazz:Class[_])]] + * + * {{{ + * Logger logger = Logger.getLogger(MyClass.class); + * }}} + */ + def getLogger(clazz: Class[_]): Logger = + Logger(clazz) + /** * Create a [[com.twitter.util.logging.Logger]] for the runtime class wrapped * by the implicit [[scala.reflect.ClassTag]] denoting the runtime class `T`. @@ -51,8 +75,9 @@ object Logger { /** * Create a [[com.twitter.util.logging.Logger]] wrapping the given underlying - * [[org.slf4j.Logger]]. + * `org.slf4j.Logger`. * @param underlying an `org.slf4j.Logger` + * @see Java users see [[getLogger(underlying:Logger)]] * * {{{ * val logger = Logger(LoggerFactory.getLogger("name")) @@ -61,14 +86,27 @@ object Logger { def apply(underlying: slf4j.Logger): Logger = { new Logger(underlying) } + + /** + * Create a [[Logger]] wrapping the given underlying + * `org.slf4j.Logger`. + * @param underlying an `org.slf4j.Logger` + * @see Scala users see [[apply(underlying:Logger)]] + * + * {{{ + * Logger logger = Logger.getLogger(LoggerFactory.getLogger("name")); + * }}} + */ + def getLogger(underlying: slf4j.Logger): Logger = + Logger(underlying) } /** - * A scala wrapper over a [[org.slf4j.Logger]]. + * A scala wrapper over a `org.slf4j.Logger`. * - * The Logger is [[Serializable]] to support it's usage through the - * [[com.twitter.util.logging.Logging]] trait when the trait is mixed - * into a [[Serializable]] class. + * The Logger extends [[https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html java.io.Serializable]] + * to support it's usage through the [[com.twitter.util.logging.Logging]] trait when the trait is mixed + * into a [[https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html java.io.Serializable]] class. * * @define isLevelEnabled * @@ -76,8 +114,8 @@ object Logger { * * @define isLevelEnabledMarker * - * Determines if the named log level is enabled taking into consideration the given [[Marker]] data. - * Returns `true` if enabled, `false` otherwise. + * Determines if the named log level is enabled taking into consideration the given + * `org.slf4j.Marker` data. Returns `true` if enabled, `false` otherwise. * * @define log * @@ -86,7 +124,7 @@ object Logger { * @define logMarker * * Logs the given message at the named log level taking into consideration the - * given [[Marker]] data. + * given `org.slf4j.Marker` data. * * @define logWith * @@ -96,8 +134,7 @@ object Logger { * @define logWithMarker * * Log the given parameterized message at the named log level using the given - * args taking into consideration the given [[Marker]] data. - * See [[https://www.slf4j.org/faq.html#logging_performance Parameterized Message Logging]] + * args taking into consideration the given `org.slf4j.Marker` data. See [[https://www.slf4j.org/faq.html#logging_performance Parameterized Message Logging]] */ @SerialVersionUID(1L) final class Logger private (underlying: slf4j.Logger) extends Serializable { diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Loggers.scala b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Loggers.scala index 3600f01f09..53317c036d 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Loggers.scala +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Loggers.scala @@ -6,7 +6,9 @@ import org.slf4j * For Java usability. * * @note Scala users see [[com.twitter.util.logging.Logger]] + * @deprecated Java users can use corresponding methods in [[com.twitter.util.logging.Logger]] */ +@deprecated("Use com.twitter.util.logging.Logger", "2020-09-30") object Loggers { /** @@ -15,9 +17,10 @@ object Loggers { * @param name name of the underlying `slf4j.Logger`. * * {{{ - * val logger = Logger("name") + * Logger logger = Logger.getLogger("name"); * }}} */ + @deprecated("Use com.twitter.util.logging.Logger", "2020-09-30") def getLogger(name: String): Logger = Logger(name) @@ -27,22 +30,24 @@ object Loggers { * @param clazz class to use for naming the underlying slf4j.Logger. * * {{{ - * val logger = Logger(classOf[MyClass]) + * Logger logger = Logger.getLogger(classOf[MyClass]); * }}} */ + @deprecated("Use com.twitter.util.logging.Logger", "2020-09-30") def getLogger(clazz: Class[_]): Logger = Logger(clazz) /** * Create a [[Logger]] wrapping the given underlying - * [[org.slf4j.Logger]]. + * `org.slf4j.Logger`. * * @param underlying an `org.slf4j.Logger` * * {{{ - * val logger = Logger(LoggerFactory.getLogger("name")) + * Logger logger = Logger.getLogger(LoggerFactory.getLogger("name")); * }}} */ + @deprecated("Use com.twitter.util.logging.Logger", "2020-09-30") def getLogger(underlying: slf4j.Logger): Logger = Logger(underlying) } diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logging.scala b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logging.scala index c58bae04ad..7b2f1d5f1b 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logging.scala +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logging.scala @@ -30,7 +30,7 @@ import scala.language.implicitConversions * @define isLevelEnabledMarker * * Determines if the named log level is enabled on the underlying logger taking into - * consideration the given [[Marker]] data. Returns `true` if enabled, `false` otherwise. + * consideration the given `org.slf4j.Marker` data. Returns `true` if enabled, `false` otherwise. * * @define log * @@ -40,7 +40,7 @@ import scala.language.implicitConversions * @define logMarker * * Logs the given message at the named log level with the underlying logger taking into - * consideration the given [[Marker]] data. + * consideration the given `org.slf4j.Marker` data. * * @define logResult * diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/LoggerCompilationTest.java b/util-slf4j-api/src/test/java/com/twitter/util/logging/LoggerCompilationTest.java new file mode 100644 index 0000000000..23c078fb9d --- /dev/null +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/LoggerCompilationTest.java @@ -0,0 +1,45 @@ +package com.twitter.util.logging; + +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import static org.junit.Assert.assertEquals; + +public class LoggerCompilationTest { + + @Test + public void testApply1() { + /* By slf4j Logger */ + + Logger logger = + Logger.getLogger(LoggerFactory.getLogger(this.getClass())); + + assertEquals( + "com.twitter.util.logging.LoggerCompilationTest", + logger.name()); + } + + @Test + public void testApply2() { + /* By Class */ + + Logger logger = + Logger.getLogger(LoggerCompilationTest.class); + + assertEquals( + "com.twitter.util.logging.LoggerCompilationTest", + logger.name()); + } + + @Test + public void testApply3() { + /* By String */ + + Logger logger = + Logger.getLogger("LoggerCompilationTest"); + + assertEquals( + "LoggerCompilationTest", + logger.name()); + } +} diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/JavaExtension.java b/util-slf4j-api/src/test/scala/com/twitter/util/logging/JavaExtension.java new file mode 100644 index 0000000000..60591e0cad --- /dev/null +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/JavaExtension.java @@ -0,0 +1,25 @@ +package com.twitter.util.logging; + +import scala.runtime.AbstractFunction0; + +public class JavaExtension extends AbstractTraitWithLogging { + private static final Logger LOG = Logger.getLogger(JavaExtension.class); + + /** + * myMethod2 + * @return a "hello world" String + */ + public String myMethod2() { + info(new AbstractFunction0() { + @Override + public Object apply() { + return "In myMethod2: using trait#info"; + } + }); + + logger().info("In myMethod2: using logger#info"); + LOG.info("In myMethod2: using LOG#info"); + + return "Hello, world"; + } +} diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/JavaItem.java b/util-slf4j-api/src/test/scala/com/twitter/util/logging/JavaItem.java index 6cdbf415b8..7abdc6c14d 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/JavaItem.java +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/JavaItem.java @@ -1,7 +1,7 @@ package com.twitter.util.logging; class JavaItem extends SecondItem { - private static final Logger LOG = Logger.apply(JavaItem.class); + private static final Logger LOG = Logger.getLogger(JavaItem.class); public JavaItem(String name, String description, int size, int count) { super(name, description, size, count); diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala index 95fd35b4f5..f35af4d1e6 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala @@ -306,7 +306,16 @@ class LoggingTest extends AnyFunSuite with Matchers with Mockito { } test("Logging#Java class with new logger") { - new TestJavaClass() + new TestJavaClass // INFO com.twitter.util.logging.TestJavaClass - Creating new TestJavaClass instance. + + val t = new TraitWithLogging {} + t.myMethod1 // INFO com.twitter.util.logging.LoggingTest$$anon$1 - In myMethod1 + + val je = new JavaExtension + je.myMethod1 // INFO com.twitter.util.logging.JavaExtension - In myMethod1 + je.myMethod2() // INFO com.twitter.util.logging.JavaExtension - In myMethod2: using trait#info + // INFO com.twitter.util.logging.JavaExtension - In myMethod2: using logger#info + // INFO com.twitter.util.logging.JavaExtension - In myMethod2: using LOG#info } test("Logging#Java class that extends Scala class with Logging trait and new logger") { @@ -353,7 +362,7 @@ class LoggingTest extends AnyFunSuite with Matchers with Mockito { val message = "msg" val cause = new RuntimeException("TEST EXCEPTION") val arg1 = "arg1" - val arg2 = new Integer(1) + val arg2 = Integer.valueOf(1) val arg3 = "arg3" val underlying: org.slf4j.Logger = mock[org.slf4j.Logger] when(isEnabledFn(underlying)).thenReturn(isEnabled) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/TestJavaClass.java b/util-slf4j-api/src/test/scala/com/twitter/util/logging/TestJavaClass.java index f3226abea2..31162aa430 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/TestJavaClass.java +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/TestJavaClass.java @@ -1,7 +1,7 @@ package com.twitter.util.logging; class TestJavaClass { - private static final Logger LOG = Logger.apply(TestJavaClass.class); + private static final Logger LOG = Logger.getLogger(TestJavaClass.class); public TestJavaClass() { LOG.info("Creating new TestJavaClass instance."); diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/TraitWithLogging.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/TraitWithLogging.scala new file mode 100644 index 0000000000..e0553edaf1 --- /dev/null +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/TraitWithLogging.scala @@ -0,0 +1,11 @@ +package com.twitter.util.logging + +abstract class AbstractTraitWithLogging extends TraitWithLogging + +trait TraitWithLogging extends Logging { + + def myMethod1: String = { + info("In myMethod1") + "Hello, World" + } +} From 7665b9eb09adaaa282e65f32b040aeb14ce29c3b Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Thu, 1 Oct 2020 21:56:53 +0000 Subject: [PATCH 391/796] util-stats: use service scopeSeparator in MetricBuilder Problem ------- Metric Metadata uses / as the scope separator in all cases. As a result, when service owners configure a custom scope separator (ex, -) the metric names will differ in metrics.json and metric_metadata.json. Solution -------- Create an object in util-stats to store a scope separating character for MetricBuilder and metric_metadata.json to make use of. Result ------ Metric names now align across metrics.json and metric_metadata.json for services with custom configured scope separators JIRA Issues: CSL-10202 Differential Revision: https://phabricator.twitter.biz/D557994 --- CHANGELOG.rst | 7 +++++++ .../twitter/finagle/stats/MetricBuilder.scala | 17 ++++++++++++++++- .../finagle/stats/MetricBuilderTest.scala | 16 ++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cedafd5c77..975e441c30 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Bug Fixes +~~~~~~~~~ +* util-stat: `MetricBuilder` now uses a configurable metadataScopeSeparator to align + more closely with the metrics.json api. Services with an overridden scopeSeparator will + now see that reflected in metric_metadata.json where previously it was erroneously using + / in all cases. ``PHAB_ID=D557994`` + * util-slf4j-api: Better Java interop. Deprecate `c.t.util.logging.Loggers` as Java users should be able to use the `c.t.util.logging.Logger` companion object with less verbosity required. ``PHAB_ID=D558605`` diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index f4f9733417..7975542ad9 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -12,6 +12,21 @@ case object NoRoleSpecified extends SourceRole case object Client extends SourceRole case object Server extends SourceRole +/** + * finagle-stats has configurable scope separators. As this package is wrapped by finagle-stats, we + * cannot retrieve it from finagle-stats. Consequently, we created this object so that the + * scope-separator can be passed in for stringification of the MetricBuilder objects. + */ +object metadataScopeSeparator { + @volatile private var separator: String = "/" + + def apply(): String = separator + + private[finagle] def setSeparator(separator: String): Unit = { + this.separator = separator + } +} + /** * A builder class used to configure settings and metadata for metrics prior to instantiating them. * Calling any of the three build methods (counter, gauge, or histogram) will cause the metric to be @@ -182,7 +197,7 @@ class MetricBuilder( } override def toString(): String = { - val nameString = name.mkString("/") + val nameString = name.mkString(metadataScopeSeparator()) s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $nameString, $relativeName, $processPath, $percentiles, $statsReceiver)" } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala new file mode 100644 index 0000000000..ead3a74cdd --- /dev/null +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala @@ -0,0 +1,16 @@ +package com.twitter.finagle.stats + +import org.scalatest.funsuite.AnyFunSuite + +class MetricBuilderTest extends AnyFunSuite { + + test("metadataScopeSeparator affects stringification") { + val mb = new MetricBuilder(name = Seq("foo", "bar"), statsReceiver = new InMemoryStatsReceiver) + assert(mb.toString.contains("foo/bar")) + metadataScopeSeparator.setSeparator("-") + assert(mb.toString.contains("foo-bar")) + metadataScopeSeparator.setSeparator("_aTerriblyLongStringForSomeReason_") + assert(mb.toString.contains("foo_aTerriblyLongStringForSomeReason_bar")) + } + +} From bae06c6a4f5ab1355b5a209397176c806faa94a8 Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Tue, 13 Oct 2020 19:05:58 +0000 Subject: [PATCH 392/796] Various core security ownership changes Problem ======= There is a proposal approved to make the included changes to project files. Solution ======== Make the changes. JIRA Issues: PSEC-9625 Differential Revision: https://phabricator.twitter.biz/D564424 --- util-security/PROJECT | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/util-security/PROJECT b/util-security/PROJECT index 4b4968d8d4..fddaac65e3 100644 --- a/util-security/PROJECT +++ b/util-security/PROJECT @@ -1,9 +1,11 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: + # Individual users are not responsible for providing a blocking review, but they + # are capable of doing so in the event that members of the groups cannot. - banderson - bdimmick - - mnakamura - roanta - - ryano - vkostyukov + - csl-core-security:phab + - platform-security:ldap From 9ff7f3776b74c51651ba15d786671df82e691d28 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Fri, 16 Oct 2020 04:39:17 +0000 Subject: [PATCH 393/796] util: Fix All Scaladoc Warnings Problem / Solution There are a few hundred warnings for Scaladoc generation in Util. Some of the warnings hide actual problems, while others cause links to not go where we want them to. Let's fix all of them up so that there aren't any more warnings. JIRA Issues: CSL-10225 Differential Revision: https://phabricator.twitter.biz/D556020 --- .../src/main/scala/com/twitter/app/App.scala | 12 ++-- .../src/main/scala/com/twitter/app/Flag.scala | 4 +- .../scala/com/twitter/app/Flaggable.scala | 4 +- .../main/scala/com/twitter/app/Flags.scala | 4 +- .../scala/com/twitter/app/Lifecycle.scala | 6 +- .../scala/com/twitter/app/LoadService.scala | 12 ++-- .../com/twitter/cache/guava/GuavaCache.scala | 14 ++--- .../twitter/cache/ConcurrentMapCache.scala | 6 +- .../scala/com/twitter/cache/FutureCache.scala | 6 +- .../scala/com/twitter/cache/Refresh.scala | 2 +- .../cache/caffeine/CaffeineCache.scala | 4 +- .../concurrent/NamedPoolThreadFactory.scala | 4 +- .../scala/com/twitter/concurrent/Offer.scala | 2 +- .../com/twitter/concurrent/Scheduler.scala | 6 +- .../scala/com/twitter/concurrent/Tx.scala | 2 +- .../VirtualMachineErrorTerminator.java | 4 +- .../com/twitter/concurrent/AsyncMeter.scala | 10 +-- .../com/twitter/concurrent/AsyncQueue.scala | 8 +-- .../com/twitter/concurrent/AsyncStream.scala | 6 +- .../com/twitter/conversions/DurationOps.scala | 2 +- .../twitter/conversions/StorageUnitOps.scala | 2 +- .../com/twitter/conversions/StringOps.scala | 2 +- .../com/twitter/conversions/U64Ops.scala | 4 +- .../src/main/scala/com/twitter/io/Buf.scala | 42 +++++++------ .../main/scala/com/twitter/io/BufReader.scala | 2 +- .../scala/com/twitter/io/ByteReader.scala | 4 +- .../main/scala/com/twitter/io/Charsets.scala | 10 +-- .../com/twitter/io/InputStreamReader.scala | 10 +-- .../src/main/scala/com/twitter/io/Pipe.scala | 13 ++-- .../main/scala/com/twitter/io/Reader.scala | 62 ++++++++++--------- .../scala/com/twitter/io/TempFolder.scala | 4 +- .../main/scala/com/twitter/io/Writer.scala | 35 ++++++----- .../scala/com/twitter/util/Awaitable.scala | 6 +- .../scala/com/twitter/util/Base64Long.scala | 8 +-- .../scala/com/twitter/util/CloseOnce.scala | 6 +- .../main/scala/com/twitter/util/Config.scala | 2 +- .../scala/com/twitter/util/Duration.scala | 8 +-- .../main/scala/com/twitter/util/Event.scala | 17 ++--- .../scala/com/twitter/util/Function.scala | 13 ++-- .../main/scala/com/twitter/util/Future.scala | 45 +++++++++----- .../scala/com/twitter/util/FuturePool.scala | 4 +- .../main/scala/com/twitter/util/Local.scala | 2 +- .../main/scala/com/twitter/util/Promise.scala | 36 +++++++---- .../scala/com/twitter/util/RandomSocket.scala | 2 +- .../scala/com/twitter/util/Stopwatch.scala | 4 +- .../scala/com/twitter/util/StorageUnit.scala | 2 +- .../main/scala/com/twitter/util/Time.scala | 4 +- .../main/scala/com/twitter/util/Timer.scala | 6 +- .../com/twitter/jvm/ContentionSnapshot.scala | 8 ++- .../src/main/scala/com/twitter/jvm/Jvm.scala | 4 +- .../scala/com/twitter/util/lint/Rule.scala | 2 +- .../scala/com/twitter/logging/Level.scala | 2 +- .../scala/com/twitter/logging/LogRecord.scala | 8 +-- .../com/twitter/logging/QueueingHandler.scala | 18 +----- .../scala/com/twitter/util/mock/Mockito.scala | 8 +-- .../com/twitter/util/registry/Library.scala | 2 +- .../twitter/util/routing/AbstractRouter.scala | 4 +- .../util/routing/ClosedRouterException.scala | 4 +- .../com/twitter/util/routing/Generator.scala | 4 +- .../com/twitter/util/routing/Router.scala | 44 +++++++------ .../twitter/util/routing/RouterBuilder.scala | 14 ++--- .../util/routing/ValidationError.scala | 2 +- .../util/routing/ValidationException.scala | 2 +- .../security/Pkcs8KeyManagerFactory.scala | 6 +- .../util/security/PrivateKeyFile.scala | 5 +- .../util/security/X500PrincipalInfo.scala | 2 +- .../security/X509TrustManagerFactory.scala | 6 +- .../com/twitter/util/logging/Logger.scala | 32 +++++++--- .../finagle/stats/InMemoryStatsReceiver.scala | 12 ++-- .../com/twitter/finagle/stats/Stat.scala | 9 ++- .../twitter/finagle/stats/StatsReceiver.scala | 50 ++++++++++----- .../com/twitter/util/ThriftSerializer.scala | 6 +- .../util/tunable/JsonTunableMapper.scala | 36 +++++------ .../com/twitter/util/tunable/TunableMap.scala | 8 +-- .../zk/coordination/ShardCoordinator.scala | 4 +- 75 files changed, 423 insertions(+), 352 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index 6eefbb74ec..51a8bd9345 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -35,6 +35,9 @@ import scala.util.control.NonFatal * * Note that a missing `main` is OK: mixins may provide behavior that * does not require defining a custom `main` method. + * + * @define LoadServiceApplyScaladocLink + * [[com.twitter.app.LoadService.apply[T]()(implicitevidence\$1:scala\.reflect\.ClassTag[T]):Seq[T]* LoadService.apply]] */ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { @@ -121,7 +124,8 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { protected lazy val shutdownTimer: Timer = new JavaTimer(isDaemon = true) /** - * Programmatically specify which implementations to use in [[LoadService.apply]] + * Programmatically specify which implementations to use in + * $LoadServiceApplyScaladocLink * for the given interfaces. This allows applications to circumvent * the standard service loading mechanism when needed. It may be useful * if the application has a broad and/or rapidly changing set of dependencies. @@ -148,7 +152,7 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { * } * }}} * - * If this is called for a `Class[T]` after [[LoadService.apply]] + * If this is called for a `Class[T]` after $LoadServiceApplyScaladocLink * has been called for that same interface, an `IllegalStateException` * will be thrown. For this reason, bindings are done as early * as possible in the application lifecycle, before even `inits` @@ -158,9 +162,9 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { * their user's implementation choice. * * @return a mapping from `Class` to the 1 or more implementations to - * be used by [[LoadService.apply]] for that interface. + * be used by $LoadServiceApplyScaladocLink for that interface. * - * @see [[LoadService.apply]] + * @see $LoadServiceApplyScaladocLink */ protected[this] def loadServiceBindings: Seq[LoadService.Binding[_]] = Nil diff --git a/util-app/src/main/scala/com/twitter/app/Flag.scala b/util-app/src/main/scala/com/twitter/app/Flag.scala index e33bf59dfb..93f262cb4d 100644 --- a/util-app/src/main/scala/com/twitter/app/Flag.scala +++ b/util-app/src/main/scala/com/twitter/app/Flag.scala @@ -19,7 +19,7 @@ object Flag { * A single command-line flag, instantiated by a [[com.twitter.app.Flags]] * instance. * - * The current value can be extracted via [[apply()]], [[get]], and + * The current value can be extracted via [[apply]], [[get]], and * [[getWithDefault]]. [[com.twitter.util.Local Local]]-scoped modifications * of their values, which can be useful for tests, can be done by calls to * [[let]] and [[letClear]]. @@ -163,7 +163,7 @@ class Flag[T: Flaggable] private[app] ( let(Some(t), f) /** - * Override the value of this flag with `arg` [[String] parsed to this [[Flag]]'s [[T]] type, + * Override the value of this flag with `arg` `String` parsed to this [[Flag]]'s `T` type, * only for the scope of the current [[com.twitter.util.Local]] for the given function `f`. * * @see [[let]] diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index a1957509b1..43646031fd 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -199,8 +199,8 @@ object Flaggable { /** * Create a Flaggable of Java Enum. - * @param clazz The [[java.lang.Enum]] class. - * @note [[java.lang.Enum]] enumeration constant value look up is case insensitive. + * @param clazz The `java.lang.Enum` class. + * @note `java.lang.Enum` enumeration constant value look up is case insensitive. */ def ofJavaEnum[T <: Enum[T]](clazz: Class[T]): Flaggable[T] = new Typed[T] { def parse(s: String): T = diff --git a/util-app/src/main/scala/com/twitter/app/Flags.scala b/util-app/src/main/scala/com/twitter/app/Flags.scala index 6e76ab9cd2..f92faf104a 100644 --- a/util-app/src/main/scala/com/twitter/app/Flags.scala +++ b/util-app/src/main/scala/com/twitter/app/Flags.scala @@ -12,7 +12,7 @@ import scala.util.control.NonFatal * account of incorrectly-interpreted or malformed flag values. * * @param message A string name of the flag for which parsing failed. - * @param cause The underlying [[java.lang.Throwable]] that caused this exception. + * @param cause The underlying `java.lang.Throwable` that caused this exception. */ case class FlagParseException(message: String, cause: Throwable = null) extends Exception(message, cause) @@ -347,7 +347,7 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) * @param includeGlobal If true, all registered * [[com.twitter.app.GlobalFlag GlobalFlags]] will be included * in output. Defaults to the `includeGlobal` settings of this instance. - * @param classLoader The [[java.lang.ClassLoader]] used to fetch + * @param classLoader The `java.lang.ClassLoader` used to fetch * [[com.twitter.app.GlobalFlag GlobalFlags]], if necessary. Defaults to this * instance's classloader. * @return All of the flags known to this this Flags instance. diff --git a/util-app/src/main/scala/com/twitter/app/Lifecycle.scala b/util-app/src/main/scala/com/twitter/app/Lifecycle.scala index 454e55d1bc..84be72f680 100644 --- a/util-app/src/main/scala/com/twitter/app/Lifecycle.scala +++ b/util-app/src/main/scala/com/twitter/app/Lifecycle.scala @@ -15,18 +15,18 @@ private[app] trait Lifecycle { self: App => private[this] val notifier = new Notifier(observers.asScala) /** - * Notifies [[Observer observers]] of [[com.twitter.app.lifecycle.Event events]] + * Notifies `Observer`s of `Event`s. */ protected final def observe(event: Event)(f: => Unit): Unit = notifier(event)(f) /** - * Notifies [[Observer observers]] of [[Future]] [[com.twitter.app.lifecycle.Event events]] + * Notifies `Observer`s of [[com.twitter.util.Future]] `Event`s. */ protected final def observeFuture(event: Event)(f: Future[Unit]): Future[Unit] = notifier.future(event)(f) - /** Add a lifecycle [[com.twitter.app.lifecycle.Event]] [[Observer]] */ + /** Add a lifecycle [[com.twitter.app.lifecycle.Event]] [[com.twitter.app.lifecycle.Observer]] */ private[twitter] final def withObserver(observer: Observer): Unit = observers.add(observer) } diff --git a/util-app/src/main/scala/com/twitter/app/LoadService.scala b/util-app/src/main/scala/com/twitter/app/LoadService.scala index 32c3eaadd2..ef65a48e40 100644 --- a/util-app/src/main/scala/com/twitter/app/LoadService.scala +++ b/util-app/src/main/scala/com/twitter/app/LoadService.scala @@ -13,11 +13,11 @@ import scala.reflect.ClassTag import scala.util.control.NonFatal /** - * Load classes in the manner of [[java.util.ServiceLoader]]. It is + * Load classes in the manner of `java.util.ServiceLoader`. It is * more resilient to varying Java packaging configurations than ServiceLoader. * * For further control, see [[App.loadServiceBindings]] and the - * [[loadServiceDenied]] flag. + * `loadServiceDenied` flag. */ object LoadService { @@ -80,11 +80,11 @@ object LoadService { * * The public API entry point for this is [[App.loadServiceBindings]]. * - * If [[bind]] is called for a `Class[T]` after [[LoadService.apply]] + * If `bind` is called for a `Class[T]` after [[LoadService.apply]] * has been called for that same interface, an `IllegalStateException` * will be thrown. * - * If [[bind]] is called multiple times for the same interface, the + * If `bind` is called multiple times for the same interface, the * last one registered is used, but warnings and lint violations will * be emitted. * @@ -113,7 +113,7 @@ object LoadService { } /** - * Return any interfaces that have been registered multiple times via [[bind]]. + * Return any interfaces that have been registered multiple times via `bind`. */ def duplicateBindings: Set[Class[_]] = bindDupes.keySet.asScala.toSet @@ -142,7 +142,7 @@ object LoadService { * Returns classes for the given `ClassTag` as specified by * resource files in `META-INF/services/FullyQualifiedClassTagsClassName`. * - * @see [[apply(Class)]] for a Java friendly API. + * @see [[apply[T](iface:Class[T]):Seq[T]*]] for a Java friendly API. */ def apply[T: ClassTag](): Seq[T] = { val iface = implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]] diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala b/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala index ab0eb7f602..6dc3678dac 100644 --- a/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala @@ -6,12 +6,12 @@ import com.twitter.util.Future import java.util.concurrent.Callable /** - * A [[com.twitter.cache.FutureCache]] backed by a - * [[com.google.common.cache.Cache]]. + * A `com.twitter.cache.FutureCache` backed by a + * `com.google.common.cache.Cache`. * * Any correct implementation should make sure that you evict failed results, * and don't interrupt the underlying request that has been fired off. - * [[EvictingCache]] and [[Future#interrupting]] are useful tools for building + * [[EvictingCache$]] and interrupting [[com.twitter.util.Future]]s are useful tools for building * correct FutureCaches. A reference implementation for caching the results of * an asynchronous function with a guava Cache can be found at * [[GuavaCache$.fromCache]]. @@ -27,11 +27,11 @@ class GuavaCache[K, V](cache: GCache[K, Future[V]]) extends ConcurrentMapCache[K /** * A [[com.twitter.cache.FutureCache]] backed by a - * [[com.google.common.cache.LoadingCache]]. + * `com.google.common.cache.LoadingCache`. * Any correct implementation should make sure that you evict failed results, * and don't interrupt the underlying request that has been fired off. - * [[EvictingCache]] and [[Future#interrupting]] are useful tools for building + * [[EvictingCache$]] and interrupting [[com.twitter.util.Future]]s are useful tools for building * correct FutureCaches. A reference implementation for caching the results of * an asynchronous function with a guava LoadingCache can be found at * [[GuavaCache$.fromLoadingCache]]. @@ -49,7 +49,7 @@ object GuavaCache { /** * Creates a function which properly handles the asynchronous behavior of - * [[com.google.common.cache.LoadingCache]]. + * `com.google.common.cache.LoadingCache`. */ def fromLoadingCache[K, V](cache: LoadingCache[K, Future[V]]): K => Future[V] = { val evicting = EvictingCache.lazily(new LoadingFutureCache(cache)); @@ -58,7 +58,7 @@ object GuavaCache { /** * Creates a function which caches the results of `fn` in a - * [[com.google.common.cache.Cache]]. + * `com.google.common.cache.Cache`. */ def fromCache[K, V](fn: K => Future[V], cache: GCache[K, Future[V]]): K => Future[V] = FutureCache.default(fn, new GuavaCache(cache)) diff --git a/util-cache/src/main/scala/com/twitter/cache/ConcurrentMapCache.scala b/util-cache/src/main/scala/com/twitter/cache/ConcurrentMapCache.scala index ba14ccb4ad..9103966d22 100644 --- a/util-cache/src/main/scala/com/twitter/cache/ConcurrentMapCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/ConcurrentMapCache.scala @@ -5,14 +5,14 @@ import java.util.concurrent.ConcurrentMap /** * A [[com.twitter.cache.FutureCache]] backed by a - * [[java.util.concurrent.ConcurrentMap]] + * `java.util.concurrent.ConcurrentMap` * * Any correct implementation should make sure that you evict failed * results, and don't interrupt the underlying request that has been - * fired off. [[EvictingCache]] and [[Future#interrupting]] are + * fired off. [[EvictingCache$]] and interrupting [[com.twitter.util.Future]]s are * useful tools for building correct FutureCaches. A reference * implementation for caching the results of an asynchronous function - * with a [[ConcurrentMap]] can be found at [[FutureCache$.fromMap]]. + * with a `ConcurrentMap` can be found at [[FutureCache$.fromMap]]. */ class ConcurrentMapCache[K, V](underlying: ConcurrentMap[K, Future[V]]) extends FutureCache[K, V] { def get(key: K): Option[Future[V]] = Option(underlying.get(key)) diff --git a/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala b/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala index 9646e664af..5e370904c5 100644 --- a/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala @@ -10,7 +10,7 @@ import java.util.concurrent.ConcurrentMap * * Any correct implementation should make sure that you evict failed * results, and don't interrupt the underlying request that has been - * fired off. [[EvictingCache]] and [[Future#interrupting]] are + * fired off. [[EvictingCache$]] and interrupting [[com.twitter.util.Future]]s are * useful tools for building correct FutureCaches. A reference * implementation for caching the results of an asynchronous function * can be found at [[FutureCache$.default]]. @@ -87,8 +87,8 @@ abstract class FutureCacheProxy[K, V](underlying: FutureCache[K, V]) extends Fut object FutureCache { /** - * A [[com.twitter.cache.FutureCache]] backed by a - * [[java.util.concurrent.ConcurrentMap]]. + * A `com.twitter.cache.FutureCache` backed by a + * `java.util.concurrent.ConcurrentMap`. */ def fromMap[K, V](fn: K => Future[V], map: ConcurrentMap[K, Future[V]]): K => Future[V] = default(fn, new ConcurrentMapCache(map)) diff --git a/util-cache/src/main/scala/com/twitter/cache/Refresh.scala b/util-cache/src/main/scala/com/twitter/cache/Refresh.scala index e94b7b6766..9c13a0c7ee 100644 --- a/util-cache/src/main/scala/com/twitter/cache/Refresh.scala +++ b/util-cache/src/main/scala/com/twitter/cache/Refresh.scala @@ -33,7 +33,7 @@ object Refresh { * * From Java: * Refresh.every(Duration.fromSeconds(3600), new Function0>() { - * @Override + * \@Override * public Future apply() { * ... * } diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala b/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala index 53a51c946e..f8a2ece734 100644 --- a/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala @@ -11,7 +11,7 @@ import java.util.function.Function * * Any correct implementation should make sure that you evict failed results, * and don't interrupt the underlying request that has been fired off. - * [[EvictingCache]] and [[Future#interrupting]] are useful tools for building + * [[EvictingCache$]] and interrupting [[com.twitter.util.Future]]s are useful tools for building * correct FutureCaches. A reference implementation for caching the results of * an asynchronous function with a caffeine Cache can be found at * [[CaffeineCache$.fromCache]]. @@ -32,7 +32,7 @@ class CaffeineCache[K, V](cache: Cache[K, Future[V]]) * Any correct implementation should make sure that you evict failed results, * and don't interrupt the underlying request that has been fired off. - * [[EvictingCache]] and [[Future#interrupting]] are useful tools for building + * [[EvictingCache$]] and interrupting [[com.twitter.util.Future]]s are useful tools for building * correct FutureCaches. A reference implementation for caching the results of * an asynchronous function with a caffeine LoadingCache can be found at * [[CaffeineCache$.fromLoadingCache]]. diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala index 8fb6807a5b..d22e7fc462 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/NamedPoolThreadFactory.scala @@ -4,10 +4,10 @@ import java.util.concurrent.ThreadFactory import java.util.concurrent.atomic.AtomicInteger /** - * A [[java.util.concurrent.ThreadFactory]] which creates threads with a name + * A `java.util.concurrent.ThreadFactory` which creates threads with a name * indicating the pool from which they originated. * - * A new [[java.lang.ThreadGroup]] (named `name`) is created as a sub-group of + * A new `java.lang.ThreadGroup` (named `name`) is created as a sub-group of * whichever group to which the thread that created the factory belongs. Each * thread created by this factory will be a member of this group and have a * unique name including the group name and an monotonically increasing number. diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala index f48510da44..4556a74cf1 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Offer.scala @@ -185,7 +185,7 @@ object Offer { * A constant offer: synchronizes the given value always. This is * call-by-name and a new value is produced for each `prepare()`. * - * Note: Updates here must also be done at [[com.twitter.concurrent.Offers.newConstOffer()]]. + * Note: Updates here must also be done at `Offers.newConstOffer`. */ def const[T](x: => T): Offer[T] = new Offer[T] { def prepare(): Future[Tx[T]] = Future.value(Tx.const(x)) diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala index 00a00838b5..7f3d897f7e 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala @@ -8,7 +8,7 @@ import java.util.logging.{Level, Logger} import scala.collection.mutable /** - * An interface for scheduling [[java.lang.Runnable]] tasks. + * An interface for scheduling `java.lang.Runnable` tasks. */ trait Scheduler { @@ -32,7 +32,7 @@ trait Scheduler { * Total time spent doing [[blocking]] operations, in nanoseconds. * * This should only include time spent on threads where - * [[CanAwait.trackElapsedBlocking]] returns `true`. + * [[com.twitter.util.Awaitable.CanAwait.trackElapsedBlocking]] returns `true`. * * @return -1 if the [[Scheduler]] does not support tracking this. * @@ -258,7 +258,7 @@ class LocalScheduler(lifo: Boolean) extends Scheduler { /** * A named Scheduler mix-in that causes submitted tasks to be dispatched according to - * an [[java.util.concurrent.ExecutorService]] created by an abstract factory + * an `java.util.concurrent.ExecutorService` created by an abstract factory * function. */ trait ExecutorScheduler { self: Scheduler => diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala index 970b6d0f91..9e11de9fc9 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Tx.scala @@ -50,7 +50,7 @@ object Tx { /** * A `Tx` that will always commit the given value immediately. * - * Note: Updates here must also be done at [[com.twitter.concurrent.Txs.newConstTx()]]. + * Note: Updates here must also be done at `Txs.newConstTx`. */ def const[T](msg: T): Tx[T] = new Tx[T] { def ack() = Future.value(Commit(msg)) diff --git a/util-core/src/main/java/com/twitter/service/VirtualMachineErrorTerminator.java b/util-core/src/main/java/com/twitter/service/VirtualMachineErrorTerminator.java index c6910d53e4..5f7fd9e649 100644 --- a/util-core/src/main/java/com/twitter/service/VirtualMachineErrorTerminator.java +++ b/util-core/src/main/java/com/twitter/service/VirtualMachineErrorTerminator.java @@ -5,8 +5,8 @@ /** * Utility class to terminate the JVM in case of a virtual machine error. * Classes wishing to use should initialize it early using - * {@link #initialize()}, as it might be unable to initialize properly after - * the virtual machine error has been thrown. See {@link #checkTerminating()} + * `initialize`, as it might be unable to initialize properly after + * the virtual machine error has been thrown. See `checkTerminating` * for the typical usage pattern. * @author Attila Szegedi */ diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala index af5a15b99d..24b46d63ff 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala @@ -32,7 +32,7 @@ object AsyncMeter { * * This is equivalent to `AsyncMeter.newMeter(permits, 1.second, maxWaiters)`. * - * @see [[perSecondUnbounded()]] for creating an async meter that allows unbounded waiters + * @see [[perSecondUnbounded]] for creating an async meter that allows unbounded waiters */ def perSecond(permits: Int, maxWaiters: Int)(implicit timer: Timer): AsyncMeter = newMeter(permits, Duration.fromSeconds(1), maxWaiters) @@ -44,7 +44,7 @@ object AsyncMeter { * This meter is not bounded by the number of allowed waiters. This is equivalent to * `AsyncMeter.newUnboundedMeter(permits, 1.second, maxWaiters)`. * - * @see [[perSecond()]] for creating an async meter that only allows a certain number of waiters + * @see [[perSecond]] for creating an async meter that only allows a certain number of waiters */ def perSecondUnbounded(permits: Int)(implicit timer: Timer): AsyncMeter = newUnboundedMeter(permits, Duration.fromSeconds(1)) @@ -220,16 +220,16 @@ class AsyncMeter private ( * If the returned [[com.twitter.util.Future]] is interrupted, we * try to cancel it. If it's successfully cancelled, the * [[com.twitter.util.Future]] is satisfied with a - * [[java.util.concurrent.CancellationException]], and the permits + * `java.util.concurrent.CancellationException`, and the permits * will not be issued, so a subsequent waiter can take advantage * of the permits. * * If `await` is invoked when there are already `maxWaiters` waiters waiting * for permits, the [[com.twitter.util.Future]] is immediately satisfied with - * a [[java.util.concurrent.RejectedExecutionException]]. + * a `java.util.concurrent.RejectedExecutionException`. * * If more permits are requested than `burstSize` then it returns a failed - * [[java.lang.IllegalArgumentException]] [[com.twitter.util.Future]] + * `java.lang.IllegalArgumentException` [[com.twitter.util.Future]] * immediately. */ def await(permits: Int): Future[Unit] = { diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala index 03f37a2bba..ae0d1d50ff 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala @@ -25,8 +25,8 @@ object AsyncQueue { /** * An asynchronous FIFO queue. In addition to providing [[offer]] - * and [[poll]], the queue can be [[fail "failed"]], flushing current - * pollers. + * and [[poll]], the queue can be [[fail(exc:Throwable,discard:Boolean):Unit* "failed"]], + * flushing current pollers. * * @param maxPendingOffers optional limit on the number of pending `offers`. * The default is unbounded, but any other positive value can be used to limit @@ -47,7 +47,7 @@ class AsyncQueue[T](maxPendingOffers: Int) { /** * An asynchronous, unbounded, FIFO queue. In addition to providing [[offer]] - * and [[poll]], the queue can be [[fail "failed"]], flushing current + * and [[poll]], the queue can be [[fail(exc:Throwable):Unit* "failed"]], flushing current * pollers. */ def this() = this(AsyncQueue.UnboundedCapacity) @@ -127,7 +127,7 @@ class AsyncQueue[T](maxPendingOffers: Int) { /** * Drains any pending elements into a `Try[Queue]`. * - * If the queue has been [[fail failed]] and is now empty, + * If the queue has been [[fail(exc:Throwable):Unit* failed]] and is now empty, * a `Throw` of the exception used to fail will be returned. * Otherwise, return a `Return(Queue)` of the pending elements. */ diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 72975f3b9d..2daf0e3ecd 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -639,7 +639,7 @@ object AsyncStream { fromFuture[A](Future.exception(e)) /** - * Transformation (or lift) from [[Seq]] into `AsyncStream`. + * Transformation (or lift) from `Seq` into `AsyncStream`. */ def fromSeq[A](seq: Seq[A]): AsyncStream[A] = seq match { case Nil => empty @@ -648,13 +648,13 @@ object AsyncStream { } /** - * Transformation (or lift) from [[Future]] into `AsyncStream`. + * Transformation (or lift) from [[com.twitter.util.Future]] into `AsyncStream`. */ def fromFuture[A](f: Future[A]): AsyncStream[A] = FromFuture(f) /** - * Transformation (or lift) from [[Option]] into `AsyncStream`. + * Transformation (or lift) from `Option` into `AsyncStream`. */ def fromOption[A](o: Option[A]): AsyncStream[A] = o match { diff --git a/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala b/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala index 0e9e2da858..8ca387897b 100644 --- a/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala @@ -4,7 +4,7 @@ import com.twitter.util.Duration import java.util.concurrent.TimeUnit /** - * Implicits for writing readable [[Duration]]s. + * Implicits for writing readable [[com.twitter.util.Duration]]s. * * @example * {{{ diff --git a/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala b/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala index 72044afcfc..28ee0ef9a5 100644 --- a/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala @@ -3,7 +3,7 @@ package com.twitter.conversions import com.twitter.util.StorageUnit /** - * Implicits for writing readable [[StorageUnit]]s. + * Implicits for writing readable [[com.twitter.util.StorageUnit]]s. * * @example * {{{ diff --git a/util-core/src/main/scala/com/twitter/conversions/StringOps.scala b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala index adef1f1edc..e19d52d16d 100644 --- a/util-core/src/main/scala/com/twitter/conversions/StringOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala @@ -82,7 +82,7 @@ object StringOps { /** * Unquote an ASCII string that has been quoted in a style like - * [[quoteC()]] and convert it into a standard unicode string. + * [[quoteC():String*]] and convert it into a standard unicode string. * `"\\uHHHH"` and `"\xHH"` expressions are unpacked * into unicode characters, as well as `"\r"`, `"\n"`, * `"\t"`, `"\\"`, and `'\"'`. diff --git a/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala index d9e7b31cb7..c4cee43411 100644 --- a/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala +++ b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala @@ -4,9 +4,9 @@ object U64Ops { /** * Parses this HEX string as an unsigned 64-bit long value. Be careful, this can throw - * [[NumberFormatException]]. + * `NumberFormatException`. * - * @see [[java.lang.Long.parseUnsignedLong()]] + * See `java.lang.Long.parseUnsignedLong()` for more information. */ implicit class RichU64String(val self: String) extends AnyVal { def toU64Long: Long = java.lang.Long.parseUnsignedLong(self, 16) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 61ef6400e3..e1dc9080b4 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -37,9 +37,10 @@ abstract class Buf { outer => * want to manually set the first bytes of an `Array[Byte]` and then efficiently copy the * contents of this `Buf` to the remaining space. * - * @see [[write(ByteBuffer)]] for writing to nio buffers. + * @see [[com.twitter.io.Buf.write(output:java\.nio\.ByteBuffer):Unit* write(ByteBuffer)]] + * for writing to nio buffers. * - * @throws IllegalArgumentException when `output` is too small to + * @note Throws `IllegalArgumentException` when `output` is too small to * contain all the data. * * @note [[Buf]] implementors should use the helper [[checkWriteArgs]]. @@ -57,12 +58,13 @@ abstract class Buf { outer => * the data is destined for an IO operation, it may be preferable to provide a direct * nio `ByteBuffer` to ensure the avoidance of intermediate heap-based representations. * - * @see [[write(Array[Byte], Int)]] for writing to byte arrays. + * @see [[com.twitter.io.Buf.write(output:Array[Byte],off:Int):Unit* write(Array[Byte],Int)]] + * for writing to byte arrays. * - * @throws java.lang.IllegalArgumentException when `output` doesn't have enough + * @note Throws `java.lang.IllegalArgumentException` when `output` doesn't have enough * space as defined by `ByteBuffer.remaining()` to hold the contents of this `Buf`. * - * @throws ReadOnlyBufferException if the provided buffer is read-only. + * @note Throws `ReadOnlyBufferException` if the provided buffer is read-only. * * @note [[Buf]] implementors should use the helper [[checkWriteArgs]]. */ @@ -261,7 +263,7 @@ abstract class Buf { outer => protected[this] def isSliceIdentity(from: Int, until: Int): Boolean = from == 0 && until >= length - /** Helps implementations validate the arguments to [[write]]. */ + /** Helps implementations validate the arguments to [[Buf.write(output:Array[Byte],off:Int):Unit*]]. */ protected[this] def checkWriteArgs(outputLen: Int, outputOff: Int): Unit = { if (outputOff < 0) throw new IllegalArgumentException(s"offset must be non-negative: $outputOff") @@ -298,7 +300,7 @@ object Buf { * @return `true` if the processor would like to continue processing * more bytes and `false` otherwise. * - * @see [[Buf.process]] + * @see [[com.twitter.io.Buf.process(processor:com\.twitter\.io\.Buf\.Processor):Int* process(Processor)]] * * @note this is not a `Function1[Byte, Boolean]` despite very * much fitting that interface. This was done to avoiding boxing @@ -884,7 +886,7 @@ object Buf { object ByteBuffer { - /** Extract a read-only view of the underlying [[java.nio.ByteBuffer]]. */ + /** Extract a read-only view of the underlying `java.nio.ByteBuffer`. */ def unapply(buf: ByteBuffer): Some[java.nio.ByteBuffer] = Some(buf.underlying.asReadOnlyBuffer) @@ -917,13 +919,13 @@ object Buf { // it prevents direct access to its underlying byte array. /** - * Create a Buf.ByteBuffer by directly wrapping the provided [[java.nio.ByteBuffer]]. + * Create a Buf.ByteBuffer by directly wrapping the provided `java.nio.ByteBuffer`. */ def apply(bb: java.nio.ByteBuffer): Buf = if (bb.remaining == 0) Buf.Empty else new ByteBuffer(bb) - /** Extract the buffer's underlying [[java.nio.ByteBuffer]]. */ + /** Extract the buffer's underlying `java.nio.ByteBuffer`. */ def unapply(buf: ByteBuffer): Some[java.nio.ByteBuffer] = Some(buf.underlying) /** @@ -1067,7 +1069,7 @@ object Buf { * Create and deconstruct Utf-8 encoded buffers. * * @note Malformed and unmappable input is silently replaced - * see [[java.nio.charset.CodingErrorAction.REPLACE]] + * see `java.nio.charset.CodingErrorAction.REPLACE` * * @note See `com.twitter.io.Bufs.UTF_8` for a Java-friendly API. */ @@ -1077,7 +1079,7 @@ object Buf { * Create and deconstruct 16-bit UTF buffers. * * @note Malformed and unmappable input is silently replaced - * see [[java.nio.charset.CodingErrorAction.REPLACE]] + * see `java.nio.charset.CodingErrorAction.REPLACE` * * @note See `com.twitter.io.Bufs.UTF_16` for a Java-friendly API. */ @@ -1088,7 +1090,7 @@ object Buf { * with big-endian byte order. * * @note Malformed and unmappable input is silently replaced - * see [[java.nio.charset.CodingErrorAction.REPLACE]] + * see `java.nio.charset.CodingErrorAction.REPLACE` * * @note See `com.twitter.io.Bufs.UTF_16BE` for a Java-friendly API. */ @@ -1099,7 +1101,7 @@ object Buf { * with little-endian byte order. * * @note Malformed and unmappable input is silently replaced - * see [[java.nio.charset.CodingErrorAction.REPLACE]] + * see `java.nio.charset.CodingErrorAction.REPLACE` * * @note See `com.twitter.io.Bufs.UTF_16LE` for a Java-friendly API. */ @@ -1110,7 +1112,7 @@ object Buf { * ISO Latin Alphabet No. 1 charset. * * @note Malformed and unmappable input is silently replaced - * see [[java.nio.charset.CodingErrorAction.REPLACE]] + * see `java.nio.charset.CodingErrorAction.REPLACE` * * @note See `com.twitter.io.Bufs.ISO_8859_1` for a Java-friendly API. */ @@ -1122,19 +1124,19 @@ object Buf { * Unicode character set. * * @note Malformed and unmappable input is silently replaced - * see [[java.nio.charset.CodingErrorAction.REPLACE]] + * see `java.nio.charset.CodingErrorAction.REPLACE` * * @note See `com.twitter.io.Bufs.US_ASCII` for a Java-friendly API. */ object UsAscii extends StringCoder(JChar.US_ASCII) /** - * A [[StringCoder]] for a given [[java.nio.charset.Charset]] provides an - * [[apply(String) encoder]]: `String` to [[Buf]] - * and an [[unapply(Buf) extractor]]: [[Buf]] to `Some[String]`. + * A [[StringCoder]] for a given `java.nio.charset.Charset` provides an + * [[apply encoder]]: `String` to [[Buf]] + * and an [[unapply extractor]]: [[Buf]] to `Some[String]`. * * @note Malformed and unmappable input is silently replaced - * see [[java.nio.charset.CodingErrorAction.REPLACE]] + * see `java.nio.charset.CodingErrorAction.REPLACE` * * @see [[Utf8]] for UTF-8 encoding and decoding, and `Bufs.UTF8` for * Java users. Constants exist for other standard charsets as well. diff --git a/util-core/src/main/scala/com/twitter/io/BufReader.scala b/util-core/src/main/scala/com/twitter/io/BufReader.scala index 75d513c6e9..d1a9415323 100644 --- a/util-core/src/main/scala/com/twitter/io/BufReader.scala +++ b/util-core/src/main/scala/com/twitter/io/BufReader.scala @@ -115,7 +115,7 @@ object BufReader { new Framed(r, new ChunkedFramer(chunkSize)) /** - * Wraps a [[ Reader[Buf] ]] and emits frames as decided by `framer`. + * Wraps a [[Reader]] and emits frames as decided by `framer`. * * @note The returned `Reader` may not be thread safe depending on the behavior * of the framer. diff --git a/util-core/src/main/scala/com/twitter/io/ByteReader.scala b/util-core/src/main/scala/com/twitter/io/ByteReader.scala index 3d71044d35..eb6f5ae3df 100644 --- a/util-core/src/main/scala/com/twitter/io/ByteReader.scala +++ b/util-core/src/main/scala/com/twitter/io/ByteReader.scala @@ -184,14 +184,14 @@ trait ByteReader extends AutoCloseable { * Extract exactly the specified number of bytes into a `String` using the * specified `Charset`, advancing the byte cursor by `bytes`. * - * @throws UnderflowException if there are < `bytes` bytes available + * @throws ByteReader.UnderflowException if there are < `bytes` bytes available */ def readString(bytes: Int, charset: Charset): String /** * Skip over the next `n` bytes. * - * @throws UnderflowException if there are < `n` bytes available + * @throws ByteReader.UnderflowException if there are < `n` bytes available */ def skip(n: Int): Unit diff --git a/util-core/src/main/scala/com/twitter/io/Charsets.scala b/util-core/src/main/scala/com/twitter/io/Charsets.scala index 42a738b952..88960097de 100644 --- a/util-core/src/main/scala/com/twitter/io/Charsets.scala +++ b/util-core/src/main/scala/com/twitter/io/Charsets.scala @@ -4,7 +4,7 @@ import java.nio.charset.{Charset, CharsetDecoder, CharsetEncoder, CodingErrorAct import java.util /** - * A set of [[java.nio.charset.Charset]] utilities. + * A set of `java.nio.charset.Charset` utilities. */ object Charsets { @@ -17,10 +17,10 @@ object Charsets { } /** - * Returns a cached thread-local [[java.nio.charset.CharsetEncoder]] for + * Returns a cached thread-local `java.nio.charset.CharsetEncoder` for * the specified `charset`. * @note Malformed and unmappable input is silently replaced - * see [[java.nio.charset.CodingErrorAction.REPLACE]] + * see `java.nio.charset.CodingErrorAction.REPLACE` */ def encoder(charset: Charset): CharsetEncoder = { var e = encoders.get.get(charset) @@ -34,10 +34,10 @@ object Charsets { } /** - * Returns a cached thread-local [[java.nio.charset.CharsetDecoder]] for + * Returns a cached thread-local `java.nio.charset.CharsetDecoder` for * the specified `charset`. * @note Malformed and unmappable input is silently replaced - * see [[java.nio.charset.CodingErrorAction.REPLACE]] + * see `java.nio.charset.CodingErrorAction.REPLACE` */ def decoder(charset: Charset): CharsetDecoder = { var d = decoders.get.get(charset) diff --git a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala index ffc9091a5c..6fc175cd7b 100644 --- a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala +++ b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala @@ -8,7 +8,7 @@ import java.io.InputStream * Provides the [[Reader]] API for an `InputStream`. * * The given `InputStream` will be closed when [[Reader.read]] - * reaches the EOF or a call to [[discard()]] or [[close()]]. + * reaches the EOF or a call to [[discard]] or [[[close()*]]]. */ class InputStreamReader private[io] (inputStream: InputStream, chunkSize: Int, pool: FuturePool) extends Reader[Buf] @@ -27,7 +27,7 @@ class InputStreamReader private[io] (inputStream: InputStream, chunkSize: Int, p /** * Asynchronously read at most min(`n`, `maxBufferSize`) bytes from - * the `InputStream`. The returned [[Future]] represents the results of + * the `InputStream`. The returned [[com.twitter.util.Future]] represents the results of * the read operation. Any failure indicates an error; an empty buffer * indicates that the stream has completed. * @@ -91,15 +91,15 @@ object InputStreamReader { /** * Create an [[InputStreamReader]] from the given `InputStream` - * using [[FuturePool.interruptibleUnboundedPool]] for executing - * all I/O. + * using [[com.twitter.util.FuturePool.interruptibleUnboundedPool]] + * for executing all I/O. */ def apply(inputStream: InputStream, chunkSize: Int = DefaultMaxBufferSize): InputStreamReader = new InputStreamReader(inputStream, chunkSize) /** * Create an [[InputStreamReader]] from the given `InputStream` - * using the provided [[FuturePool]] for executing all I/O. The resulting + * using the provided [[com.twitter.util.FuturePool]] for executing all I/O. The resulting * [[InputStreamReader]] emits chunks of at most `DefaultMaxBufferSize`. */ def apply(inputStream: InputStream, pool: FuturePool): InputStreamReader = diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index e8c87202bd..7f257775b6 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -21,8 +21,8 @@ import com.twitter.util.{Future, Promise, Return, Throw, Time, Timer} * * Reads and writes on the pipe are matched one to one and only one outstanding `read` * or `write` is permitted in the current implementation (multiple pending writes or reads - * resolve into [[IllegalStateException]] while leaving the pipe healthy). That is, the `write` - * (its returned [[Future]]) is resolved when the `read` consumes the written data. + * resolve into `IllegalStateException` while leaving the pipe healthy). That is, the `write` + * (its returned [[com.twitter.util.Future]]) is resolved when the `read` consumes the written data. * * Here is, for example, a very typical write-loop that writes into a pipe-backed [[Writer]]: * @@ -55,8 +55,8 @@ import com.twitter.util.{Future, Promise, Return, Throw, Time, Timer} * The following behavior is expected with regards to reading from or writing into a closed or a * failed pipe: * - * - Writing into a closed pipe yields [[IllegalStateException]] - * - Reading from a closed pipe yields EOF ([[Future.None]]) + * - Writing into a closed pipe yields `IllegalStateException` + * - Reading from a closed pipe yields EOF ([[com.twitter.util.Future.None]]) * - Reading from or writing into a failed pipe yields a failure it was failed with * * It's also worth discussing how pipes are being closed. As closure is always initiated by a @@ -65,8 +65,9 @@ import com.twitter.util.{Future, Promise, Return, Throw, Time, Timer} * * The following rules should help reasoning about closure signals in pipes: * - * - Closing a pipe with a pending read resolves said read into EOF and returns a [[Future.Unit]] - * - Closing a pipe with a pending write by default fails said write with [[IllegalStateException]] and + * - Closing a pipe with a pending read resolves said read into EOF and returns a + * [[com.twitter.util.Future.Unit]] + * - Closing a pipe with a pending write by default fails said write with `IllegalStateException` and * returns a future that will be satisfied when a consumer observes the closure (EOF) via read. If * a timer is provided, the pipe will wait until the provided deadline for a successful read before * failing the write. diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 979973cf1d..d2c0681750 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -26,13 +26,13 @@ import java.io.{File, FileInputStream, InputStream} * * == Error Handling == * - * Given the read-loop above, its returned [[Future]] could be used to observe both successful and - * unsuccessful outcomes. + * Given the read-loop above, its returned [[com.twitter.util.Future]] could be used to observe both + * successful and unsuccessful outcomes. * * {{{ * consume(stream)(processor).respond { * case Return(()) => println("Consumed an entire stream successfully.") - * case Throw(e) => println(s"Encountered an error while consuming a stream: $e") + * case Throw(e) => println(s"Encountered an error while consuming a stream: \$e") * } * }}} * @@ -51,11 +51,11 @@ import java.io.{File, FileInputStream, InputStream} * * == Back Pressure == * - * The pattern above leverages [[Future]] recursion to exert back-pressure via allowing only one - * outstanding read. It's usually a good idea to structure consumers this way (i.e., the next read - * isn't issued until the previous read finishes). This would always ensure a finer grained - * back-pressure in network systems allowing the consumers to artificially slow down the producers - * and not rely solely on transport and IO buffering. + * The pattern above leverages [[com.twitter.util.Future]] recursion to exert back-pressure via + * allowing only one outstanding read. It's usually a good idea to structure consumers this way + * (i.e., the next read isn't issued until the previous read finishes). This would always ensure a + * finer grained back-pressure in network systems allowing the consumers to artificially slow down + * the producers and not rely solely on transport and IO buffering. * * @note Whether or not multiple outstanding reads are allowed on a `Reader` type is an undefined * behaviour but could be changed in a refinement. @@ -77,16 +77,18 @@ import java.io.{File, FileInputStream, InputStream} trait Reader[+A] { self => /** - * Asynchronously read the next element of this stream. Returned [[Future]] will resolve into - * `Some(e)` when the element is available or into `None` when stream is exhausted. + * Asynchronously read the next element of this stream. Returned [[com.twitter.util.Future]] + * will resolve into `Some(e)` when the element is available or into `None` when stream + * is exhausted. * - * Stream failures are terminal such that all subsequent reads will resolve in failed [[Future]]s. + * Stream failures are terminal such that all subsequent reads will resolve in failed + * [[com.twitter.util.Future]]s. */ def read(): Future[Option[A]] /** * Discard this stream as its output is no longer required. This could be used to signal the - * producer of this stream similarly how [[Future.raise]] used to propagate interrupts across + * producer of this stream similarly how [[com.twitter.util.Future.raise]] used to propagate interrupts across * future chains. * * @note Although unnecessary, it's always safe to discard a fully-consumed stream. @@ -94,7 +96,8 @@ trait Reader[+A] { self => def discard(): Unit /** - * A [[Future]] that resolves once this reader is closed upon reading of end-of-stream. + * A [[com.twitter.util.Future]] that resolves once this reader is closed upon reading + * of end-of-stream. * * If the result is a failed future, this indicates that it was not closed either by reading * until the end of the stream nor by discarding. This is useful for any extra resource cleanup @@ -129,7 +132,6 @@ trait Reader[+A] { self => /** * Converts a `Reader[Reader[B]]` into a `Reader[B]` * - * @define flattenBehavior * @note All operations of the new Reader will be in sync with the outermost Reader. Discarding * one Reader will discard the other Reader. When one Reader's onClose resolves, the other * Reader's onClose will be resolved immediately with the same value. @@ -218,32 +220,32 @@ object Reader { def fromBuf(buf: Buf, chunkSize: Int): Reader[Buf] = BufReader(buf, chunkSize) /** - * Create a new [[Reader]] from a given [[File]]. The output of a returned reader is chunked by + * Create a new [[Reader]] from a given `File`. The output of a returned reader is chunked by * at most `chunkSize` (bytes). * * The resources held by the returned [[Reader]] are released on reading of EOF and - * [[Reader.discard()]]. + * [[Reader.discard]]. * * @see `Readers.fromFile` for a Java API */ def fromFile(f: File): Reader[Buf] = fromFile(f, InputStreamReader.DefaultMaxBufferSize) /** - * Create a new [[Reader]] from a given [[File]]. The output of a returned reader is chunked by + * Create a new [[Reader]] from a given `File`. The output of a returned reader is chunked by * at most `chunkSize` (bytes). * * The resources held by the returned [[Reader]] are released on reading of EOF and - * [[Reader.discard()]]. + * [[Reader.discard]]. * * @see `Readers.fromFile` for a Java API */ def fromFile(f: File, chunkSize: Int): Reader[Buf] = fromStream(new FileInputStream(f), chunkSize) /** - * Create a new [[Reader]] from a given [[Iterator]]. + * Create a new [[Reader]] from a given `Iterator`. * * The resources held by the returned [[Reader]] are released on reading of EOF and - * [[Reader.discard()]]. + * [[Reader.discard]]. * * @note It is not recommended to call `it.next()` after creating a `Reader` from it. * Doing so will affect the behavior of `Reader.read()` because it will skip @@ -252,38 +254,38 @@ object Reader { def fromIterator[A](it: Iterator[A]): Reader[A] = new IteratorReader[A](it) /** - * Create a new [[Reader]] from a given [[InputStream]]. The output of a returned reader is + * Create a new [[Reader]] from a given `InputStream`. The output of a returned reader is * chunked by at most `chunkSize` (bytes). * * The resources held by the returned [[Reader]] are released on reading of EOF and - * [[Reader.discard()]]. + * [[Reader.discard]]. */ def fromStream(s: InputStream): Reader[Buf] = fromStream(s, InputStreamReader.DefaultMaxBufferSize) /** - * Create a new [[Reader]] from a given [[InputStream]]. The output of a returned reader is + * Create a new [[Reader]] from a given `InputStream`. The output of a returned reader is * chunked by at most `chunkSize` (bytes). * * The resources held by the returned [[Reader]] are released on reading of EOF and - * [[Reader.discard()]]. + * [[Reader.discard]]. * * @see `Readers.fromStream` for a Java API */ def fromStream(s: InputStream, chunkSize: Int): Reader[Buf] = InputStreamReader(s, chunkSize) /** - * Create a new [[Reader]] from a given [[Seq]]. + * Create a new [[Reader]] from a given `Seq`. * * The resources held by the returned [[Reader]] are released on reading of EOF and - * [[Reader.discard()]]. + * [[Reader.discard]]. * * @note Multiple outstanding reads are not allowed on this reader. */ def fromSeq[A](seq: Seq[A]): Reader[A] = fromIterator(seq.iterator) /** - * Allow [[AsyncStream]] to be consumed as a [[Reader]] + * Allow [[com.twitter.concurrent.AsyncStream]] to be consumed as a [[Reader]] */ def fromAsyncStream[A](as: AsyncStream[A]): Reader[A] = { val pipe = new Pipe[A] @@ -346,7 +348,11 @@ object Reader { * Convenient abstraction to read from a stream (Reader) of Readers as if it were a single Reader. * @param readers A Reader holds a stream of Reader[A] * - * $flattenBehavior + * @note All operations of the new Reader will be in sync with the outermost Reader. Discarding + * one Reader will discard the other Reader. When one Reader's onClose resolves, the other + * Reader's onClose will be resolved immediately with the same value. + * The subsequent readers are unmanaged, the caller is responsible for discarding those + * when abandoned. */ def flatten[A](readers: Reader[Reader[A]]): Reader[A] = new Reader[A] { self => // access currentReader and curReaderClosep are synchronized on `self` diff --git a/util-core/src/main/scala/com/twitter/io/TempFolder.scala b/util-core/src/main/scala/com/twitter/io/TempFolder.scala index d45067d14d..25b1fd492c 100644 --- a/util-core/src/main/scala/com/twitter/io/TempFolder.scala +++ b/util-core/src/main/scala/com/twitter/io/TempFolder.scala @@ -53,13 +53,13 @@ trait TempFolder { /** * @return The current thread's active temporary folder. - * @throws RuntimeException if not running within a withTempFolder block + * @note Throws `RuntimeException` if not running within a withTempFolder block */ def folderName: String = { _folderName.get.getPath } /** * @return The canonical path of the current thread's active temporary folder. - * @throws RuntimeException if not running within a withTempFolder block + * @note Throws `RuntimeException` if not running within a withTempFolder block */ def canonicalFolderName: String = { _folderName.get.getCanonicalPath } } diff --git a/util-core/src/main/scala/com/twitter/io/Writer.scala b/util-core/src/main/scala/com/twitter/io/Writer.scala index 389aece660..ae4d465dcc 100644 --- a/util-core/src/main/scala/com/twitter/io/Writer.scala +++ b/util-core/src/main/scala/com/twitter/io/Writer.scala @@ -20,22 +20,23 @@ import java.io.OutputStream * * == Closures and Failures == * - * Writers are [[Closable]] and the producer MUST `close` the stream when it finishes or `fail` the - * stream when it encounters a failure and can no longer continue. Streams backed by network - * connections are particularly prone to resource leaks when they aren't cleaned up properly. + * Writers are [[com.twitter.util.Closable]] and the producer MUST `close` the stream when it + * finishes or `fail` the stream when it encounters a failure and can no longer continue. Streams + * backed by network connections are particularly prone to resource leaks when they aren't + * cleaned up properly. * - * Observing stream failures in the producer could be done via a [[Future]] returned form - * a write-loop: + * Observing stream failures in the producer could be done via a [[com.twitter.util.Future]] + * returned form a write-loop: * * {{{ * produce(writer)(generator).respond { * case Return(()) => println("Produced a stream successfully.") - * case Throw(e) => println(s"Could not produce a stream because of a failure: $e") + * case Throw(e) => println(s"Could not produce a stream because of a failure: \$e") * } * }}} * - * @note Encountering a stream failure would terminate the write-loop given the [[Future]] - * recursion semantics. + * @note Encountering a stream failure would terminate the write-loop given the + * [[com.twitter.util.Future]] recursion semantics. * * @note Once failed or closed, a stream can not be restarted such that all future writes will * resolve into a failure. @@ -44,10 +45,10 @@ import java.io.OutputStream * * == Back Pressure == * - * By analogy with read-loops (see [[Reader]] API), write-loops leverage [[Future]] recursion to - * exert back-pressure: the next write isn't issued until the previous write finishes. This will - * always ensure a finer grained back-pressure in network systems allowing the producers to - * adjust the flow rate based on consumer's speed and not on IO buffering. + * By analogy with read-loops (see [[Reader]] API), write-loops leverage[[com.twitter.util.Future]] + * recursion to exert back-pressure: the next write isn't issued until the previous write finishes. + * This will always ensure a finer grained back-pressure in network systems allowing the producers + * to adjust the flow rate based on consumer's speed and not on IO buffering. * * @note Whether or not multiple pending writes are allowed on a `Writer` type is an undefined * behaviour but could be changed in a refinement. @@ -56,23 +57,23 @@ trait Writer[-A] extends Closable { self => /** * Write an `element` into this stream. Although undefined by this contract, a trustworthy - * implementation (such as [[Pipe]]) would do its best to resolve the returned [[Future]] only - * when a consumer observes a written `element`. + * implementation (such as [[Pipe]]) would do its best to resolve the returned + * [[com.twitter.util.Future]] only when a consumer observes a written `element`. * - * The write can also resolve into a failure (failed [[Future]]). + * The write can also resolve into a failure (failed [[com.twitter.util.Future]]). */ def write(element: A): Future[Unit] /** * Fail this stream with a given `cause`. No further writes are allowed, but if happen, will - * resolve into a [[Future]] failed with `cause`. + * resolve into a [[com.twitter.util.Future]] failed with `cause`. * * @note Failing an already closed stream does not have an effect. */ def fail(cause: Throwable): Unit /** - * A [[Future]] that resolves once this writer is closed and flushed. + * A [[com.twitter.util.Future]] that resolves once this writer is closed and flushed. * * It may contain an error if the writer is failed. * diff --git a/util-core/src/main/scala/com/twitter/util/Awaitable.scala b/util-core/src/main/scala/com/twitter/util/Awaitable.scala index 1cec24401e..0f5e44b595 100644 --- a/util-core/src/main/scala/com/twitter/util/Awaitable.scala +++ b/util-core/src/main/scala/com/twitter/util/Awaitable.scala @@ -46,7 +46,8 @@ object Awaitable { * * This should only be enabled for threads where blocking is discouraged. * - * @see [[Scheduler.blockingTimeNanos]] and [[Scheduler.blocking]] + * @see [[com.twitter.concurrent.Scheduler.blockingTimeNanos]] and + * [[com.twitter.concurrent.Scheduler.blocking]] */ def enableBlockingTimeTracking(): Unit = TrackElapsedBlocking.set(java.lang.Boolean.TRUE) @@ -60,7 +61,8 @@ object Awaitable { /** * Whether or not this thread should track time spent in blocking operations. * - * @see [[Scheduler.blockingTimeNanos]] and [[Scheduler.blocking]] + * @see [[com.twitter.concurrent.Scheduler.blockingTimeNanos]] and + * [[com.twitter.concurrent.Scheduler.blocking]] */ def getBlockingTimeTracking: Boolean = { TrackElapsedBlocking.get() != null diff --git a/util-core/src/main/scala/com/twitter/util/Base64Long.scala b/util-core/src/main/scala/com/twitter/util/Base64Long.scala index 94e5bc5577..f2b6c33509 100644 --- a/util-core/src/main/scala/com/twitter/util/Base64Long.scala +++ b/util-core/src/main/scala/com/twitter/util/Base64Long.scala @@ -127,16 +127,16 @@ object Base64Long { /** * Decode a sequence of characters representing a base-64-encoded - * Long, as produced by [[toBase64]]. An empty range of characters - * will yield 0L. Leading "zero" digits will not cause overflow. + * Long, as produced by [[com.twitter.util.Base64Long.toBase64(l:Long):String* toBase64(Long)]]. + * An empty range of characters will yield 0L. Leading "zero" digits will not cause overflow. * * @param alphabet defines the mapping between characters in the * string and 6-bit numbers. * - * @throws IllegalArgumentException if any characters in the specified + * @note Throws `IllegalArgumentException` if any characters in the specified * range are not in the specified base-64 alphabet. * - * @throws ArithmeticException if the resulting value overflows a Long. + * @note Throws `ArithmeticException` if the resulting value overflows a Long. */ def fromBase64( chars: CharSequence, diff --git a/util-core/src/main/scala/com/twitter/util/CloseOnce.scala b/util-core/src/main/scala/com/twitter/util/CloseOnce.scala index 8efa1466be..f27ea6aca6 100644 --- a/util-core/src/main/scala/com/twitter/util/CloseOnce.scala +++ b/util-core/src/main/scala/com/twitter/util/CloseOnce.scala @@ -10,7 +10,7 @@ import scala.util.control.NonFatal * as the first invocation irrespective of the deadline provided. * * @see [[ClosableOnce]] if you are not mixing in or extending an existing [[Closable]] - * @see [[ClosableOnce.of(closable: Closable)]] for creating a proxy to a [[Closable]] + * @see [[ClosableOnce.of]] for creating a proxy to a [[Closable]] * that has already been instantiated. */ trait CloseOnce { self: Closable => @@ -40,9 +40,9 @@ trait CloseOnce { self: Closable => protected def closeOnce(deadline: Time): Future[Unit] /** - * The [[Future]] satisfied upon completion of [[close()]]. + * The [[Future]] satisfied upon completion of [[close]]. * - * @note we do not expose direct access to the underlying [[closePromise Promise]], because + * @note we do not expose direct access to the underlying [[Promise closePromise]], because * the [[Promise]] state is mutable - we only want mutation to occur within this * [[CloseOnce]] trait. */ diff --git a/util-core/src/main/scala/com/twitter/util/Config.scala b/util-core/src/main/scala/com/twitter/util/Config.scala index dfdd2b9b80..fbd834ba98 100644 --- a/util-core/src/main/scala/com/twitter/util/Config.scala +++ b/util-core/src/main/scala/com/twitter/util/Config.scala @@ -178,7 +178,7 @@ trait Config[T] extends (() => T) { } /** - * @throws RequiredValuesMissing if there are any missing values. + * @note Throws `RequiredValuesMissing` if there are any missing values. */ def validate(): Unit = { val missing = missingValues diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index aa3dd18c2a..763abb5f81 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -26,13 +26,13 @@ object Duration extends TimeLikeOps[Duration] { val NanosPerDay: Long = NanosPerHour * 24 /** - * Create a duration from a [[java.util.concurrent.TimeUnit]]. - * Synonym for `apply`. + * Create a duration from a `java.util.concurrent.TimeUnit`. + * Synonym for [[apply]]. */ def fromTimeUnit(value: Long, unit: TimeUnit): Duration = apply(value, unit) /** - * Create a duration from a [[java.util.concurrent.TimeUnit]]. + * Create a duration from a `java.util.concurrent.TimeUnit`. */ def apply(value: Long, unit: TimeUnit): Duration = { val ns = TimeUnit.NANOSECONDS.convert(value, unit) @@ -222,7 +222,7 @@ object Duration extends TimeLikeOps[Duration] { * * The parser is case-insensitive. * - * @throws RuntimeException if the string cannot be parsed. + * @note Throws `RuntimeException` if the string cannot be parsed. */ def parse(s: String): Duration = { val ss = s.toLowerCase diff --git a/util-core/src/main/scala/com/twitter/util/Event.scala b/util-core/src/main/scala/com/twitter/util/Event.scala index ea6701aace..2a9421df64 100644 --- a/util-core/src/main/scala/com/twitter/util/Event.scala +++ b/util-core/src/main/scala/com/twitter/util/Event.scala @@ -18,15 +18,15 @@ import scala.language.higherKinds * To listen to an Event stream for values, register a handler. * * {{{ - * val you = ints.respond { n => println(s"$n for you") } + * val you = ints.respond { n => println(s"\$n for you") } * }}} * - * Handlers in Event-speak are called [[com.twitter.util.Witness Witnesses]]. + * Handlers in Event-speak are called [[Witness Witnesses]]. * And, respond is equivalent to `register(Witness { ... })`, so we can also * write: * * {{{ - * val me = ints.register(Witness { n => println(s"$n for me") }) + * val me = ints.register(Witness { n => println(s"\$n for me") }) * }}} * * Registration is time-sensitive. Witnesses registered to an Event stream @@ -473,11 +473,14 @@ abstract class AbstractWitness[T] extends Witness[T] /** * Note: There is Java-friendly API for this object: [[com.twitter.util.Witnesses]]. + * + * @define VarApplyScaladocLink + * [[com.twitter.util.Var.apply[T](init:T,e:com\.twitter\.util\.Event[T]):com\.twitter\.util\.Var[T]* Var.apply(T, Event[T])]] */ object Witness { /** - * Create a [[Witness]] from an [[AtomicReference atomic reference]]. + * Create a [[Witness]] from an atomic reference. */ def apply[T](ref: AtomicReference[T]): Witness[T] = new Witness[T] { def notify(t: T): Unit = ref.set(t) @@ -491,7 +494,7 @@ object Witness { } /** - * Create a [[Witness]] from a [[Function1]]. + * Create a [[Witness]] from a `Function1`. */ def apply[T](f: T => Unit): Witness[T] = new Witness[T] { def notify(t: T): Unit = f(t) @@ -506,7 +509,7 @@ object Witness { /** * Create a [[Witness]] which keeps only a `java.lang.ref.WeakReference` - * to the [[Function1]]. + * to the `Function1`. * * @see [[Var.patch]] for example. */ @@ -524,7 +527,7 @@ object Witness { * Create a [[Witness]] which keeps only a `java.lang.ref.WeakReference` * to `u`. * - * @see [[Var.apply]] for example. + * @see $VarApplyScaladocLink for example. */ def weakReference[T](u: Updatable[T]): Witness[T] = new Witness[T] { private[this] val weakRef = new WeakReference(u) diff --git a/util-core/src/main/scala/com/twitter/util/Function.scala b/util-core/src/main/scala/com/twitter/util/Function.scala index 7100f09aef..6b7b449684 100644 --- a/util-core/src/main/scala/com/twitter/util/Function.scala +++ b/util-core/src/main/scala/com/twitter/util/Function.scala @@ -51,7 +51,7 @@ object Function { def ofCallable[A](c: Callable[A]): () => A = () => c.call() /** - * Creates a [[Function]] of `T` to `R` from a [[JavaFunction]]. + * Creates a [[Function]] of `T` to `R` from a [[com.twitter.function.JavaFunction]]. * * Allows for better interop with Scala from Java 8 using lambdas. * @@ -99,7 +99,7 @@ object Function { } /** - * Creates a [[Function]] of `T` to `Unit` from a [[JavaConsumer]]. + * Creates a [[Function]] of `T` to `Unit` from a [[com.twitter.function.JavaConsumer]]. * * Allows for better interop with Scala from Java 8 using lambdas. * @@ -123,7 +123,8 @@ object Function { } /** - * Creates an [[ExceptionalFunction]] of `T` to `R` from an [[ExceptionalJavaFunction]]. + * Creates an [[ExceptionalFunction]] of `T` to `R` from an + * [[com.twitter.function.ExceptionalJavaFunction]]. * * Allows for better interop with Scala from Java 8 using lambdas. * @@ -143,7 +144,8 @@ object Function { } /** - * Creates an [[ExceptionalFunction]] of `T` to `Unit` from an [[ExceptionalJavaConsumer]]. + * Creates an [[ExceptionalFunction]] of `T` to `Unit` from an + * [[com.twitter.function.ExceptionalJavaConsumer]]. * * Allows for better interop with Scala from Java 8 using lambdas. * @@ -163,7 +165,8 @@ object Function { } /** - * Creates an [[ExceptionalFunction0]] to `T` from an [[ExceptionalSupplier]]. + * Creates an [[ExceptionalFunction0]] to `T` from an + * [[com.twitter.function.ExceptionalSupplier]]. * * Allows for better interop with Scala from Java 8 using lambdas. * diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 68ce681a9f..20a2350d1b 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -14,6 +14,21 @@ import scala.util.control.NoStackTrace * @see [[Futures]] for Java-friendly APIs. * @see The [[https://twitter.github.io/finagle/guide/Futures.html user guide]] * on concurrent programming with Futures. + * + * @define FutureJoinScaladocLink + * [[[com.twitter.util.Future.join[A](fs:Seq[com\.twitter\.util\.Future[A]]):com\.twitter\.util\.Future[Unit]* join(Seq[A])]]] + * + * @define FuturesJoinScaladocLink + * [[com.twitter.util.Futures.join[A,B](a:com\.twitter\.util\.Future[A],b:com\.twitter\.util\.Future[B]):com\.twitter\.util\.Future[(A,B)]* Futures.join(Future[A],Future[B])]] + * + * @define FutureCollectScaladocLink + * [[[com.twitter.util.Future.collect[A](fs:Seq[com\.twitter\.util\.Future[A]]):com\.twitter\.util\.Future[Seq[A]]* collect(Seq[Future[A]])]]] + * + * @define FutureCollectToTryScaladocLink + * [[[[com.twitter.util.Future.collectToTry[A](fs:Seq[com\.twitter\.util\.Future[A]]):com\.twitter\.util\.Future[Seq[com\.twitter\.util\.Try[A]]]* collectToTry(Seq[Future[A]])]]]] + * + * @define FutureSelectScaladocLink + * [[[com.twitter.util.Future.select[A](fs:Seq[com\.twitter\.util\.Future[A]]):com\.twitter\.util\.Future[(com\.twitter\.util\.Try[A],Seq[com\.twitter\.util\.Future[A]])]* select(Seq[Future])]]] */ object Future { val DEFAULT_TIMEOUT: Duration = Duration.Top @@ -87,7 +102,7 @@ object Future { @inline private final def raiseException = RaiseException /** - * A failed `Future` analogous to [[Predef.???]]. + * A failed `Future` analogous to `Predef.???`. */ def ??? : Future[Nothing] = Future.exception(new NotImplementedError("an implementation is missing")) @@ -258,10 +273,10 @@ object Future { * * @param fs a sequence of Futures * - * @see [[Futures.join]] for a Java friendly API. - * @see [[collect]] if you want to be able to see the results of each `Future`. - * @see [[collectToTry]] if you want to be able to see the results of each - * `Future` regardless of if they succeed or fail. + * @see $FuturesJoinScaladocLink for a Java friendly API. + * @see $FutureCollectScaladocLink if you want to be able to see the results of each `Future`. + * @see $FutureCollectToTryScaladocLink if you want to be able to see the results of each + * `Future` regardless of if they succeed or fail. */ def join[A](fs: AnySeq[Future[A]]): Future[Unit] = { if (fs.isEmpty) Future.Unit @@ -1029,10 +1044,10 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * * @param fs a java.util.List of Futures * - * @see [[Futures.join]] for a Java friendly API. - * @see [[collect]] if you want to be able to see the results of each `Future`. - * @see [[collectToTry]] if you want to be able to see the results of each - * `Future` regardless of if they succeed or fail. + * @see $FuturesJoinScaladocLink for a Java friendly API. + * @see $FutureCollectScaladocLink if you want to be able to see the results of each `Future`. + * @see $FutureCollectToTryScaladocLink if you want to be able to see the results of each + * `Future` regardless of if they succeed or fail. */ def join[A](fs: JList[Future[A]]): Future[Unit] = Futures.join(fs) @@ -1102,9 +1117,9 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * @param fs a sequence of Futures * @return a `Future[Seq[A]]` containing the collected values from fs. * - * @see [[collectToTry]] if you want to be able to see the results of each + * @see $FutureCollectToTryScaladocLink if you want to be able to see the results of each * `Future` regardless of if they succeed or fail. - * @see [[join]] if you are not interested in the results of the individual + * @see $FutureJoinScaladocLink if you are not interested in the results of the individual * `Futures`, only when they are complete. */ def collect[A](fs: AnySeq[Future[A]]): Future[Seq[A]] = @@ -1252,7 +1267,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( * * @param fs cannot be empty * - * @see [[select]] which can be an easier API to use. + * @see $FutureSelectScaladocLink which can be an easier API to use. */ def selectIndex[A](fs: IndexedSeq[Future[A]]): Future[Int] = if (fs.isEmpty) { @@ -2196,7 +2211,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * For example: * {{{ * val p = new Promise[Int]() - * p.setInterruptHandler { case x => println(s"interrupt handler for ${x.getClass}") } + * p.setInterruptHandler { case x => println(s"interrupt handler for \${x.getClass}") } * val f1: Future[Int] = p.mask { * case _: IllegalArgumentException => true * } @@ -2211,7 +2226,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * * @see [[raise]] * @see [[masked]] - * @see [[interruptible()]] + * @see [[interruptible]] */ def mask(pred: PartialFunction[Throwable, Boolean]): Future[A] = { val p = Promise[A]() @@ -2242,7 +2257,7 @@ abstract class Future[+A] extends Awaitable[A] { self => * * @see [[raise]] * @see [[mask]] - * @see [[interruptible()]] + * @see [[interruptible]] */ def masked: Future[A] = mask(Future.AlwaysMasked) diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index 4790f5f6ae..9ba549dd6e 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -75,7 +75,7 @@ object FuturePool { /** * The default future pool, using a cached threadpool, provided by - * [[java.util.concurrent.Executors.newCachedThreadPool]]. Note + * `java.util.concurrent.Executors.newCachedThreadPool`. Note * that this is intended for IO concurrency; computational * parallelism typically requires special treatment. If an interrupt * is raised on a returned Future and the work has started, the worker @@ -89,7 +89,7 @@ object FuturePool { /** * The default future pool, using a cached threadpool, provided by - * [[java.util.concurrent.Executors.newCachedThreadPool]]. Note + * `java.util.concurrent.Executors.newCachedThreadPool`. Note * that this is intended for IO concurrency; computational * parallelism typically requires special treatment. If an interrupt * is raised on a returned Future and the work has started, an attempt diff --git a/util-core/src/main/scala/com/twitter/util/Local.scala b/util-core/src/main/scala/com/twitter/util/Local.scala index 8f1b165926..2f0b3d56ec 100644 --- a/util-core/src/main/scala/com/twitter/util/Local.scala +++ b/util-core/src/main/scala/com/twitter/util/Local.scala @@ -4928,7 +4928,7 @@ object Local { } /** - * A Local is a [[ThreadLocal]] whose scope is flexible. The state of all Locals may + * A Local is a `ThreadLocal` whose scope is flexible. The state of all Locals may * be saved or restored onto the current thread by the user. This is useful for * threading Locals through execution contexts. * diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 0a43527d92..054306811b 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -5,13 +5,23 @@ import scala.annotation.tailrec import scala.runtime.NonLocalReturnControl import scala.util.control.NonFatal +/** + * @define PromiseInterruptsSingleArgLink + * [[com.twitter.util.Promise.interrupts[A](f:com\.twitter\.util\.Future[_]):com\.twitter\.util\.Promise[A]* interrupts(Future)]] + * + * @define PromiseInterruptsTwoArgsLink + * [[com.twitter.util.Promise.interrupts[A](a:com\.twitter\.util\.Future[_],b:com\.twitter\.util\.Future[_]):com\.twitter\.util\.Promise[A]* interrupts(Future,Future)]] + * + * @define PromiseInterruptsVarargsLink + * [[com.twitter.util.Promise.interrupts[A](fs:com\.twitter\.util\.Future[_]*):com\.twitter\.util\.Promise[A]* interrupts(Future*)]] + */ object Promise { /** * Embeds an "interrupt handler" into a [[Promise]]. * - * This is a total handler such that it's defined on any [[Throwable]]. Use - * [[Promise.setInterruptHandler()]] if you need to leave an interrupt handler + * This is a total handler such that it's defined on any `Throwable`. Use + * [[Promise.setInterruptHandler]] if you need to leave an interrupt handler * undefined for certain types of exceptions. * * Example: (`p` and `q` are equivalent, but `p` allocates less): @@ -358,8 +368,8 @@ object Promise { /** * Single-arg version to avoid object creation and take advantage of `forwardInterruptsTo`. * - * @see [[interrupts(Future, Future)]] - * @see [[interrupts(Future*)]] + * @see $PromiseInterruptsTwoArgsLink + * @see $PromiseInterruptsVarargsLink */ def interrupts[A](f: Future[_]): Promise[A] = new Promise[A](f) @@ -367,8 +377,8 @@ object Promise { * Create a promise that interrupts `a` and `b` futures. In particular: * the returned promise handles an interrupt when either `a` or `b` does. * - * @see [[interrupts(Future)]] - * @see [[interrupts(Future*)]] + * @see $PromiseInterruptsSingleArgLink + * @see $PromiseInterruptsVarargsLink */ def interrupts[A](a: Future[_], b: Future[_]): Promise[A] = new Promise[A] with InterruptHandler { @@ -382,8 +392,8 @@ object Promise { * Create a promise that interrupts all of `fs`. In particular: * the returned promise handles an interrupt when any of `fs` do. * - * @see [[interrupts(Future)]] - * @see [[interrupts(Future, Future)]] + * @see $PromiseInterruptsSingleArgLink + * @see $PromiseInterruptsTwoArgsLink */ def interrupts[A](fs: Future[_]*): Promise[A] = new Promise[A] with InterruptHandler { @@ -436,9 +446,9 @@ object Promise { * `Interrupted`, `Transforming`, `Done` and `Linked` where `Interruptible`, * `Interrupted`, and `Transforming` are variants of `Waiting` to deal with future * interrupts. Promises are concurrency-safe, using lock-free operations - * throughout. Callback dispatch is scheduled with [[Scheduler]]. + * throughout. Callback dispatch is scheduled with [[com.twitter.concurrent.Scheduler]]. * - * Waiters (i.e., continuations) are stored in a [[Promise.WaitQueue]] and + * Waiters (i.e., continuations) are stored in a `Promise.WaitQueue` and * executed in the LIFO order. * * `Promise.become` merges two promises: they are declared equivalent. @@ -726,14 +736,14 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ /** * Populate the Promise with the given result. * - * @throws ImmutableResult if the Promise is already populated + * @throws Promise.ImmutableResult if the Promise is already populated */ def setValue(result: A): Unit = update(Return(result)) /** * Populate the Promise with the given exception. * - * @throws ImmutableResult if the Promise is already populated + * @throws Promise.ImmutableResult if the Promise is already populated */ def setException(throwable: Throwable): Unit = update(Throw(throwable)) @@ -749,7 +759,7 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ * value or an exception. setValue and setException are generally * more readable methods to use. * - * @throws ImmutableResult if the Promise is already populated + * @throws Promise.ImmutableResult if the Promise is already populated */ def update(result: Try[A]): Unit = { updateIfEmpty(result) || { diff --git a/util-core/src/main/scala/com/twitter/util/RandomSocket.scala b/util-core/src/main/scala/com/twitter/util/RandomSocket.scala index 2cb02c5355..796a171955 100644 --- a/util-core/src/main/scala/com/twitter/util/RandomSocket.scala +++ b/util-core/src/main/scala/com/twitter/util/RandomSocket.scala @@ -5,7 +5,7 @@ import java.net.{InetSocketAddress, Socket} import scala.util.control.NonFatal /** - * A generator of random local [[java.net.InetSocketAddress]] objects with + * A generator of random local `java.net.InetSocketAddress` objects with * ephemeral ports. */ object RandomSocket { diff --git a/util-core/src/main/scala/com/twitter/util/Stopwatch.scala b/util-core/src/main/scala/com/twitter/util/Stopwatch.scala index acf756c333..9f733e3506 100644 --- a/util-core/src/main/scala/com/twitter/util/Stopwatch.scala +++ b/util-core/src/main/scala/com/twitter/util/Stopwatch.scala @@ -122,7 +122,7 @@ object Stopwatch extends Stopwatch { object Stopwatches { import Stopwatch.Elapsed - /** @see [[Stopwatch.start()]] */ + /** @see [[Stopwatch.start]] */ def start(): Elapsed = Stopwatch.start() @@ -159,7 +159,7 @@ object Stopwatches { * A trivial implementation of [[Stopwatch]] for use as a null * object. * - * All calls to [[Stopwatch.start()]] return an [[Stopwatch.Elapsed]] + * All calls to [[Stopwatch.start]] return an [[Stopwatch.Elapsed]] * instance that always returns [[Duration.Bottom]] for its elapsed time. * * @see `NilStopwatch.get` for accessing this instance from Java. diff --git a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala index 90d0ac2d50..ae90a150ce 100644 --- a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala +++ b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala @@ -69,7 +69,7 @@ object StorageUnit { /** * Representation of storage units. * - * Use either `StorageUnit.fromX` or [[com.twitter.conversions.storage implicit conversions]] + * Use either `StorageUnit.fromX` or `com.twitter.conversions.storage` `implicit conversions` * from `Long` and `Int` to construct instances. * * {{{ diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index da179c4ce6..dcf80ad0d0 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -405,7 +405,7 @@ object Time extends TimeLikeOps[Time] { * * Note that nanosecond precision is only available in JDK versions * greater than JDK8. In JDK8 this API has the same precision as - * [[Time#now]] and [[System#currentTimeMillis]]. In JDK9+ this will + * [[Time#now]] and `System#currentTimeMillis`. In JDK9+ this will * change and all timestamps taken from this API will have nanosecond * resolution. * @@ -440,7 +440,7 @@ object Time extends TimeLikeOps[Time] { private[util] val localGetTimer: Local[MockTimer] = new Local[MockTimer] /** - * Creates a [[Time]] instance of the given [[Date]]. + * Creates a [[Time]] instance of the given `Date`. */ def apply(date: Date): Time = fromMilliseconds(date.getTime) diff --git a/util-core/src/main/scala/com/twitter/util/Timer.scala b/util-core/src/main/scala/com/twitter/util/Timer.scala index bb4f1328c0..a5890c53ea 100644 --- a/util-core/src/main/scala/com/twitter/util/Timer.scala +++ b/util-core/src/main/scala/com/twitter/util/Timer.scala @@ -188,7 +188,7 @@ class ReferenceCountingTimer(factory: () => Timer) extends ProxyTimer with Refer } /** - * A [[Timer]] that is implemented via a [[java.util.Timer]]. + * A [[Timer]] that is implemented via a `java.util.Timer`. * * This timer has millisecond granularity. * @@ -202,9 +202,9 @@ class ReferenceCountingTimer(factory: () => Timer) extends ProxyTimer with Refer * scheduling tasks that take a long time to execute (e.g. blocking IO) * as this blocks other tasks from running at their scheduled time. * - * @param isDaemon whether or not the associated [[Thread]] should run + * @param isDaemon whether or not the associated `Thread` should run * as a daemon. - * @param name used as the name of the associated [[Thread]] when specified. + * @param name used as the name of the associated `Thread` when specified. */ class JavaTimer(isDaemon: Boolean, name: Option[String]) extends Timer { def this(isDaemon: Boolean) = this(isDaemon, None) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala b/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala index 5ce32a7205..8e189a0ed1 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/ContentionSnapshot.scala @@ -5,13 +5,15 @@ import java.lang.management.{ManagementFactory, ThreadInfo} /** * A thread contention summary providing a brief overview of threads - * that are [[https://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#BLOCKED BLOCKED]], [[https://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#WAITING WAITING]], or [[https://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#TIMED_WAITING TIMED_WAITING]] + * that are [[https://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#BLOCKED BLOCKED]], + * [[https://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#WAITING WAITING]], + * or [[https://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Thread.State.html#TIMED_WAITING TIMED_WAITING]] * * While this could be an object, we use instantiation as a signal of * intent and enable contention monitoring. * - * @note users should ensure that the [[java.lang.management.ManagementPermission("control")]] is - * allowed via the [[java.lang.SecurityManager]]. + * @note users should ensure that the `java.lang.management.ManagementPermission("control")` is + * allowed via the `java.lang.SecurityManager`. */ class ContentionSnapshot { ManagementFactory.getThreadMXBean.setThreadContentionMonitoringEnabled(true) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala index fb1f92cf2b..2b6f45be5a 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala @@ -190,7 +190,7 @@ object Jvm { /** * Return an instance of the [[Jvm]] for this runtime. * - * See [[Jvms.apply()]] for Java compatibility. + * See `Jvms.apply` for Java compatibility. */ def apply(): Jvm = _jvm @@ -218,7 +218,7 @@ object Jvms { Jvm.ProcessId /** - * Java compatibility for [[Jvm.apply()]]. + * Java compatibility for [[Jvm.apply]]. */ def get(): Jvm = Jvm() diff --git a/util-lint/src/main/scala/com/twitter/util/lint/Rule.scala b/util-lint/src/main/scala/com/twitter/util/lint/Rule.scala index 85b059c6a8..006d5601e8 100644 --- a/util-lint/src/main/scala/com/twitter/util/lint/Rule.scala +++ b/util-lint/src/main/scala/com/twitter/util/lint/Rule.scala @@ -3,7 +3,7 @@ package com.twitter.util.lint import java.util.regex.Pattern /** - * A single lint rule, that when [[Rule.apply() run]] evaluates + * A single lint rule, that when [[Rule.apply run]] evaluates * whether or not there are any issues. */ trait Rule { diff --git a/util-logging/src/main/scala/com/twitter/logging/Level.scala b/util-logging/src/main/scala/com/twitter/logging/Level.scala index 8e7881ef9f..56fcfa13e4 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Level.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Level.scala @@ -23,7 +23,7 @@ object Level { Seq(OFF, FATAL, CRITICAL, ERROR, WARNING, INFO, DEBUG, TRACE, ALL) /** - * Associate [[java.util.logging.Level]] and `Level` by their integer + * Associate `java.util.logging.Level` and `Level` by their integer * values. If there is no match, we return `None`. */ def fromJava(level: javalog.Level): Option[Level] = diff --git a/util-logging/src/main/scala/com/twitter/logging/LogRecord.scala b/util-logging/src/main/scala/com/twitter/logging/LogRecord.scala index 0f25e890d4..39a365d53e 100644 --- a/util-logging/src/main/scala/com/twitter/logging/LogRecord.scala +++ b/util-logging/src/main/scala/com/twitter/logging/LogRecord.scala @@ -19,12 +19,12 @@ package com.twitter.logging import java.util.{logging => javalog} /** - * Wrapper around {@link java.util.logging.LogRecord}. Should only be accessed from a single thread. + * Wrapper around `java.util.logging.LogRecord`. Should only be accessed from a single thread. * - * Messages are formatted by Java's LogRecord using {@link java.text.MessageFormat} whereas - * this class uses a regular {@link java.text.StringFormat} + * Messages are formatted by Java's LogRecord using `java.text.MessageFormat` whereas + * this class uses a regular `java.text.StringFormat`. * - * This class takes {@link com.twitter.logging.Logger} into account when inferring the + * This class takes [[com.twitter.logging.Logger]] into account when inferring the * `sourceMethod` and `sourceClass` names. */ class LogRecord(level: javalog.Level, msg: String) extends javalog.LogRecord(level, msg) { diff --git a/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala b/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala index 9dad3f8b29..2cc4e8f258 100644 --- a/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/QueueingHandler.scala @@ -1,19 +1,3 @@ -/* - * Copyright 2011 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package com.twitter.logging import com.twitter.concurrent.{NamedPoolThreadFactory, AsyncQueue} @@ -69,7 +53,7 @@ object QueueingHandler { * [[onOverflow]] when it is at capacity. * * @param inferClassNames [[com.twitter.logging.LogRecord]] and - * [[java.util.logging.LogRecord]] both attempt to infer the class and + * `java.util.logging.LogRecord` both attempt to infer the class and * method name of the caller, but the inference needs the stack trace at * the time that the record is logged. QueueingHandler breaks the * inference because the log record is rendered out of band, so the stack diff --git a/util-mock/src/main/scala/com/twitter/util/mock/Mockito.scala b/util-mock/src/main/scala/com/twitter/util/mock/Mockito.scala index 8f6f3da407..9e87fe1dfa 100644 --- a/util-mock/src/main/scala/com/twitter/util/mock/Mockito.scala +++ b/util-mock/src/main/scala/com/twitter/util/mock/Mockito.scala @@ -4,14 +4,14 @@ import org.mockito.{ArgumentMatchersSugar, IdiomaticMockito} /** * Helper for Mockito Scala sugar with [[https://github.com/mockito/mockito-scala#idiomatic-mockito idiomatic stubbing]]. - * Java users are encouraged to use [[org.mockito.Mockito]] directly. + * Java users are encouraged to use `org.mockito.Mockito` directly. * * Note that the Specs2 `smartMock[]` or `mock[].smart` is the default behavior * for [[https://github.com/mockito/mockito-scala Mockito Scala]]. * * =Usage= * - * This trait uses [[org.mockito.IdiomaticMockito]] which is heavily influenced by ScalaTest Matchers. + * This trait uses `org.mockito.IdiomaticMockito` which is heavily influenced by ScalaTest Matchers. * * To use, mix in the [[com.twitter.util.mock.Mockito]] trait where desired. * @@ -156,8 +156,8 @@ import org.mockito.{ArgumentMatchersSugar, IdiomaticMockito} * [[https://github.com/mockito/mockito-scala#getting-started Getting Started]] documentation for general * information. * - * @see [[org.mockito.IdiomaticMockito]] - * @see [[org.mockito.ArgumentMatchersSugar]] + * see `org.mockito.IdiomaticMockito` + * see `org.mockito.ArgumentMatchersSugar` */ trait Mockito extends IdiomaticMockito with ArgumentMatchersSugar diff --git a/util-registry/src/main/scala/com/twitter/util/registry/Library.scala b/util-registry/src/main/scala/com/twitter/util/registry/Library.scala index 02f32299d3..d2225d48df 100644 --- a/util-registry/src/main/scala/com/twitter/util/registry/Library.scala +++ b/util-registry/src/main/scala/com/twitter/util/registry/Library.scala @@ -16,7 +16,7 @@ object Library { * May only be called once with a given `name`. `params` and `name` must abide * by the guidelines for keys and values set in [[Registry]]. * - * @returns None if a library has already been registered with the given `name`, + * @return None if a library has already been registered with the given `name`, * or a [[Roster]] for resetting existing fields in the map otherwise. */ def register(name: String, params: Map[String, String]): Option[Roster] = { diff --git a/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala b/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala index 7e0409793b..49c64282df 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala @@ -10,8 +10,8 @@ import scala.jdk.CollectionConverters._ * * @param label A label used for identifying this Router (i.e. for distinguishing between [[Router]] * instances in error messages or for StatsReceiver scope). - * @param jRoutes All of the [[Route]] routes contained within this [[Router]], represented as - * a Java [[JIterable]]. + * @param jRoutes All of the `Route` routes contained within this [[Router]], represented as + * a Java `Iterable`. * @tparam Input The Input type used to determine a route destination * @tparam Route The output/resulting route type */ diff --git a/util-routing/src/main/scala/com/twitter/util/routing/ClosedRouterException.scala b/util-routing/src/main/scala/com/twitter/util/routing/ClosedRouterException.scala index d6ae1f5a53..abd38405f3 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/ClosedRouterException.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/ClosedRouterException.scala @@ -3,10 +3,10 @@ package com.twitter.util.routing import scala.util.control.NoStackTrace /** - * Exception thrown when an attempt to route an `Input` to a [[Router]] where [[close()]] has been + * Exception thrown when an attempt to route an `Input` to a [[Router]] where `close` has been * initiated on the [[Router]]. * - * @param router The [[Router]] where [[close()]] has already been initiated. + * @param router The [[Router]] where `close` has already been initiated. */ case class ClosedRouterException(router: Router[_, _]) extends RuntimeException(s"Attempting to route on a closed router with label '${router.label}'") diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala b/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala index ed0a598e56..774bc60c70 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala @@ -17,8 +17,8 @@ object Generator { } /** - * Functional alias for creating a [[RouterType router]] given a [[String label]] and - * all of the defined [[Route routes]] for the router to be generated. + * Functional alias for creating a [[Router router]] given a label and + * all of the defined routes for the router to be generated. */ abstract class Generator[Input, Route, +RouterType <: Router[Input, Route]] extends ((String, Iterable[Route]) => RouterType) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Router.scala b/util-routing/src/main/scala/com/twitter/util/routing/Router.scala index 35818c1a39..24346dd272 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/Router.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/Router.scala @@ -8,23 +8,23 @@ import scala.util.control.NonFatal * A generic interface for routing an input to an optional matching route. * * @tparam Input The Input type used when determining a route destination - * @tparam Route The resulting route type. A [[Route]] may have dynamic - * properties and may be found to satisfy multiple [[Input]] + * @tparam Route The resulting route type. A `Route` may have dynamic + * properties and may be found to satisfy multiple `Input` * instances. As there may not be a 1-to-1 mapping of - * [[Input]] to [[Route]], it is encouraged that a [[Route]] - * encapsulates its relationship to an [[Input]]. + * `Input` to `Route`, it is encouraged that a `Route` + * encapsulates its relationship to an `Input`. * * An example to consider would be an HTTP [[Router]]. * An HTTP request contains a Method (i.e. GET, POST) and * a URI. A [[Router]] for a REST API may match "GET /users/{id}" * for multiple HTTP requests (i.e. "GET /users/123" AND "GET /users/456" would map - * to the same destination [[Route]]). As a result, the [[Route]] in this example + * to the same destination `Route`). As a result, the `Route` in this example * should be aware of the method, path, and and the destination/logic responsible - * for handling an [[Input]] that matches. + * for handling an `Input` that matches. * - * Another property to consider for a [[Route]] is "uniqueness". It may or may not + * Another property to consider for a `Route` is "uniqueness". It may or may not * be desirable for a [[Router]] to contain multiple routes that correspond to - * the same [[Input]]. + * the same `Input`. * * @note A [[Router]] should be considered immutable unless explicitly noted. */ @@ -37,9 +37,9 @@ trait Router[Input, +Route] extends (Input => Option[Route]) with ClosableOnce w def label: String /** - * All of the [[Route]] routes contained within this [[Router]] + * All of the `Route` routes contained within this [[Router]] * - * @note This property is used to linearly determine the routes when [[closed()]] is called. + * @note This property is used to linearly determine the routes when `close()` is called. * It may also be used as a way to determine what routes are defined within a [[Router]] * (i.e. for generating meaningful error messages). This property does not imply * any relationship with [[apply]] or [[find]] or that a [[Router]]'s runtime behavior @@ -48,38 +48,36 @@ trait Router[Input, +Route] extends (Input => Option[Route]) with ClosableOnce w def routes: Iterable[Route] /** - * Attempt to route an [[Input]] to a [[Route]] route known by this Router. + * Attempt to route an `Input` to a `Route` route known by this Router. * - * @param input The [[Input]] to use for determining an available route + * @param input The `Input` to use for determining an available route * * @return `Some(Route)` if a route is found to match, `None` otherwise * - * @note A [[ClosedRouterException]] will be thrown if [[close]] has been initiated on this - * [[router]] and subsequent routing attempts are received. + * @note A [[ClosedRouterException]] will be thrown if `close()` + * has been initiated on this [[Router]] and subsequent routing attempts are received. */ final def apply(input: Input): Option[Route] = if (isClosed) throw ClosedRouterException(this) else find(input) /** - * Logic used to determine if this [[Router]] contains a [[Route]] - * for an [[Input]]. + * Logic used to determine if this [[Router]] contains a `Route` + * for an `Input`. * - * @param input The [[Input]] to determine a route for. + * @param input The `Input` to determine a route for. * * @return `Some(Route)` if a matching route is defined, `None` if a route destination is - * not defined for the [[Input]]. + * not defined for the `Input`. * - * @note This method is only meant to be called within the [[Router]]'s [[apply]], which + * @note This method is only meant to be called within the [[Router]]'s `apply`, which * handles lifecycle concerns. It should not be accessed directly. */ protected def find(input: Input): Option[Route] /** - * @inheritdoc - * @note NonFatal exceptions encountered when calling [[close()]] on a [[Route]] will be - * suppressed and Fatal exceptions will take on the same exception behavior of a - * [[Future.join]]. + * @note NonFatal exceptions encountered when calling `close()` on a `Route` will be + * suppressed and Fatal exceptions will take on the same exception behavior of a `Future.join`. */ override protected def closeOnce(deadline: Time): Future[Unit] = Future.join(routes.map(closeIfClosable(_, deadline)).toSeq) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala b/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala index 6889c61ca9..7249657ba2 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala @@ -12,14 +12,14 @@ object RouterBuilder { } /** - * Utility for building and creating [[RouterType routers]]. The resulting [[RouterType router]] - * should be considered immutable, unless the [[RouterType router's]] implementation + * Utility for building and creating [[Router routers]]. The resulting [[Router router]] + * should be considered immutable, unless the [[Router router's]] implementation * explicitly states otherwise. * * @tparam Input The [[Router router's]] `Input` type. * @tparam Route The [[Router router's]] destination `Route` type. It is recommended that the `Route` * is a self-contained/self-describing type for the purpose of validation via - * the [[validator]]. Put differently, the `Route` should know of the + * the [[Validator]]. Put differently, the `Route` should know of the * `Input` that maps to itself. * @tparam RouterType The type of [[Router]] to build. */ @@ -29,13 +29,13 @@ case class RouterBuilder[Input, Route, +RouterType <: Router[Input, Route]] priv private val routes: Queue[Route] = Queue.empty, private val validator: Validator[Route] = Validator.None) { - /** Set the [[Router.label label]] for the resulting [[RouterType router]] */ + /** Set the [[Router.label label]] for the resulting [[Router router]] */ def withLabel(label: String): RouterBuilder[Input, Route, RouterType] = copy(label = label) /** - * Add the [[Route route]] to the routes that will be present in - * the [[RouterType router]] when [[newRouter()]] is called. + * Add the route to the routes that will be present in + * the [[Router router]] when [[newRouter]] is called. */ def withRoute(route: Route): RouterBuilder[Input, Route, RouterType] = copy(routes = routes :+ route) @@ -48,7 +48,7 @@ case class RouterBuilder[Input, Route, +RouterType <: Router[Input, Route]] priv ): RouterBuilder[Input, Route, RouterType] = copy(validator = validator) - /** Generate a new [[RouterType router]] from the defined [[routes]] */ + /** Generate a new [[Router router]] from the defined routes */ def newRouter(): RouterType = { val failures = validator(routes) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/ValidationError.scala b/util-routing/src/main/scala/com/twitter/util/routing/ValidationError.scala index 27c72e30fa..e19e1f5a85 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/ValidationError.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/ValidationError.scala @@ -2,7 +2,7 @@ package com.twitter.util.routing /** * Container class for an error that is encountered as part of validating routes via - * a [[RouteBuilder]]. + * a [[RouterBuilder]]. * * @param msg The message that explains the error. */ diff --git a/util-routing/src/main/scala/com/twitter/util/routing/ValidationException.scala b/util-routing/src/main/scala/com/twitter/util/routing/ValidationException.scala index dd436156a5..38bebd3d23 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/ValidationException.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/ValidationException.scala @@ -9,7 +9,7 @@ private object ValidationException { } /** - * Exception thrown when a [[RouterBuilder]] observes any [[Route routes]] that are not valid for + * Exception thrown when a [[RouterBuilder]] observes any routes that are not valid for * the [[Router]] type it is building. */ case class ValidationException private[routing] (failures: Iterable[ValidationError]) diff --git a/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala b/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala index 40bdcd3a46..f7c2741acc 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala @@ -11,7 +11,7 @@ import java.util.UUID import javax.net.ssl.{KeyManager, KeyManagerFactory} /** - * A factory which can create a [[javax.net.ssl.KeyManager KeyManager]] which contains + * A factory which can create a `javax.net.ssl.KeyManager` which contains * an X.509 Certificate (or Certificate chain) and a PKCS#8 private key. */ class Pkcs8KeyManagerFactory(certsFile: File, keyFile: File) { @@ -46,9 +46,9 @@ class Pkcs8KeyManagerFactory(certsFile: File, keyFile: File) { /** * Attempts to read the contents of both the X.509 Certificates file and the PKCS#8 - * Private Key file and combine the contents into a [[javax.net.ssl.KeyManager KeyManager]]. + * Private Key file and combine the contents into a `javax.net.ssl.KeyManager` * The singular value is returned in an Array for ease of use with - * [[javax.net.ssl.SSLContext SSLContext's]] init method. + * `javax.net.ssl.SSLContext` init method. */ def getKeyManagers(): Try[Array[KeyManager]] = { val tryCerts: Try[Seq[X509Certificate]] = diff --git a/util-security/src/main/scala/com/twitter/util/security/PrivateKeyFile.scala b/util-security/src/main/scala/com/twitter/util/security/PrivateKeyFile.scala index 744b550cfe..5fb18c86c2 100644 --- a/util-security/src/main/scala/com/twitter/util/security/PrivateKeyFile.scala +++ b/util-security/src/main/scala/com/twitter/util/security/PrivateKeyFile.scala @@ -47,10 +47,9 @@ class PrivateKeyFile(file: File) { /** * Attemps to read the contents of an unencrypted PrivateKey file. * - * @throws InvalidPemFormatException if the file is not a PEM format - * file. + * @note Throws `InvalidPemFormatException` if the file is not a PEM format file. * - * @throws InvalidKeySpecException if the file cannot be loaded as an + * @note Throws `InvalidKeySpecException` if the file cannot be loaded as an * RSA, DSA, or EC PrivateKey. * * @note This method will attempt to load the key first as an RSA key, then diff --git a/util-security/src/main/scala/com/twitter/util/security/X500PrincipalInfo.scala b/util-security/src/main/scala/com/twitter/util/security/X500PrincipalInfo.scala index 29859fcbea..4835e5f632 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X500PrincipalInfo.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X500PrincipalInfo.scala @@ -6,7 +6,7 @@ import com.twitter.util.security.X500PrincipalInfo._ import scala.jdk.CollectionConverters._ /** - * Parses an [[javax.security.auth.x500.X500Principal X500Principal]] into + * Parses an `javax.security.auth.x500.X500Principal` into * its individual pieces for easily extracting widely used items like an X.509 * certificate's Common Name. */ diff --git a/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala b/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala index 31ae19a267..4497fb54be 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala @@ -10,7 +10,7 @@ import java.util.UUID import javax.net.ssl.{TrustManager, TrustManagerFactory} /** - * A factory which can create a [[javax.net.ssl.TrustManager TrustManager]] which contains + * A factory which can create a `javax.net.ssl.TrustManager` which contains * a collection of X.509 certificates. */ class X509TrustManagerFactory(certsFile: File) { @@ -47,9 +47,9 @@ class X509TrustManagerFactory(certsFile: File) { /** * Attempts to read the contents of a file containing a collection of X.509 certificates, - * and combines them into a [[javax.net.ssl.TrustManager TrustManager]]. + * and combines them into a `javax.net.ssl.TrustManager`. * The singular value is returned in an Array for ease of use with - * [[javax.net.ssl.SSLContext SSLContext's]] init method. + * `javax.net.ssl.SSLContext`'s init method. */ def getTrustManagers(): Try[Array[TrustManager]] = { val tryCerts: Try[Seq[X509Certificate]] = diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala index ad00ab5cba..295f2e8ca5 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala @@ -7,13 +7,31 @@ import scala.reflect.{ClassTag, classTag} /** * Companion object for [[com.twitter.util.logging.Logger]] which provides * factory methods for creation. + * + * @define ApplyScaladocLink + * [[com.twitter.util.logging.Logger.apply(name:String):com\.twitter\.util\.logging\.Logger* apply(String)]] + * + * @define ApplyLoggerScaladocLink + * [[com.twitter.util.logging.Logger.apply(underlying:org\.slf4j\.Logger):com\.twitter\.util\.logging\.Logger* apply(Logger)]] + * + * @define ApplyClassScaladocLink + * [[com.twitter.util.logging.Logger.apply(clazz:Class[_]):com\.twitter\.util\.logging\.Logger* apply(Class)]] + * + * @define GetLoggerScaladocLink + * [[com.twitter.util.logging.Logger.getLogger(name:String):com\.twitter\.util\.logging\.Logger* getLogger(String)]] + * + * @define GetLoggerClassScaldocLink + * [[com.twitter.util.logging.Logger.getLogger(clazz:Class[_]):com\.twitter\.util\.logging\.Logger* getLogger(Class)]] + * + * @define GetLoggerLoggerScaladocLink + * [[com.twitter.util.logging.Logger.getLogger(underlying:org\.slf4j\.Logger):com\.twitter\.util\.logging\.Logger* getLogger(Logger)]] */ object Logger { /** * Create a [[com.twitter.util.logging.Logger]] for the given name. * @param name name of the underlying `org.slf4j.Logger`. - * @see Java users see [[getLogger(name:String)]] + * @see Java users see $GetLoggerScaladocLink * * {{{ * val logger = Logger("name") @@ -26,7 +44,7 @@ object Logger { /** * Create a [[Logger]] for the given name. * @param name name of the underlying `org.slf4j.Logger`. - * @see Scala users see [[apply(name:String)]] + * @see Scala users see $ApplyScaladocLink * * {{{ * Logger logger = Logger.getLogger("name"); @@ -38,7 +56,7 @@ object Logger { /** * Create a [[com.twitter.util.logging.Logger]] named for the given class. * @param clazz class to use for naming the underlying Logger. - * @see Java users see [[getLogger(clazz:Class[_])]] + * @see Java users see $GetLoggerClassScaldocLink * * {{{ * val logger = Logger(classOf[MyClass]) @@ -51,7 +69,7 @@ object Logger { /** * Create a [[Logger]] named for the given class. * @param clazz class to use for naming the underlying `org.slf4j.Logger`. - * @see Scala users see [[apply(clazz:Class[_])]] + * @see Scala users see $ApplyClassScaladocLink * * {{{ * Logger logger = Logger.getLogger(MyClass.class); @@ -62,7 +80,7 @@ object Logger { /** * Create a [[com.twitter.util.logging.Logger]] for the runtime class wrapped - * by the implicit [[scala.reflect.ClassTag]] denoting the runtime class `T`. + * by the implicit `scala.reflect.ClassTag` denoting the runtime class `T`. * @tparam T the runtime class type * * {{{ @@ -77,7 +95,7 @@ object Logger { * Create a [[com.twitter.util.logging.Logger]] wrapping the given underlying * `org.slf4j.Logger`. * @param underlying an `org.slf4j.Logger` - * @see Java users see [[getLogger(underlying:Logger)]] + * @see Java users see $GetLoggerLoggerScaladocLink * * {{{ * val logger = Logger(LoggerFactory.getLogger("name")) @@ -91,7 +109,7 @@ object Logger { * Create a [[Logger]] wrapping the given underlying * `org.slf4j.Logger`. * @param underlying an `org.slf4j.Logger` - * @see Scala users see [[apply(underlying:Logger)]] + * @see Scala users see $ApplyLoggerScaladocLink * * {{{ * Logger logger = Logger.getLogger(LoggerFactory.getLogger("name")); diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index f086918004..71305afc8e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -157,16 +157,16 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { override def toString: String = "InMemoryStatsReceiver" /** - * Dumps this in-memory stats receiver to the given [[PrintStream]]. - * @param p the [[PrintStream]] to which to write in-memory values. + * Dumps this in-memory stats receiver to the given `PrintStream`. + * @param p the `PrintStream` to which to write in-memory values. */ def print(p: PrintStream): Unit = { print(p, includeHeaders = false) } /** - * Dumps this in-memory stats receiver to the given [[PrintStream]]. - * @param p the [[PrintStream]] to which to write in-memory values. + * Dumps this in-memory stats receiver to the given `PrintStream`. + * @param p the `PrintStream` to which to write in-memory values. * @param includeHeaders optionally include printing underlines headers for the different types * of stats printed, e.g., "Counters:", "Gauges:", "Stats;" */ @@ -197,8 +197,8 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } /** - * Dumps this in-memory stats receiver's MetricMetadata to the given [[PrintStream]]. - * @param p the [[PrintStream]] to which to write in-memory metadata. + * Dumps this in-memory stats receiver's MetricMetadata to the given `PrintStream`. + * @param p the `PrintStream` to which to write in-memory metadata. */ def printSchemas(p: PrintStream): Unit = { val sortedSchemas = schemas.mapKeys(_.mkString("/")).toSortedMap diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala index 1aa75644c7..a8e3ef97c8 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala @@ -8,9 +8,14 @@ import scala.util.control.NonFatal * An append-only collection of time-series data. Example Stats are * "queue depth" or "query width in a stream of requests". * + * @define StatTimeScaladocLink + * [[com.twitter.finagle.stats.Stat.time[A](stat:com\.twitter\.finagle\.stats\.Stat,unit:java\.util\.concurrent\.TimeUnit)(f:=>A):A* time(Stat,TimeUnit)]] + * + * @define StatTimeFutureScaladocLink + * [[com.twitter.finagle.stats.Stat.timeFuture[A](stat:com\.twitter\.finagle\.stats\.Stat,unit:java\.util\.concurrent\.TimeUnit)(f:=>com\.twitter\.util\.Future[A]):com\.twitter\.util\.Future[A]* timeFuture(Stat,TimeUnit)]] + * * Utilities for timing synchronous execution and asynchronous - * execution are on the companion object ([[Stat.time(Stat)]] and - * [[Stat.timeFuture(Stat)]]. + * execution are on the companion object ($StatTimeScaladocLink and $StatTimeFutureScaladocLink). */ trait Stat { def add(value: Float): Unit diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 071940439e..b7687f0fbc 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -29,12 +29,15 @@ object StatsReceiver { /** * [[StatsReceiver]] utility methods for ease of use from java. + * + * @define ProvideGaugeScaladocLink + * [[com.twitter.finagle.stats.StatsReceiver.provideGauge(name:String*)(f:=>Float):Unit* provideGauge(String*)(=>Float)]] */ @deprecated("Use StatsReceiver addGauge and provideGauge methods directly", "2020-06-10") object StatsReceivers { /** - * Java compatible version of [[StatsReceiver.addGauge]]. + * Java compatible version of [[addGauge]]. */ @varargs def addGauge(statsReceiver: StatsReceiver, callable: Callable[JFloat], name: String*): Gauge = { @@ -44,7 +47,7 @@ object StatsReceivers { } /** - * Java compatible version of [[StatsReceiver.provideGauge]]. + * Java compatible version of $ProvideGaugeScaladocLink. */ @varargs def provideGauge(statsReceiver: StatsReceiver, callable: Callable[JFloat], name: String*): Unit = @@ -66,6 +69,24 @@ object StatsReceivers { * * Metrics created w/o an explicitly specified [[Verbosity]] level, will use [[Verbosity.Default]]. * Use [[VerbosityAdjustingStatsReceiver]] to adjust this behaviour. + * + * @define ProvideGaugeScaladocLink + * [[com.twitter.finagle.stats.StatsReceiver.provideGauge(name:String*)(f:=>Float):Unit* provideGauge(String*)(=>Float)]] + * + * @define AddGaugeScaladocLink + * [[com.twitter.finagle.stats.StatsReceiver.addGauge(name:String*)(f:=>Float):com\.twitter\.finagle\.stats\.Gauge* addGauge(String*)(=>Float)]] + * + * @define JavaProvideGaugeScaladocLink + * [[com.twitter.finagle.stats.StatsReceiver.provideGauge(f:java\.util\.function\.Supplier[Float],name:String*):Unit* provideGauge(Supplier[Float],String*)]] + * + * @define JavaAddGaugeScaladocLink + * [[com.twitter.finagle.stats.StatsReceiver.addGauge(f:java\.util\.function\.Supplier[Float],name:String*):com\.twitter\.finagle\.stats\.Gauge* addGauge(Supplier[Float],String*)]] + * + * @define AddVerboseGaugeScaladocLink + * [[com.twitter.finagle.stats.StatsReceiver.addGauge(verbosity:com\.twitter\.finagle\.stats\.Verbosity,name:String*)(f:=>Float):com\.twitter\.finagle\.stats\.Gauge* addGauge(Verbosity,String*)(=>Float)]] + * + * @define JavaVerboseAddGaugeScaladocLink + * [[com.twitter.finagle.stats.StatsReceiver.addGauge(f:java\.util\.function\.Supplier[Float],verbosity:com\.twitter\.finagle\.stats\.Verbosity,name:String*):com\.twitter\.finagle\.stats\.Gauge* addGauge(Supplier[Float],Verbosity,String*)]] */ trait StatsReceiver { @@ -132,11 +153,10 @@ trait StatsReceiver { * * Measurements under the same name are added together. * - * @see [[StatsReceiver.addGauge]] if you can properly control the lifecycle + * @see $AddGaugeScaladocLink if you can properly control the lifecycle * of the returned [[Gauge gauge]]. * - * @see [[StatsReceiver.provideGauge(java.util.function.Supplier, String*)]] for a Java-friendly - * version. + * @see $JavaProvideGaugeScaladocLink for a Java-friendly version. */ def provideGauge(name: String*)(f: => Float): Unit = { val gauge = addGauge(name: _*)(f) @@ -146,7 +166,7 @@ trait StatsReceiver { } /** - * Just like [[provideGauge()]] but optimized for better Java experience. + * Just like $ProvideGaugeScaladocLink but optimized for better Java experience. */ @varargs def provideGauge(f: Supplier[Float], name: String*): Unit = @@ -164,11 +184,10 @@ trait StatsReceiver { * * Measurements under the same name are added together. * - * @see [[StatsReceiver.provideGauge]] when there is not a good location + * @see $ProvideGaugeScaladocLink when there is not a good location * to store the returned [[Gauge gauge]] that can give the desired lifecycle. * - * @see [[StatsReceiver.addGauge(java.util.function.Supplier,String*)]] for a Java-friendly - * version. + * @see $JavaAddGaugeScaladocLink for a Java-friendly version. * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ @@ -186,11 +205,10 @@ trait StatsReceiver { * * Measurements under the same name are added together. * - * @see [[StatsReceiver.provideGauge]] when there is not a good location + * @see $ProvideGaugeScaladocLink when there is not a good location * to store the returned [[Gauge gauge]] that can give the desired lifecycle. * - * @see [[StatsReceiver.addGauge(java.util.function.Supplier,Verbosity,String*)]] for a - * Java-friendly version. + * @see $JavaVerboseAddGaugeScaladocLink for a Java-friendly version. * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ @@ -198,13 +216,13 @@ trait StatsReceiver { addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name)))(f) /** - * Just like [[addGauge(String*,=>Float)]] but optimized for better Java experience. + * Just like $AddGaugeScaladocLink but optimized for better Java experience. */ @varargs def addGauge(f: Supplier[Float], name: String*): Gauge = addGauge(f, Verbosity.Default, name: _*) /** - * Just like [[addGauge(Verbosity,String*,=>Float)]] but optimized for better Java experience. + * Just like $AddVerboseGaugeScaladocLink but optimized for better Java experience. */ @varargs def addGauge(f: Supplier[Float], verbosity: Verbosity, name: String*): Gauge = @@ -222,8 +240,8 @@ trait StatsReceiver { * * Measurements under the same name are added together. * - * @see [[StatsReceiver.provideGauge]] when there is not a good location - * to store the returned [[Gauge gauge]] that can give the desired lifecycle. + * @see $ProvideGaugeScaladocLink when there is not a + * good location to store the returned [[Gauge gauge]] that can give the desired lifecycle. * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ def addGauge(schema: GaugeSchema)(f: => Float): Gauge diff --git a/util-thrift/src/main/scala/com/twitter/util/ThriftSerializer.scala b/util-thrift/src/main/scala/com/twitter/util/ThriftSerializer.scala index 2395ca0952..c327dc93d5 100644 --- a/util-thrift/src/main/scala/com/twitter/util/ThriftSerializer.scala +++ b/util-thrift/src/main/scala/com/twitter/util/ThriftSerializer.scala @@ -33,7 +33,7 @@ trait ThriftSerializer extends StringEncoder { } /** - * A thread-safe [[ThriftSerializer]] that uses [[TSimpleJSONProtocol]]. + * A thread-safe [[ThriftSerializer]] that uses `TSimpleJSONProtocol`. */ class JsonThriftSerializer extends ThriftSerializer { override def protocolFactory: TProtocolFactory = new TSimpleJSONProtocol.Factory @@ -46,7 +46,7 @@ class JsonThriftSerializer extends ThriftSerializer { } /** - * A thread-safe [[ThriftSerializer]] that uses [[TBinaryProtocol]]. + * A thread-safe [[ThriftSerializer]] that uses `TBinaryProtocol`. * * @note an implementation using `com.twitter.finagle.thrift.Protocols.binaryFactory` * instead of this is recommended. @@ -56,7 +56,7 @@ class BinaryThriftSerializer extends ThriftSerializer with Base64StringEncoder { } /** - * A thread-safe [[ThriftSerializer]] that uses [[TCompactProtocol]]. + * A thread-safe [[ThriftSerializer]] that uses `TCompactProtocol`. */ class CompactThriftSerializer extends ThriftSerializer with Base64StringEncoder { override def protocolFactory: TProtocolFactory = new TCompactProtocol.Factory diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala index 6dfc4007f2..8b6fe1a003 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala @@ -22,12 +22,12 @@ object JsonTunableMapper { /** * The Deserializers that [[JsonTunableMapper]] uses by default, in addition to Scala data type - * deserializers afforded by [[com.fasterxml.jackson.module.scala.DefaultScalaModule]]. + * deserializers afforded by `com.fasterxml.jackson.module.scala.DefaultScalaModule`. * * These deserializers are: * - * - [[com.twitter.util.tunable.json.DurationFromString]] - * - [[com.twitter.util.tunable.json.StorageUnitFromString]] + * - `com.twitter.util.tunable.json.DurationFromString` + * - `com.twitter.util.tunable.json.StorageUnitFromString` */ val DefaultDeserializers: Seq[JsonDeserializer[_]] = Seq(DurationFromString, StorageUnitFromString) @@ -49,10 +49,10 @@ object JsonTunableMapper { * in priority order. Where environementOpt and instanceOpt are available (env, instance), * paths are ordered: * - * i. $root/$env/instance-$id.json - * i. $root/$env/instances.json - * i. $root/instance-$id.json - * i. $root/instances.json + * i. \$root/\$env/instance-\$id.json + * i. \$root/\$env/instances.json + * i. \$root/instance-\$id.json + * i. \$root/instances.json */ def pathsByPriority( root: String, @@ -86,24 +86,24 @@ object JsonTunableMapper { * "tunables": * [ * { - * "id" : "$id1", - * "value" : $value, - * "type" : "$class" + * "id" : "\$id1", + * "value" : \$value, + * "type" : "\$class" * }, * { - * "id" : "$id2", - * "value" : $value, - * "type" : "$class", + * "id" : "\$id2", + * "value" : \$value, + * "type" : "\$class", * "comment": "optional comment" * } * ] * } * - * Where $id1 and $id2 are unique identifiers used to access the [[Tunable]], $value is the value, - * and $class is the fully-qualified class name (e.g. com.twitter.util.Duration) + * Where \$id1 and \$id2 are unique identifiers used to access the [[Tunable]], \$value is the value, + * and \$class is the fully-qualified class name (e.g. com.twitter.util.Duration) * * If the JSON is invalid, or contains duplicate ids for [[Tunable]]s, `parse` will - * return a [[Throw]]. Otherwise, `parse` returns [[Return[TunableMap]] + * return a [[Throw]]. Otherwise, `parse` returns [[Return Return[TunableMap]] */ final class JsonTunableMapper(deserializers: Seq[JsonDeserializer[_ <: Any]]) { @@ -171,8 +171,8 @@ final class JsonTunableMapper(deserializers: Seq[JsonDeserializer[_ <: Any]]) { * Load and parse the JSON file located at `path` in the application's resources. * * If no configuration files exist, return [[NullTunableMap]]. - * If multiple configuration files exists, return [[IllegalArgumentException]] - * If the configuration file cannot be parsed, return [[IllegalArgumentException]] + * If multiple configuration files exists, return `IllegalArgumentException` + * If the configuration file cannot be parsed, return `IllegalArgumentException` */ def loadJsonTunables(id: String, path: String): TunableMap = { val classLoader = getClass.getClassLoader diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala index f5d72a00eb..8ac4b9aaaa 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/TunableMap.scala @@ -13,7 +13,7 @@ abstract class TunableMap { self => /** * Returns a [[Tunable]] for the given `key.id` in the [[TunableMap]]. If the [[Tunable]] is not - * of type `T` or a subclass of type `T`, throws a [[ClassCastException]] + * of type `T` or a subclass of type `T`, throws a `ClassCastException` */ def apply[T](key: TunableMap.Key[T]): Tunable[T] @@ -40,8 +40,8 @@ abstract class TunableMap { self => def entries: Iterator[TunableMap.Entry[_]] /** - * Compose this [[TunableMap]] with another [[TunableMap]]. [[Tunables]] retrieved from - * the composed map prioritize the values of [[Tunables]] in the this map over the other + * Compose this [[TunableMap]] with another [[TunableMap]]. [[Tunable]]s retrieved from + * the composed map prioritize the values of [[Tunable]]s in the this map over the other * [[TunableMap]]. */ def orElse(that: TunableMap): TunableMap = (self, that) match { @@ -141,7 +141,7 @@ object TunableMap { * A [[TunableMap]] that can be updated via `put` and `clear` operations. Putting * a value for a given `id` will update the current value for the `id`, or create * a new [[Tunable]] if it does not exist. The type of the new value must match that of the - * existing value, or a [[ClassCastException]] will be thrown. + * existing value, or a `ClassCastException` will be thrown. * * `apply` returns a [[Tunable]] for a given [[TunableMap.Key]] and creates it if does not already * exist. Updates to the [[TunableMap]] update the underlying [[Tunable]]s; for example, a diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala b/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala index 4b36b732ae..7d02868971 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/ShardCoordinator.scala @@ -48,8 +48,8 @@ class ShardCoordinator(zk: ZkClient, path: String, numShards: Int) { * client is finished performing work for the shard. * * @return A Future of ShardPermit that is satisfied when a shard slot becomes available. - * @throws SemaphoreError when the underlying semaphore throws an exception. - * @throws RejectedExecutionException when a shard cannot be acquired due to an unexpected + * @note Throws `SemaphoreError` when the underlying semaphore throws an exception. + * @note Throws `RejectedExecutionException` when a shard cannot be acquired due to an unexpected * state in the zookeeper tree. Assumed to only happen if some * zookeeper client clobbers the tree location for this * ShardCoordinator. From b368ca379958706c537d3e8862e2e3c573b1c181 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Wed, 21 Oct 2020 16:51:04 +0000 Subject: [PATCH 394/796] Don't update the key everytime for Local.let Problem: Local.let always updates the key, even if it hasn't actually changed. The current calls to `this.apply()`, `this.update()`, and `this.set()` rely on ThreadLocal's set and get methods, which look into map. Solution: Remove calls to `this.apply()`, `this.update()`, and `this.set()`, and change key/values using Context methods instead of ThreadLocal methods JIRA Issues: CSL-6573 Differential Revision: https://phabricator.twitter.biz/D565448 --- .../main/scala/com/twitter/util/Local.scala | 40 ++++++++++++++++--- .../scala/com/twitter/util/LocalTest.scala | 33 +++++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Local.scala b/util-core/src/main/scala/com/twitter/util/Local.scala index 2f0b3d56ec..d97b078c54 100644 --- a/util-core/src/main/scala/com/twitter/util/Local.scala +++ b/util-core/src/main/scala/com/twitter/util/Local.scala @@ -4973,10 +4973,24 @@ final class Local[T] { * upon completion. */ def let[U](value: T)(f: => U): U = { - val saved = apply() - update(value) + val oldCtx = Local.save() + val newCtx = oldCtx.set(key, Some(value)) + Local.restore(newCtx) try f - finally set(saved) + finally { + val now = Local.save() + if (newCtx eq now) { + // Fast path: no other ctx modifications to worry about + Local.restore(oldCtx) + } else { + // Another element was updated in the meantime. We just have to set the old value. + val next = oldCtx.get(key) match { + case s @ Some(_) => now.set(key, s) + case None => now.remove(key) + } + Local.restore(next) + } + } } /** @@ -4984,10 +4998,24 @@ final class Local[T] { * completion. */ def letClear[U](f: => U): U = { - val saved = apply() - clear() + val oldCtx = Local.save() + val newCtx = oldCtx.remove(key) + Local.restore(newCtx) try f - finally set(saved) + finally { + val now = Local.save() + if (newCtx eq now) { + // Fast path: no other ctx modifications to worry about + Local.restore(oldCtx) + } else { + // Another element was updated in the meantime. We just have to set the old value. + val next = oldCtx.get(key) match { + case s @ Some(_) => now.set(key, s) + case None => now.remove(key) + } + Local.restore(next) + } + } } /** diff --git a/util-core/src/test/scala/com/twitter/util/LocalTest.scala b/util-core/src/test/scala/com/twitter/util/LocalTest.scala index 91cd396a7d..1edf886072 100644 --- a/util-core/src/test/scala/com/twitter/util/LocalTest.scala +++ b/util-core/src/test/scala/com/twitter/util/LocalTest.scala @@ -129,6 +129,22 @@ class LocalTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { assert(local() == Some(123)) } + test("Local.let: should only restore the given key, value pair") { + val l1, l2 = new Local[Int] + l1() = 1 + l2() = 2 + val ctx = Local.save() + + l1.let(2) { + l2() = 4 + assert(l1() == Some(2)) + assert(l2() == Some(4)) + } + + assert(l1() == Some(1)) + assert(l2() == Some(4)) + } + test("Local.letClear: should clear Local and restore previous value") { val local = new Local[Int] local() = 123 @@ -138,6 +154,23 @@ class LocalTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { assert(local() == Some(123)) } + test("Local.letClear: should only restore the given key, value pair") { + val l1, l2 = new Local[Int] + l1() = 1 + l2() = 2 + val ctx = Local.save() + + l1.letClear { + assert(l1() == None) + assert(l2() == Some(2)) + l2() = 4 + assert(l2() == Some(4)) + } + + assert(l1() == Some(1)) + assert(l2() == Some(4)) + } + test("Local.clear: should make a copy when clearing") { val l = new Local[Int] l() = 1 From 6be9568264b5c758209fb8a8209d5c4cc76673e5 Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 21 Oct 2020 23:06:29 +0000 Subject: [PATCH 395/796] util: Fix Scaladoc Issues for 2.11 with JDK 11 Problem / Solution A bunch of Scaladoc changes were made recently in order to clean up all of the warnings when building documentation for Util. Unfortunately, when building that documentation using the combination of Scala 2.11 with JDK 11, some of the changes result in an "[error] bad constant pool tag x at byte y" error message, which fails the build. This commit cleans up the offenders that have a problem with the combination of Scala 2.11 and JDK 11. JIRA Issues: CSL-10300 Differential Revision: https://phabricator.twitter.biz/D568686 --- util-app/src/main/scala/com/twitter/app/LoadService.scala | 2 -- .../main/scala/com/twitter/concurrent/AsyncQueue.scala | 8 +++----- .../main/scala/com/twitter/conversions/StringOps.scala | 2 +- .../src/main/scala/com/twitter/io/InputStreamReader.scala | 2 +- .../com/twitter/util/tunable/JsonTunableMapper.scala | 4 ++-- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/LoadService.scala b/util-app/src/main/scala/com/twitter/app/LoadService.scala index ef65a48e40..e80a0fe66c 100644 --- a/util-app/src/main/scala/com/twitter/app/LoadService.scala +++ b/util-app/src/main/scala/com/twitter/app/LoadService.scala @@ -141,8 +141,6 @@ object LoadService { /** * Returns classes for the given `ClassTag` as specified by * resource files in `META-INF/services/FullyQualifiedClassTagsClassName`. - * - * @see [[apply[T](iface:Class[T]):Seq[T]*]] for a Java friendly API. */ def apply[T: ClassTag](): Seq[T] = { val iface = implicitly[ClassTag[T]].runtimeClass.asInstanceOf[Class[T]] diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala index ae0d1d50ff..9333dc8174 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncQueue.scala @@ -25,8 +25,7 @@ object AsyncQueue { /** * An asynchronous FIFO queue. In addition to providing [[offer]] - * and [[poll]], the queue can be [[fail(exc:Throwable,discard:Boolean):Unit* "failed"]], - * flushing current pollers. + * and [[poll]], the queue can be failed, flushing current pollers. * * @param maxPendingOffers optional limit on the number of pending `offers`. * The default is unbounded, but any other positive value can be used to limit @@ -47,8 +46,7 @@ class AsyncQueue[T](maxPendingOffers: Int) { /** * An asynchronous, unbounded, FIFO queue. In addition to providing [[offer]] - * and [[poll]], the queue can be [[fail(exc:Throwable):Unit* "failed"]], flushing current - * pollers. + * and [[poll]], the queue can be failed, flushing current pollers. */ def this() = this(AsyncQueue.UnboundedCapacity) @@ -127,7 +125,7 @@ class AsyncQueue[T](maxPendingOffers: Int) { /** * Drains any pending elements into a `Try[Queue]`. * - * If the queue has been [[fail(exc:Throwable):Unit* failed]] and is now empty, + * If the queue has been failed and is now empty, * a `Throw` of the exception used to fail will be returned. * Otherwise, return a `Return(Queue)` of the pending elements. */ diff --git a/util-core/src/main/scala/com/twitter/conversions/StringOps.scala b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala index e19d52d16d..67d3b7dd70 100644 --- a/util-core/src/main/scala/com/twitter/conversions/StringOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala @@ -82,7 +82,7 @@ object StringOps { /** * Unquote an ASCII string that has been quoted in a style like - * [[quoteC():String*]] and convert it into a standard unicode string. + * `quoteC` and convert it into a standard unicode string. * `"\\uHHHH"` and `"\xHH"` expressions are unpacked * into unicode characters, as well as `"\r"`, `"\n"`, * `"\t"`, `"\\"`, and `'\"'`. diff --git a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala index 6fc175cd7b..50e0028939 100644 --- a/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala +++ b/util-core/src/main/scala/com/twitter/io/InputStreamReader.scala @@ -8,7 +8,7 @@ import java.io.InputStream * Provides the [[Reader]] API for an `InputStream`. * * The given `InputStream` will be closed when [[Reader.read]] - * reaches the EOF or a call to [[discard]] or [[[close()*]]]. + * reaches the EOF or a call to [[discard]] or `close`. */ class InputStreamReader private[io] (inputStream: InputStream, chunkSize: Int, pool: FuturePool) extends Reader[Buf] diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala index 8b6fe1a003..ff3a7add75 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/JsonTunableMapper.scala @@ -82,7 +82,7 @@ object JsonTunableMapper { /** * Parses a given JSON string into a [[TunableMap]]. The expected format is: * - * { + * {{{ * "tunables": * [ * { @@ -97,7 +97,7 @@ object JsonTunableMapper { * "comment": "optional comment" * } * ] - * } + * }}} * * Where \$id1 and \$id2 are unique identifiers used to access the [[Tunable]], \$value is the value, * and \$class is the fully-qualified class name (e.g. com.twitter.util.Duration) From d51e965665848d9730ffd5ee40f274d08e782f73 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 22 Oct 2020 23:18:33 +0000 Subject: [PATCH 396/796] finagle-internal/slo: Mark SLIs as key indicators in Metrics Metadata Problem The user-defined SLIs in SLO can be seen as more important metrics. And we want to mark these metrics in the metrics metadata for future usage. Solution The `keyIndicator` field is already defined in the MetricBuilder and by default the value is false. If a metric is instrumented through SLO indicators, the `keyIndicator` should be filled as true. Introduce the KeyIndicatorMarkingStatsReceiverProxy for this purpose. JIRA Issues: CSL-9961 Differential Revision: https://phabricator.twitter.biz/D568839 --- .../KeyIndicatorMarkingStatsReceiver.scala | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/KeyIndicatorMarkingStatsReceiver.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/KeyIndicatorMarkingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/KeyIndicatorMarkingStatsReceiver.scala new file mode 100644 index 0000000000..eb4239280d --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/KeyIndicatorMarkingStatsReceiver.scala @@ -0,0 +1,23 @@ +package com.twitter.finagle.stats + +/** + * A StatsReceiver proxy that configures all counters, stats, and gauges marked as KeyIndicators. + * This is used with [[ReadableIndicator]]. + * + * @param self The underlying StatsReceiver to which modified schemas are passed + */ +private[finagle] class KeyIndicatorMarkingStatsReceiver(protected val self: StatsReceiver) + extends StatsReceiverProxy { + + override def counter(counterSchema: CounterSchema): Counter = { + self.counter(CounterSchema(counterSchema.metricBuilder.withKeyIndicator())) + } + + override def stat(histogramSchema: HistogramSchema): Stat = { + self.stat(HistogramSchema(histogramSchema.metricBuilder.withKeyIndicator())) + } + + override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { + self.addGauge(GaugeSchema(gaugeSchema.metricBuilder.withKeyIndicator()))(f) + } +} From 83c90273f960078edbb1fa4126a6d8c7f967eeb4 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 27 Oct 2020 01:02:07 +0000 Subject: [PATCH 397/796] Twitter-oss: Prepare OSS libraries for release 20.10.0 Problem We want to release the next versions of our Twitter OSS libraries 20.10.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-10323 Differential Revision: https://phabricator.twitter.biz/D571081 --- CHANGELOG.rst | 3 +++ README.md | 2 +- build.sbt | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 975e441c30..4bcca5c1e7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.10.0 +------- + Bug Fixes ~~~~~~~~~ * util-stat: `MetricBuilder` now uses a configurable metadataScopeSeparator to align diff --git a/README.md b/README.md index 492ce5cae5..53ff20d3b3 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.9.0" +val utilCore = "com.twitter" %% "util-core" % "20.10.0" ``` # Units diff --git a/build.sbt b/build.sbt index 60578a391a..f856ddd34e 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.10.0-SNAPSHOT" +val releaseVersion = "20.10.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 5be08fae0bb7ca91375f93607efc62ad2f87907f Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 27 Oct 2020 15:18:54 +0000 Subject: [PATCH 398/796] Twitter-oss: update OSS libraries post-release to 20.11.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.11.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10323 Differential Revision: https://phabricator.twitter.biz/D571343 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 53ff20d3b3..94ce6b9c25 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index f856ddd34e..0623b79eda 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.10.0" +val releaseVersion = "20.11.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From bda30d5ae519761d2f74bbbefb8c2b4c5499a862 Mon Sep 17 00:00:00 2001 From: Waleed Khan Date: Thu, 29 Oct 2020 18:32:08 +0000 Subject: [PATCH 399/796] util-stats: fix `InMemoryStatsReceiver` to handle printing floats in different locales My laptop is set to use the Polish locale, which causes this test to fail locally. This is probably because in Polish, the thousands separator is `.` and the decimal separator is `,`, which is the reverse of English. (For example, in English, you write `1,234.56`, but in Polish, you write `1.234,56`.) In particular, this assertion fails: ``` assert(parts(5) == "g/h/i 1.000000") ``` because it's actually getting a value in the form `g/h/i 1,000000`. Differential Revision: https://phabricator.twitter.biz/D566297 --- .../com/twitter/finagle/stats/InMemoryStatsReceiver.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 71305afc8e..934ebbb4a4 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -1,6 +1,7 @@ package com.twitter.finagle.stats import java.io.PrintStream +import java.util.Locale import java.util.concurrent.ConcurrentHashMap import scala.collection.{SortedMap, mutable} import scala.jdk.CollectionConverters._ @@ -185,14 +186,15 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { p.println("\nGauges:") p.println("-------") } - for ((k, g) <- sortedGauges) - p.print(f"$k%s ${g()}%f\n") + for ((k, g) <- sortedGauges) { + p.println("%s %f".formatLocal(Locale.US, k, g())) + } if (includeHeaders && sortedStats.nonEmpty) { p.println("\nStats:") p.println("------") } for ((k, s) <- sortedStats if s.nonEmpty) { - p.print(f"$k%s ${s.sum / s.size}%f ${statValuesToStr(s)}\n") + p.println("%s %f %s".formatLocal(Locale.US, k, s.sum / s.size, statValuesToStr(s))) } } From 45340fb4d1a433c09401e11065754b5ffb482108 Mon Sep 17 00:00:00 2001 From: Bonnie Eisenman Date: Fri, 30 Oct 2020 18:49:51 +0000 Subject: [PATCH 400/796] [util-core] Add convenience functions for converting between twitter and jdk 8 duration Problem: JDK8 added a `java.time.Duration` type. However, there is no provided way to convert between a c.t.u.Duration and a java.time.Duration. Solution: Similar to the existing TimeUnit-based convenience functions, add convenience functions for converting between Java and Twitter Durations. JIRA Issues: CSL-10306 Differential Revision: https://phabricator.twitter.biz/D571885 --- CHANGELOG.rst | 5 +++++ .../src/main/scala/com/twitter/util/Duration.scala | 11 +++++++++++ .../test/scala/com/twitter/util/DurationTest.scala | 14 ++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4bcca5c1e7..dd63bec29b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-core: `c.t.util.Duration` now includes `fromJava` and `asJava` conversions to `java.time.Duration` types. ``PHAB_ID=D571885`` + 20.10.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index 763abb5f81..3125fa086b 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -39,6 +39,11 @@ object Duration extends TimeLikeOps[Duration] { fromNanoseconds(ns) } + /** + * Create a duration from a [[java.time.Duration]]. + */ + def fromJava(value: java.time.Duration): Duration = fromNanoseconds(value.toNanos) + // This is needed for Java compatibility. override val Zero: Duration = new Duration(0) @@ -322,6 +327,12 @@ sealed class Duration private[util] (protected val nanos: Long) def inUnit(unit: TimeUnit): Long = unit.convert(inNanoseconds, TimeUnit.NANOSECONDS) + /** + * Converts this value to a [[java.time.Duration]]. + */ + def asJava: java.time.Duration = + java.time.Duration.ofNanos(inNanoseconds) + /** * toString produces a representation that * diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 9b7f540382..3342621b79 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -207,6 +207,20 @@ class DurationTest extends { val ops: Duration.type = Duration } with TimeLikeSp assert((-9999999.seconds).toString == "-115.days-17.hours-46.minutes-39.seconds") } + "convert java.time.Duration" in { + val javaD = java.time.Duration.ofDays(365) + val twitterD = Duration.fromJava(javaD) + assert(twitterD.inDays == 365) + + assert(Duration.fromJava(javaD) == Duration.fromJava(javaD)) + } + + "asJavaDuration" in { + val t0 = Duration.fromDays(10) + val t1 = t0 + 100.nanosecond + assert(t1.asJava.getNano - t0.asJava.getNano == 100) + } + "parse the format from toString" in { Seq( -10.minutes, From ba55bd0e87fbd84a1bbf2954f03ea6f5f86cacdd Mon Sep 17 00:00:00 2001 From: Bonnie Eisenman Date: Thu, 12 Nov 2020 18:14:48 +0000 Subject: [PATCH 401/796] [util/util-stats] Add DenyListStatsReceiver methods Problem There's no existing way to make a DenyListStatsReceiver from a PartialFunction, though a PartialFunction is a natural API for filtering. Solution Add two new convenience APIs to DenyListStatsReceiver. Because there's ambiguity about what should happen in the undefined case, this change provides both an `orElseAdmitted` and an `orElseDenied` method. JIRA Issues: CSL-10134 Differential Revision: https://phabricator.twitter.biz/D576833 --- CHANGELOG.rst | 4 +++ .../finagle/stats/DenylistStatsReceiver.scala | 33 +++++++++++++++++ .../stats/DenylistStatsReceiverTest.scala | 36 +++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dd63bec29b..91fd6e5408 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,10 @@ New Features * util-core: `c.t.util.Duration` now includes `fromJava` and `asJava` conversions to `java.time.Duration` types. ``PHAB_ID=D571885`` +* util-stats: `c.t.finagle.stats.DenylistStatsReceiver` now includes methods for creating + `DenyListStatsReceiver` from partial functions. ``PHAB_ID=D576833`` + + 20.10.0 ------- diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala index a3e23164a5..df6074abed 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala @@ -1,5 +1,38 @@ package com.twitter.finagle.stats +object DenylistStatsReceiver { + + /** + * Creates a DenyListStatsReceiver based on a PartialFunction[Seq[String], Boolean]. + * + * @param underlying a base [[StatsReceiver]]. + * @param pf a PartialFunction that returns true for metrics which should be denylisted, + * and false for metrics which should be recorded. + * If pf is undefined for a given metric name, + * then the metric will NOT be recorded. + */ + def orElseDenied( + underlying: StatsReceiver, + pf: PartialFunction[Seq[String], Boolean] + ): StatsReceiver = + new DenylistStatsReceiver(underlying, pf.orElse { case _ => true }) + + /** + * Creates a DenyListStatsReceiver based on a PartialFunction[Seq[String], Boolean]. + * + * @param underlying a base [[StatsReceiver]]. + * @param pf a PartialFunction that returns true for metrics which should be denylisted, + * and false for metrics which should be recorded. + * If pf is undefined for a given metric name, + * then the metric WILL be recorded. + */ + def orElseAdmitted( + underlying: StatsReceiver, + pf: PartialFunction[Seq[String], Boolean] + ): StatsReceiver = + new DenylistStatsReceiver(underlying, pf.orElse { case _ => false }) +} + /** * A denylisting [[StatsReceiver]]. If the name for a metric is found to be * denylisted, nothing is recorded. diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala index 173efcaf8b..2dd90fc64a 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DenylistStatsReceiverTest.scala @@ -60,5 +60,41 @@ class DenylistStatsReceiverTest extends AnyFunSuite { assert(inmemory.gauges(Seq("foo", "foo", "baz"))() == 3.0) assert(inmemory.stats.isEmpty) } + + test("Denylist.orElseDenied") { + val inmemory = new InMemoryStatsReceiver() + val pf: PartialFunction[Seq[String], Boolean] = { + case s if s.contains("foo") => false + } + val bsr = DenylistStatsReceiver.orElseDenied(inmemory, pf) + + val ctr = bsr.counter("foo", "bar") + ctr.incr() + val gauge = bsr.addGauge("foo", "baz") { 3.0f } + val stat = bsr.stat("qux") + stat.add(3) + + assert(inmemory.counters.nonEmpty) + assert(inmemory.gauges.nonEmpty) + assert(inmemory.stats.isEmpty) + } + + test("Denylist.orElseAdmitted") { + val inmemory = new InMemoryStatsReceiver() + val pf: PartialFunction[Seq[String], Boolean] = { + case s if s.contains("foo") => true + } + val bsr = DenylistStatsReceiver.orElseAdmitted(inmemory, pf) + + val ctr = bsr.counter("foo", "bar") + ctr.incr() + val gauge = bsr.addGauge("baz") { 3.0f } + val stat = bsr.stat("qux") + stat.add(3) + + assert(inmemory.counters.isEmpty) + assert(inmemory.gauges.nonEmpty) + assert(inmemory.stats.nonEmpty) + } // scalafix:on StoreGaugesAsMemberVariables } From 5cc09f7e1a181c3dee3650e0d3ddef9042c8b744 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 16 Nov 2020 20:21:29 +0000 Subject: [PATCH 402/796] changelogs: Mark version 20.4.0 and 20.8.0 do not use Problem We have two versions in the past are released with corrupt jars. Solution With patch released fixed the issue, we want to mark the problematic releases to avoid usage. JIRA Issues: CSL-10338 Differential Revision: https://phabricator.twitter.biz/D579740 --- CHANGELOG.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 91fd6e5408..650e06eada 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -65,8 +65,8 @@ New Features * util-app: Add support for flags of Java Enum types. ``PHAB_ID=D530205`` -20.8.0 ------- +20.8.0 (DO NOT USE) +------------------- New Features ~~~~~~~~~~~~ @@ -135,8 +135,8 @@ Runtime Behavior Changes * util: Bump jackson version to 2.11.0. ``PHAB_ID=D457496`` -20.4.0 ------- +20.4.0 (DO NOT USE) +------------------- New Features ~~~~~~~~~~~~ From f70326eb4a90293a23d0e99a5c7abc64ee497bce Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Mon, 16 Nov 2020 21:13:17 +0000 Subject: [PATCH 403/796] util-core: Activity.apply(Event) and Activity#stabilize should be safer Problem Activity.apply(Event) currently uses Var.apply(Event) to convert events to vars. We've seen problems with this approach in the past, so let's avoid it. Activity#stabilize relies on Activity.apply(Event), so we should fix that too. Solution Activity.apply(Event) has a natural "no current known event" phase, which is `Pending`! We'll propagate witness registry to the underlying Event, and when we're not watching, we'll fill in the gaps by describing it as being in `Pending` state. We'll also rewrite Activity#stabilize to not have to rely on Activity.apply(Event) anymore, since it's incompatible with the new semantics. JIRA Issues: CSL-10156 Differential Revision: https://phabricator.twitter.biz/D574843 --- CHANGELOG.rst | 19 +- .../scala/com/twitter/util/Activity.scala | 21 +- .../scala/com/twitter/util/ActivityTest.scala | 190 +++++++++++++++--- 3 files changed, 195 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 650e06eada..bf4798a553 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,7 +10,24 @@ Unreleased New Features ~~~~~~~~~~~~ -* util-core: `c.t.util.Duration` now includes `fromJava` and `asJava` conversions to `java.time.Duration` types. ``PHAB_ID=D571885`` +* util-core: `c.t.util.Duration` now includes `fromJava` and `asJava` conversions to + `java.time.Duration` types. ``PHAB_ID=D571885`` + + Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-core: `Activity.apply(Event)` will now propagate registry events to the underlying + `Event` instead of registering once and deregistering on garbage collection. This means + that if the underlying `Event` is "notified" while the derived `Activity` is not actively + being observed, it will not pick up the notification. Furthermore, the derived `Activity` + will revert to the `Activity.Pending` state while it is not under observation. ``PHAB_ID=D574843`` + +* util-core: `Activity#stabilize` will now propagate registry events to the underlying + `Activity` instead of registering once and deregistering on garbage collection. This means + that if the underlying `Activity` is changed to a new state while the derived `Activity` is not actively + being observed, it will not update its own state. The derived `Activity` will maintain its last + "stable" state when it's next observed, unless the underlying `Activity` was updated to a new "stable" + state, in which case it will pick that up instead. ``PHAB_ID=D574843`` * util-stats: `c.t.finagle.stats.DenylistStatsReceiver` now includes methods for creating `DenyListStatsReceiver` from partial functions. ``PHAB_ID=D576833`` diff --git a/util-core/src/main/scala/com/twitter/util/Activity.scala b/util-core/src/main/scala/com/twitter/util/Activity.scala index dabc6446a7..b4671dc2af 100644 --- a/util-core/src/main/scala/com/twitter/util/Activity.scala +++ b/util-core/src/main/scala/com/twitter/util/Activity.scala @@ -147,12 +147,17 @@ case class Activity[+T](run: Var[Activity.State[T]]) { * once an [[com.twitter.util.Activity.Ok Ok]] value has been returned * it stops error propagation and instead returns the last Ok value. */ - def stabilize: Activity[T] = - Activity(states.foldLeft[Activity.State[T]](Activity.Pending) { - case (_, next @ Activity.Ok(_)) => next - case (prev @ Activity.Ok(_), _) => prev - case (_, next) => next + def stabilize: Activity[T] = { + @volatile var stable: Activity.State[T] = Activity.Pending + Activity(run.map { + case next @ Activity.Ok(_) => { + stable = next + next + } + case other if stable == Activity.Pending => other + case other => stable }) + } } /** @@ -178,7 +183,11 @@ object Activity { * Constructs an Activity from a state Event. */ def apply[T](states: Event[State[T]]): Activity[T] = - Activity(Var(Pending, states)) + Activity(Var.async[State[T]](Pending) { updatable => + states.respond { state => + updatable() = state + } + }) /** * Collect a collection of activities into an activity of a collection diff --git a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala index 57808afc26..44693c17ac 100644 --- a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala @@ -1,5 +1,6 @@ package com.twitter.util +import com.twitter.conversions.DurationOps._ import java.util.concurrent.atomic.AtomicReference import org.scalatest.funsuite.AnyFunSuite @@ -110,7 +111,7 @@ class ActivityTest extends AnyFunSuite { ) { val p = new Promise[Unit] val obs = Activity.future(p).run.changes.register(Witness((_: Any) => ())) - Await.ready(obs.close()) + Await.result(obs.close(), 5.seconds) assert(!p.isDefined) } @@ -197,47 +198,180 @@ class ActivityTest extends AnyFunSuite { ) } - test("Activity.stabilize") { + test("Activity#stabilize will update until an Ok event") { val ex = new Exception val v = Var[Activity.State[Int]](Activity.Pending) + val a = Activity(v) - val w = Witness(v) val ref = new AtomicReference[Seq[Activity.State[Int]]] a.stabilize.states.build.register(Witness(ref)) assert(ref.get == Seq(Activity.Pending)) - w.notify(Activity.Failed(ex)) + v() = Activity.Failed(ex) assert(ref.get == Seq(Activity.Pending, Activity.Failed(ex))) - w.notify(Activity.Ok(1)) + v() = Activity.Ok(1) assert(ref.get == Seq(Activity.Pending, Activity.Failed(ex), Activity.Ok(1))) + } - w.notify(Activity.Failed(ex)) - assert(ref.get == Seq(Activity.Pending, Activity.Failed(ex), Activity.Ok(1), Activity.Ok(1))) + test("Activity#stabilize will stop updating on non-Oks after an Ok event") { + val ex = new Exception + val v = Var[Activity.State[Int]](Activity.Pending) - w.notify(Activity.Pending) - assert( - ref.get == Seq( - Activity.Pending, - Activity.Failed(ex), - Activity.Ok(1), - Activity.Ok(1), - Activity.Ok(1) - ) - ) + val a = Activity(v) - w.notify(Activity.Ok(2)) - assert( - ref.get == Seq( - Activity.Pending, - Activity.Failed(ex), - Activity.Ok(1), - Activity.Ok(1), - Activity.Ok(1), - Activity.Ok(2) - ) - ) + val ref = new AtomicReference[Seq[Activity.State[Int]]] + a.stabilize.states.build.register(Witness(ref)) + + assert(ref.get == Seq(Activity.Pending)) + + v() = Activity.Ok(1) + assert(ref.get == Seq(Activity.Pending, Activity.Ok(1))) + + v() = Activity.Failed(ex) + assert(ref.get == Seq(Activity.Pending, Activity.Ok(1), Activity.Ok(1))) + + v() = Activity.Pending + assert(ref.get == Seq(Activity.Pending, Activity.Ok(1), Activity.Ok(1), Activity.Ok(1))) + } + + test("Activity#stabilize will also keep the old event on non-Ok after it starts in Ok") { + val ex = new Exception + val v = Var[Activity.State[Int]](Activity.Ok(1)) + + val a = Activity(v) + + val ref = new AtomicReference[Seq[Activity.State[Int]]] + a.stabilize.states.build.register(Witness(ref)) + + assert(ref.get == Seq(Activity.Ok(1))) + + v() = Activity.Failed(ex) + assert(ref.get == Seq(Activity.Ok(1), Activity.Ok(1))) + } + + test("Activity#stabilize can still transition to a different Ok") { + val v = Var[Activity.State[Int]](Activity.Ok(1)) + val a = Activity(v) + + val ref = new AtomicReference[Seq[Activity.State[Int]]] + a.stabilize.states.build.register(Witness(ref)) + + assert(ref.get == Seq(Activity.Ok(1))) + + v() = Activity.Ok(2) + assert(ref.get == Seq(Activity.Ok(1), Activity.Ok(2))) + } + + test("Activity#stabilize will keep the old stable value when not observed") { + val ex = new Exception + val v = Var[Activity.State[Int]](Activity.Ok(1)) + val a = Activity(v).stabilize + + val ref = new AtomicReference[Seq[Activity.State[Int]]] + val c = a.states.build.register(Witness(ref)) + + assert(ref.get == Seq(Activity.Ok(1))) + Await.result(c.close(), 5.seconds) + + assert(a.sample() == 1) + } + + test("Activity#stabilize will ignore new values until it is observed again after deregistration") { + val ex = new Exception + val v = Var[Activity.State[Int]](Activity.Ok(1)) + val a = Activity(v).stabilize + + val ref = new AtomicReference[Seq[Activity.State[Int]]] + val c = a.states.build.register(Witness(ref)) + + assert(ref.get == Seq(Activity.Ok(1))) + Await.result(c.close(), 5.seconds) + + assert(a.sample() == 1) + + // this value does not get cached + v() = Activity.Ok(2) + + // this is the true value of the underlying item + v() = Activity.Pending + + // uses the cached value + assert(a.sample() == 1) + } + + test("Activity#stabilize will see new valid values immediately on observation after deregistration") { + val ex = new Exception + val v = Var[Activity.State[Int]](Activity.Ok(1)) + val a = Activity(v).stabilize + + val ref = new AtomicReference[Seq[Activity.State[Int]]] + val c = a.states.build.register(Witness(ref)) + + assert(ref.get == Seq(Activity.Ok(1))) + Await.result(c.close(), 5.seconds) + + assert(a.sample() == 1) + + v() = Activity.Ok(2) + assert(a.sample() == 2) + } + + class FakeEvent extends Event[Activity.State[Int]] { + var count = 0 + var w: Witness[Activity.State[Int]] = null + + def register(witness: Witness[Activity.State[Int]]): Closable = { + count += 1 + w = witness + Closable.make { deadline: Time => + count -= 1 + w = null + Future.Done + } + } + } + + test("Activity.apply(Event) doesn't leak when sampled") { + val evt = new FakeEvent + val activity = Activity(evt) + assert(evt.count == 0) + + assert(activity.run.sample() == Activity.Pending) + assert(evt.count == 0) + } + + test("Activity.apply(Event) doesn't leak with derived events") { + val evt = new FakeEvent + val activity = Activity(evt) + assert(evt.count == 0) + + val derivedEvt = activity.run.changes + assert(evt.count == 0) + val ref = new AtomicReference[Activity.State[Int]]() + val closable = derivedEvt.register(Witness(ref)) + assert(evt.count == 1) + assert(ref.get == Activity.Pending) + evt.w.notify(Activity.Ok(1)) + assert(Activity.sample(activity) == 1) + assert(evt.count == 1) + Await.result(closable.close(), 5.seconds) + assert(evt.count == 0) + } + + test("Activity.apply(Event) doesn't preserve events after it stops watching") { + val evt = new FakeEvent + val activity = Activity(evt) + + val derivedEvt = activity.run.changes + val ref = new AtomicReference[Activity.State[Int]]() + val closable = derivedEvt.register(Witness(ref)) + evt.w.notify(Activity.Ok(1)) + assert(Activity.sample(activity) == 1) + Await.result(closable.close(), 5.seconds) + + assert(activity.run.sample() == Activity.Pending) } } From e5049992462483322362540ffce39f73af0d44a5 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 18 Nov 2020 19:46:42 +0000 Subject: [PATCH 404/796] [bazel] mark util tests parallelizable Differential Revision: https://phabricator.twitter.biz/D580808 --- .../src/test/scala/com/twitter/app/lifecycle/BUILD | 1 + util-app/src/test/java/BUILD | 1 + util-app/src/test/scala/BUILD | 1 + util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD | 1 + util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD | 1 + util-cache/src/test/java/BUILD | 1 + util-cache/src/test/scala/com/twitter/cache/BUILD | 1 + util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD | 1 + util-codec/src/test/scala/com/twitter/util/BUILD | 1 + util-core/src/test/java/com/twitter/concurrent/BUILD | 1 + util-core/src/test/java/com/twitter/io/BUILD | 1 + util-core/src/test/java/com/twitter/service/BUILD | 1 + util-core/src/test/java/com/twitter/util/BUILD | 1 + util-core/src/test/java/com/twitter/util/javainterop/BUILD | 1 + util-core/src/test/scala/com/twitter/concurrent/BUILD | 1 + util-core/src/test/scala/com/twitter/conversions/BUILD | 1 + util-core/src/test/scala/com/twitter/io/BUILD | 1 + util-core/src/test/scala/com/twitter/util/BUILD | 1 + util-hashing/src/test/scala/com/twitter/hashing/BUILD | 1 + util-inject/src/test/scala/com/twitter/util/inject/BUILD | 1 + .../src/test/scala/com/twitter/util/capturepoints/BUILD | 1 + util-jvm/src/test/java/BUILD | 1 + util-jvm/src/test/scala/BUILD | 1 + util-lint/src/test/scala/com/twitter/util/lint/BUILD | 1 + util-logging/src/test/java/BUILD | 1 + util-logging/src/test/scala/BUILD | 1 + util-mock/src/test/scala/com/twitter/util/mock/BUILD | 1 + util-reflect/src/test/scala/com/twitter/util/reflect/BUILD | 1 + util-registry/src/test/java/BUILD | 1 + util-registry/src/test/scala/BUILD | 1 + util-routing/src/test/java/com/twitter/util/routing/BUILD | 1 + util-routing/src/test/scala/com/twitter/util/routing/BUILD | 1 + util-security/src/test/java/com/twitter/util/security/BUILD | 1 + util-security/src/test/scala/com/twitter/util/security/BUILD | 1 + util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD | 1 + util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD | 1 + util-stats/src/test/java/BUILD | 1 + util-stats/src/test/scala/BUILD | 1 + util-test/src/test/scala/com/twitter/util/testing/BUILD | 1 + util-thrift/src/test/scala/com/twitter/util/BUILD | 1 + util-tunable/src/test/java/com/twitter/util/tunable/BUILD | 1 + util-tunable/src/test/scala/com/twitter/util/tunable/BUILD | 1 + util-zk-test/src/test/java/BUILD | 1 + util-zk-test/src/test/scala/BUILD | 1 + util-zk/src/test/scala/com/twitter/zk/BUILD | 1 + util-zk/src/test/scala/com/twitter/zk/coordination/BUILD | 1 + 46 files changed, 46 insertions(+) diff --git a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD index f1ca84a0c1..32ececd3e5 100644 --- a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD +++ b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-app/src/test/java/BUILD b/util-app/src/test/java/BUILD index 73cb1a6444..c04d96c844 100644 --- a/util-app/src/test/java/BUILD +++ b/util-app/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index 1750c7515f..37ce6c89b8 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD index 204ec7fa8b..198952405c 100644 --- a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/com/google/guava", "3rdparty/jvm/junit", diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD index 8c27750fe4..234b8cf7ab 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/com/google/code/findbugs:jsr305", "3rdparty/jvm/com/google/guava", diff --git a/util-cache/src/test/java/BUILD b/util-cache/src/test/java/BUILD index d5d9a6b412..98b491de57 100644 --- a/util-cache/src/test/java/BUILD +++ b/util-cache/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index 1af0fddf95..e9d74512bd 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -5,6 +5,7 @@ junit_tests( "*.scala", ], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ ":abstract_tests", "3rdparty/jvm/com/github/ben-manes/caffeine", diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index fc69ccd618..c940903901 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/com/google/code/findbugs:jsr305", diff --git a/util-codec/src/test/scala/com/twitter/util/BUILD b/util-codec/src/test/scala/com/twitter/util/BUILD index 9537a62c5a..153c15c7ef 100644 --- a/util-codec/src/test/scala/com/twitter/util/BUILD +++ b/util-codec/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-core/src/test/java/com/twitter/concurrent/BUILD b/util-core/src/test/java/com/twitter/concurrent/BUILD index 9ff36924bc..7df5c79937 100644 --- a/util-core/src/test/java/com/twitter/concurrent/BUILD +++ b/util-core/src/test/java/com/twitter/concurrent/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD index e14e5536d5..a85d8e909e 100644 --- a/util-core/src/test/java/com/twitter/io/BUILD +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/test/java/com/twitter/service/BUILD b/util-core/src/test/java/com/twitter/service/BUILD index 03c081ebc3..ed94882638 100644 --- a/util-core/src/test/java/com/twitter/service/BUILD +++ b/util-core/src/test/java/com/twitter/service/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index 632c62cb01..7a034e0a85 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -14,6 +14,7 @@ junit_tests( "*.java", ], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ ":object_size_calculator", "3rdparty/jvm/com/github/ben-manes/caffeine", diff --git a/util-core/src/test/java/com/twitter/util/javainterop/BUILD b/util-core/src/test/java/com/twitter/util/javainterop/BUILD index 7c178fa76d..d7d64df4e1 100644 --- a/util-core/src/test/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/test/java/com/twitter/util/javainterop/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index b0fd41e8f9..8463984478 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD index 591c088df5..049ebae433 100644 --- a/util-core/src/test/scala/com/twitter/conversions/BUILD +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 2092607878..65f1debc2f 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index 8f2b611846..3c133f5d43 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-hashing/src/test/scala/com/twitter/hashing/BUILD b/util-hashing/src/test/scala/com/twitter/hashing/BUILD index 45f47eb8b6..88b50365b7 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/test/scala/com/twitter/hashing/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", diff --git a/util-inject/src/test/scala/com/twitter/util/inject/BUILD b/util-inject/src/test/scala/com/twitter/util/inject/BUILD index c13d7fd0bc..07728b313a 100644 --- a/util-inject/src/test/scala/com/twitter/util/inject/BUILD +++ b/util-inject/src/test/scala/com/twitter/util/inject/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-testlib", diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD index 2632ef1de6..4fbb4c3c95 100644 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD +++ b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD @@ -2,6 +2,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], strict_deps = True, + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/org/scala-lang/modules:scala-xml", "3rdparty/jvm/org/scalatest", diff --git a/util-jvm/src/test/java/BUILD b/util-jvm/src/test/java/BUILD index 9ecd1b3da6..d58dfaaab6 100644 --- a/util-jvm/src/test/java/BUILD +++ b/util-jvm/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index a6581d7a1f..83848efb47 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -15,6 +15,7 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = COMMON_DEPS + ["3rdparty/jvm/org/scalatestplus:junit"], ) diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index 7099a7af87..59d9cbb8fa 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-logging/src/test/java/BUILD b/util-logging/src/test/java/BUILD index 460923374e..397b0eee4d 100644 --- a/util-logging/src/test/java/BUILD +++ b/util-logging/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 30b94630a2..9bbbd0d745 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-mock/src/test/scala/com/twitter/util/mock/BUILD b/util-mock/src/test/scala/com/twitter/util/mock/BUILD index 32dffaed56..8e28a51df1 100644 --- a/util-mock/src/test/scala/com/twitter/util/mock/BUILD +++ b/util-mock/src/test/scala/com/twitter/util/mock/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/mockito:mockito-scala", diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD index d9a7039540..2d3efd5a8c 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-registry/src/test/java/BUILD b/util-registry/src/test/java/BUILD index 441bf008aa..d0b5cc4eb8 100644 --- a/util-registry/src/test/java/BUILD +++ b/util-registry/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index 35775d0ab3..7c28e3d942 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-routing/src/test/java/com/twitter/util/routing/BUILD b/util-routing/src/test/java/com/twitter/util/routing/BUILD index 2bd7dc18a3..388804e4fd 100644 --- a/util-routing/src/test/java/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/java/com/twitter/util/routing/BUILD @@ -3,6 +3,7 @@ junit_tests( "*.java", ], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index 35acdc1be9..f4bb6e901c 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD index 4e3f011d4d..f8e3dcd324 100644 --- a/util-security/src/test/java/com/twitter/util/security/BUILD +++ b/util-security/src/test/java/com/twitter/util/security/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index 559ea8b8fc..11ac11f450 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD index 4b72642c0c..e0b2ae8b7d 100644 --- a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index b23a6c4266..7bcb2605b0 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -21,6 +21,7 @@ scala_library( junit_tests( compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ ":testutil", "3rdparty/jvm/org/mockito:mockito-scala", diff --git a/util-stats/src/test/java/BUILD b/util-stats/src/test/java/BUILD index 13a4fba0c9..00d649f2f5 100644 --- a/util-stats/src/test/java/BUILD +++ b/util-stats/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-stats/src/test/scala/BUILD b/util-stats/src/test/scala/BUILD index ee68d7b328..62a6571513 100644 --- a/util-stats/src/test/scala/BUILD +++ b/util-stats/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-test/src/test/scala/com/twitter/util/testing/BUILD b/util-test/src/test/scala/com/twitter/util/testing/BUILD index 4846f80967..e8f1d0def4 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/test/scala/com/twitter/util/testing/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-thrift/src/test/scala/com/twitter/util/BUILD b/util-thrift/src/test/scala/com/twitter/util/BUILD index c5543e819f..053408cec9 100644 --- a/util-thrift/src/test/scala/com/twitter/util/BUILD +++ b/util-thrift/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD index e22239aaf7..eacf558a79 100644 --- a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD index cf8ea3414a..8622c6d62b 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-zk-test/src/test/java/BUILD b/util-zk-test/src/test/java/BUILD index dc52d1bd40..3c3e474ba4 100644 --- a/util-zk-test/src/test/java/BUILD +++ b/util-zk-test/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-zk-test/src/test/scala/BUILD b/util-zk-test/src/test/scala/BUILD index e0111f0e8e..a23c01ad33 100644 --- a/util-zk-test/src/test/scala/BUILD +++ b/util-zk-test/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index 529da964b6..059184e0f2 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index 13299b3a06..4234b79a12 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["non-exclusive"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", From 2d7ec5c2ebc419c1642b47a31d445632b5102bef Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Wed, 18 Nov 2020 20:07:56 +0000 Subject: [PATCH 405/796] [util-core, inject-utils]: Move c.t.inject.conversions.string.{toOption, getOrElse} to util-core Problem Finatra's inject-utils/conversions contains useful string conversion methods that would fit better in util-core/conversions/StringOps. Solution Deprecate `toOption` and `getOrElse`, and move the functionality to util-core, along with the corresponding tests. JIRA Issues: CSL-10372 Differential Revision: https://phabricator.twitter.biz/D578549 --- CHANGELOG.rst | 4 +- .../com/twitter/conversions/StringOps.scala | 42 +++++++++++++++++++ .../StringOpsTest.scala} | 22 +++++++++- 3 files changed, 65 insertions(+), 3 deletions(-) rename util-core/src/test/scala/com/twitter/{util/StringConversionsTest.scala => conversions/StringOpsTest.scala} (90%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bf4798a553..813ddab2f4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-core: `c.t.conversions` now includes conversion methods `toOption`, and `getOrElse` + under `StringOps`. ``PHAB_ID=D578549`` + * util-core: `c.t.util.Duration` now includes `fromJava` and `asJava` conversions to `java.time.Duration` types. ``PHAB_ID=D571885`` @@ -32,7 +35,6 @@ New Features * util-stats: `c.t.finagle.stats.DenylistStatsReceiver` now includes methods for creating `DenyListStatsReceiver` from partial functions. ``PHAB_ID=D576833`` - 20.10.0 ------- diff --git a/util-core/src/main/scala/com/twitter/conversions/StringOps.scala b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala index 67d3b7dd70..9e5ad18f02 100644 --- a/util-core/src/main/scala/com/twitter/conversions/StringOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/StringOps.scala @@ -157,6 +157,20 @@ object StringOps { * will return the String `"hello_world"`. */ def toSnakeCase: String = StringOps.toSnakeCase(string) + + /** + * Wrap the string in an Option, returning None when it is null or empty. + */ + def toOption: Option[String] = StringOps.toOption(string) + + /** + * Return the string unless it is null or empty, in which + * case return the default. + * + * @param default the string to return if inputString is null or empty + * @return either the inputString or the default + */ + def getOrElse(default: => String): String = StringOps.getOrElse(string, default) } def hexlify(array: Array[Byte], from: Int, to: Int): String = { @@ -261,6 +275,34 @@ object StringOps { } } + /** + * Wrap the string in an Option, returning None when it is null or empty. + * + * @param inputString the string to wrap in an Option + * @return an Option of the given string + */ + def toOption(inputString: String): Option[String] = { + if (inputString == null || inputString.isEmpty) + None + else + Some(inputString) + } + + /** + * Return the string unless it is null or empty, in which + * case return the default. + * + * @param inputString the string to "get" + * @param default the string to return if inputString is null or empty + * @return either the inputString or the default + */ + def getOrElse(inputString: String, default: => String): String = { + if (inputString == null || inputString.isEmpty) + default + else + inputString + } + implicit final class RichByteArray(val bytes: Array[Byte]) extends AnyVal { /** diff --git a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala b/util-core/src/test/scala/com/twitter/conversions/StringOpsTest.scala similarity index 90% rename from util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala rename to util-core/src/test/scala/com/twitter/conversions/StringOpsTest.scala index eaaa43e682..f00df0459d 100644 --- a/util-core/src/test/scala/com/twitter/util/StringConversionsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/StringOpsTest.scala @@ -14,12 +14,12 @@ * limitations under the License. */ -package com.twitter.util +package com.twitter.conversions import com.twitter.conversions.StringOps._ import org.scalatest.funsuite.AnyFunSuite -class StringConversionsTest extends AnyFunSuite { +class StringOpsTest extends AnyFunSuite { test("string#quoteC") { assert("nothing".quoteC == "nothing") @@ -144,4 +144,22 @@ class StringConversionsTest extends AnyFunSuite { assert(toSnakeCase("Editor2TOC") == "editor2_toc") assert(toSnakeCase("HTML5Editor2TOC") == "html5_editor2_toc") } + + test("string#toOption when nonEmpty") { + assert(toOption("foo") == Some("foo")) + } + + test("string#toOption when empty") { + assert(toOption("") == None) + assert(toOption(null) == None) + } + + test("string#getOrElse when nonEmpty") { + assert(getOrElse("foo", "purple") == "foo") + } + + test("string#getOrElse when empty") { + assert(getOrElse("", "purple") == "purple") + assert(getOrElse(null, "purple") == "purple") + } } From 654a0a374e5d2aae3a10988efb8f71e38041c2dc Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Thu, 19 Nov 2020 00:30:09 +0000 Subject: [PATCH 406/796] [util-core, inject-utils]: Move c.t.inject.conversions.seq to util-core Problem Finatra's inject-utils/conversions contains useful seq conversion methods that would fit better in util-core/conversions. Solution Deprecate everything in `c.t.inject.conversions.seq`, and move the functionality to util-core, along with the corresponding tests. JIRA Issues: CSL-10371 Differential Revision: https://phabricator.twitter.biz/D578605 --- CHANGELOG.rst | 3 + .../com/twitter/conversions/SeqOps.scala | 113 ++++++++++++++++++ .../com/twitter/conversions/SeqOpsTest.scala | 42 +++++++ 3 files changed, 158 insertions(+) create mode 100644 util-core/src/main/scala/com/twitter/conversions/SeqOps.scala create mode 100644 util-core/src/test/scala/com/twitter/conversions/SeqOpsTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 813ddab2f4..9c50463266 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-core: `c.t.conversions` now includes conversion methods for seqs (under `SeqOps`) + that were moved from Finatra. ``PHAB_ID=D578605`` + * util-core: `c.t.conversions` now includes conversion methods `toOption`, and `getOrElse` under `StringOps`. ``PHAB_ID=D578549`` diff --git a/util-core/src/main/scala/com/twitter/conversions/SeqOps.scala b/util-core/src/main/scala/com/twitter/conversions/SeqOps.scala new file mode 100644 index 0000000000..29e0727fe0 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/SeqOps.scala @@ -0,0 +1,113 @@ +package com.twitter.conversions + +import scala.annotation.tailrec + +/** + * Implicits for converting [[Seq]]s. + * + * @example + * {{{ + * import com.twitter.conversions.SeqOps._ + * + * Seq("a", "and").createMap(_.size, _) + * Seq("a", "and").groupBySingleValue(_.size) + * Seq("a", "and").findItemAfter("a") + * + * var numStrings = 0 + * Seq(1, 2) foreachPartial { + * case e: Int => + * numStrings += 1 + * } + * }}} + */ +object SeqOps { + + implicit class RichSeq[A](val self: Seq[A]) extends AnyVal { + + def createMap[K, V](keys: A => K, values: A => V): Map[K, V] = + SeqOps.createMap(self, keys, values) + + def createMap[K, V](values: A => V): Map[A, V] = + SeqOps.createMap(self, values) + + def foreachPartial(pf: PartialFunction[A, Unit]): Unit = + SeqOps.foreachPartial(self, pf) + + /** + * Creates a map from the given sequence by mapping each transformed + * element (the key) by applying the function `keys` to the sequence and + * using the element in the sequence as the value. Chooses the last + * element in the seq when a key collision occurs. + * + * @param keys the function that transforms a given element in the seq to a key + * @return a map of the given sequence + */ + def groupBySingleValue[B](keys: A => B): Map[B, A] = + createMap(keys, identity) + + def findItemAfter(itemToFind: A): Option[A] = + SeqOps.findItemAfter(self, itemToFind) + } + + /** + * Creates a map from the given sequence by applying the functions `keys` + * and `values` to the sequence. + * + * @param sequence the seq to convert to a map + * @param keys the function that transforms a given element in the seq to a key + * @param values the function that transforms a given element in the seq to a value + * @return a map of the given sequence + */ + def createMap[A, K, V](sequence: Seq[A], keys: A => K, values: A => V): Map[K, V] = { + sequence.iterator.map { elem => + keys(elem) -> values(elem) + }.toMap + } + + /** + * Creates a map from the given sequence by mapping each element of the + * sequence (the key) to the transformed element (the value) by applying + * the function `values`. + * + * @param sequence the seq to convert to a map + * @param values the function that transforms a given element in the seq to a value + * @return a map of the given sequence + */ + def createMap[A, K, V](sequence: Seq[A], values: A => V): Map[A, V] = { + sequence.iterator.map { elem => + elem -> values(elem) + }.toMap + } + + /** + * Applies the given partial function to the sequence. + * + * @param sequence the seq onto which the partial function is applied + * @param pf the partial function to apply to the seq + */ + def foreachPartial[A](sequence: Seq[A], pf: PartialFunction[A, Unit]): Unit = { + sequence.foreach { elem => + if (pf.isDefinedAt(elem)) { + pf(elem) + } + } + } + + /** + * Finds the element after the given element to find. + * + * @param sequence the seq to scan for the element + * @param itemToFind the element before the one returned + * @return an Option of the element after the item to find + */ + def findItemAfter[A](sequence: Seq[A], itemToFind: A): Option[A] = { + @tailrec + def recurse(itemToFind: A, seq: Seq[A]): Option[A] = seq match { + case Seq(x, xs @ _*) if x == itemToFind => xs.headOption + case Seq(x, xs @ _*) => recurse(itemToFind, xs) + case Seq() => None + } + recurse(itemToFind, sequence) + } + +} diff --git a/util-core/src/test/scala/com/twitter/conversions/SeqOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/SeqOpsTest.scala new file mode 100644 index 0000000000..d92e84c498 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/conversions/SeqOpsTest.scala @@ -0,0 +1,42 @@ +package com.twitter.conversions + +import com.twitter.conversions.SeqOps._ +import org.scalatest.funsuite.AnyFunSuite + +class SeqOpsTest extends AnyFunSuite { + + test("RichSeq#createMap") { + val map = Seq("a", "and") createMap (_.size, _.toUpperCase) + assert(map == Map(1 -> "A", 3 -> "AND")) + } + + test("RichSeq#groupBySingle chooses last element in seq when key collision occurs") { + val map = Seq("a", "and", "the") groupBySingleValue { _.size } + assert(map == Map(3 -> "the", 1 -> "a")) + } + + test("RichSeq#findItemAfter") { + assert(Seq(1, 2, 3).findItemAfter(1) == Some(2)) + assert(Seq(1, 2, 3).findItemAfter(2) == Some(3)) + assert(Seq(1, 2, 3).findItemAfter(3) == None) + assert(Seq(1, 2, 3).findItemAfter(4) == None) + assert(Seq(1, 2, 3).findItemAfter(5) == None) + assert(Seq[Int]().findItemAfter(5) == None) + + assert(Stream(1, 2, 3).findItemAfter(1) == Some(2)) + assert(Stream[Int]().findItemAfter(5) == None) + + assert(Vector(1, 2, 3).findItemAfter(1) == Some(2)) + assert(Vector[Int]().findItemAfter(5) == None) + } + + test("RichSeq#foreachPartial") { + var numStrings = 0 + Seq("a", 1) foreachPartial { + case str: String => + numStrings += 1 + } + assert(numStrings == 1) + } + +} From 905bbf2abde0384b54ddb95cec26a9fb870d1152 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Thu, 19 Nov 2020 04:19:54 +0000 Subject: [PATCH 407/796] [util-core, inject-utils]: Move c.t.inject.conversions.tuple to util-core Problem Finatra's inject-utils/conversions contains useful tuple conversion methods that would fit better in util-core/conversions. Solution Deprecate everything in `c.t.inject.conversions.tuple`, and move the functionality to util-core, along with the corresponding tests. JIRA Issues: CSL-10370 Differential Revision: https://phabricator.twitter.biz/D578804 --- CHANGELOG.rst | 3 + .../com/twitter/conversions/TupleOps.scala | 172 ++++++++++++++++++ .../twitter/conversions/TupleOpsTest.scala | 47 +++++ 3 files changed, 222 insertions(+) create mode 100644 util-core/src/main/scala/com/twitter/conversions/TupleOps.scala create mode 100644 util-core/src/test/scala/com/twitter/conversions/TupleOpsTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9c50463266..2f566dbf44 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-core: `c.t.conversions` now includes conversion methods for tuples (under `TupleOps`) + that were moved from Finatra. ``PHAB_ID=D578804`` + * util-core: `c.t.conversions` now includes conversion methods for seqs (under `SeqOps`) that were moved from Finatra. ``PHAB_ID=D578605`` diff --git a/util-core/src/main/scala/com/twitter/conversions/TupleOps.scala b/util-core/src/main/scala/com/twitter/conversions/TupleOps.scala new file mode 100644 index 0000000000..3669681599 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/TupleOps.scala @@ -0,0 +1,172 @@ +package com.twitter.conversions + +import scala.collection.{SortedMap, immutable, mutable} +import scala.math.Ordering + +/** + * Implicits for converting tuples. + * + * @example + * {{{ + * import com.twitter.conversions.TupleOps._ + * + * Seq(1 -> "tada", 2 -> "lala").toKeys + * Seq(1 -> "tada", 2 -> "lala").mapValues(_.size) + * Seq(1 -> "tada", 2 -> "lala").groupByKeyAndReduce(_ + "," + _) + * Seq(1 -> "tada", 2 -> "lala").sortByKey + * }}} + */ +object TupleOps { + + implicit class RichTuples[A, B](val self: Iterable[(A, B)]) extends AnyVal { + def toKeys: Seq[A] = TupleOps.toKeys(self) + + def toKeySet: Set[A] = TupleOps.toKeySet(self) + + def toValues: Seq[B] = TupleOps.toValues(self) + + def mapValues[C](func: B => C): Seq[(A, C)] = + TupleOps.mapValues(self, func) + + def groupByKey: Map[A, Seq[B]] = + TupleOps.groupByKey(self) + + def groupByKeyAndReduce(reduceFunc: (B, B) => B): Map[A, B] = + TupleOps.groupByKeyAndReduce(self, reduceFunc) + + def sortByKey(implicit ord: Ordering[A]): Seq[(A, B)] = + TupleOps.sortByKey(self) + + def toSortedMap(implicit ord: Ordering[A]): SortedMap[A, B] = + TupleOps.toSortedMap(self) + } + + /** + * Returns a new seq of the first element of each tuple. + * + * @param tuples the tuples for which all the first elements will be returned + * @return a seq of the first element of each tuple + */ + def toKeys[A, B](tuples: Iterable[(A, B)]): Seq[A] = { + tuples.toSeq map { case (key, value) => key } + } + + /** + * Returns a new set of the first element of each tuple. + * + * @param tuples the tuples for which all the first elements will be returned + * @return a set of the first element of each tuple + */ + def toKeySet[A, B](tuples: Iterable[(A, B)]): Set[A] = { + toKeys(tuples).toSet + } + + /** + * Returns a new seq of the second element of each tuple. + * + * @param tuples the tuples for which all the second elements + * will be returned + * @return a seq of the second element of each tuple + */ + def toValues[A, B](tuples: Iterable[(A, B)]): Seq[B] = { + tuples.toSeq map { case (key, value) => value } + } + + /** + * Applies `func` to the second element of each tuple. + * + * @param tuples the tuples to which the func is applied + * @param func the function literal which will be applied + * to the second element of each tuple + * @return a seq of `func` transformed tuples + */ + def mapValues[A, B, C](tuples: Iterable[(A, B)], func: B => C): Seq[(A, C)] = { + tuples.toSeq map { + case (key, value) => + key -> func(value) + } + } + + /** + * Groups the tuples by matching the first element (key) in each tuple + * returning a map from the key to a seq of the according second elements. + * + * For example: + * + * {{{ + * groupByKey(Seq(1 -> "a", 1 -> "b", 2 -> "ab")) + * }}} + * + * will return Map(2 -> Seq("ab"), 1 -> Seq("a", "b")) + * + * @param tuples the tuples or organize into a grouped map + * @return a map of matching first elements to a sequence of + * their second elements + */ + def groupByKey[A, B](tuples: Iterable[(A, B)]): Map[A, Seq[B]] = { + val mutableMapBuilder = mutable.Map.empty[A, mutable.Builder[B, Seq[B]]] + for ((a, b) <- tuples) { + val seqBuilder = mutableMapBuilder.getOrElseUpdate(a, immutable.Seq.newBuilder[B]) + seqBuilder += b + } + + val mapBuilder = immutable.Map.newBuilder[A, Seq[B]] + for ((k, v) <- mutableMapBuilder) { + mapBuilder += ((k, v.result())) + } + + mapBuilder.result() + } + + /** + * First groups the tuples by matching first elements creating a + * map of the first elements to a seq of the according second elements, + * then reduces the seq to a single value according to the + * applied `reduceFunc`. + * + * For example: + * + * {{{ + * groupByKeyAndReduce(Seq(1 -> 5, 1 -> 6, 2 -> 7), _ + _) + * }}} + * + * will return Map(2 -> 7, 1 -> 11) + * + * @param tuples the tuples or organize into a grouped map + * @param reduceFunc the function to apply to the seq of second + * elements that have been grouped together + * @return a map of matching first elements to a value after + * reducing the grouped values with the `reduceFunc` + */ + def groupByKeyAndReduce[A, B](tuples: Iterable[(A, B)], reduceFunc: (B, B) => B): Map[A, B] = { + // use map instead of mapValues(deprecated since 2.13.0) for cross-building. + groupByKey(tuples).map { + case (k, values) => + k -> values.reduce(reduceFunc) + } + } + + /** + * Sorts the tuples by the first element in each tuple. + * + * @param tuples the tuples to sort + * @param ord the order in which to sort the tuples + * @return the sorted tuples + */ + def sortByKey[A, B](tuples: Iterable[(A, B)])(implicit ord: Ordering[A]): Seq[(A, B)] = { + tuples.toSeq sortBy { case (key, value) => key } + } + + /** + * Sorts the tuples by the first element in each tuple and + * returns a SortedMap. + * + * @param tuples the tuples to sort + * @param ord the order in which to sort the tuples + * @return a SortedMap of the tuples + */ + def toSortedMap[A, B](tuples: Iterable[(A, B)])(implicit ord: Ordering[A]): SortedMap[A, B] = { + SortedMap(tuples.toSeq: _*) + } + +} diff --git a/util-core/src/test/scala/com/twitter/conversions/TupleOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/TupleOpsTest.scala new file mode 100644 index 0000000000..19d6dc947e --- /dev/null +++ b/util-core/src/test/scala/com/twitter/conversions/TupleOpsTest.scala @@ -0,0 +1,47 @@ +package com.twitter.conversions + +import com.twitter.conversions.TupleOps._ +import org.scalatest.funsuite.AnyFunSuite +import scala.collection.SortedMap + +class TupleOpsTest extends AnyFunSuite { + + val tuples = Seq(1 -> "bob", 2 -> "sally") + + test("RichTuple#toKeys") { + assert(tuples.toKeys == Seq(1, 2)) + } + + test("RichTuple#toKeySet") { + assert(tuples.toKeySet == Set(1, 2)) + } + + test("RichTuple#toValues") { + assert(tuples.toValues == Seq("bob", "sally")) + } + + test("RichTuple#mapValues") { + assert(tuples.mapValues { _.length } == Seq(1 -> 3, 2 -> 5)) + } + + test("RichTuple#groupByKey") { + val multiTuples = Seq(1 -> "a", 1 -> "b", 2 -> "ab") + assert(multiTuples.groupByKey == Map(2 -> Seq("ab"), 1 -> Seq("a", "b"))) + } + + test("RichTuple#groupByKeyAndReduce") { + val multiTuples = Seq(1 -> 5, 1 -> 6, 2 -> 7) + assert(multiTuples.groupByKeyAndReduce(_ + _) == Map(2 -> 7, 1 -> 11)) + } + + test("RichTuple#sortByKey") { + val unorderedTuples = Seq(2 -> "sally", 1 -> "bob") + assert(unorderedTuples.sortByKey == Seq(1 -> "bob", 2 -> "sally")) + } + + test("RichTuple#toSortedMap") { + val unorderedTuples = Seq(2 -> "sally", 1 -> "bob") + assert(unorderedTuples.toSortedMap == SortedMap(1 -> "bob", 2 -> "sally")) + } + +} From 3aa49339e21963e434dbb3ce6f6ef80d9b2fcac9 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Thu, 19 Nov 2020 08:17:23 +0000 Subject: [PATCH 408/796] [util-core, inject-utils]: Move c.t.inject.conversions.map.RichMap to util-core Problem Finatra's inject-utils/conversions contains useful map conversion methods that would fit better in util-core/conversions. Solution Deprecate RichMap and its methods, move the functionality to util-core, along with the corresponding tests. JIRA Issues: CSL-10368 Differential Revision: https://phabricator.twitter.biz/D578819 --- CHANGELOG.rst | 3 + .../com/twitter/conversions/MapOps.scala | 140 ++++++++++++++++++ .../com/twitter/conversions/MapOpsTest.scala | 49 ++++++ 3 files changed, 192 insertions(+) create mode 100644 util-core/src/main/scala/com/twitter/conversions/MapOps.scala create mode 100644 util-core/src/test/scala/com/twitter/conversions/MapOpsTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2f566dbf44..b43215839b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-core: `c.t.conversions` now includes conversion methods for maps (under `MapOps`) + that were moved from Finatra. ``PHAB_ID=D578819`` + * util-core: `c.t.conversions` now includes conversion methods for tuples (under `TupleOps`) that were moved from Finatra. ``PHAB_ID=D578804`` diff --git a/util-core/src/main/scala/com/twitter/conversions/MapOps.scala b/util-core/src/main/scala/com/twitter/conversions/MapOps.scala new file mode 100644 index 0000000000..55f2c69f64 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/conversions/MapOps.scala @@ -0,0 +1,140 @@ +package com.twitter.conversions + +import scala.collection.{SortedMap, immutable, mutable} + +/** + * Implicits for converting [[Map]]s. + * + * @example + * {{{ + * import com.twitter.conversions.MapOps._ + * + * Map(1 -> "a").mapKeys { _.toString } + * Map(1 -> "a").invert + * Map(1 -> "a", 2 -> "b").filterValues { _ == "b" } + * Map(2 -> "b", 1 -> "a").toSortedMap + * }}} + */ +object MapOps { + + implicit class RichMap[K, V](val self: Map[K, V]) extends AnyVal { + def mapKeys[T](func: K => T): Map[T, V] = MapOps.mapKeys(self, func) + + def invert: Map[V, Seq[K]] = MapOps.invert(self) + + def invertSingleValue: Map[V, K] = MapOps.invertSingleValue(self) + + def filterValues(func: V => Boolean): Map[K, V] = + MapOps.filterValues(self, func) + + def filterNotValues(func: V => Boolean): Map[K, V] = + MapOps.filterNotValues(self, func) + + def filterNotKeys(func: K => Boolean): Map[K, V] = + MapOps.filterNotKeys(self, func) + + def toSortedMap(implicit ordering: Ordering[K]): SortedMap[K, V] = + MapOps.toSortedMap(self) + } + + /** + * Transforms the keys of the map according to the given `func`. + * + * @param inputMap the map to transform the keys + * @param func the function literal which will be applied to the + * keys of the map + * @return the map with transformed keys and original values + */ + def mapKeys[K, V, T](inputMap: Map[K, V], func: K => T): Map[T, V] = { + for ((k, v) <- inputMap) yield { + func(k) -> v + } + } + + /** + * Inverts the map so that the input map's values are the distinct keys and + * the corresponding keys, represented in a sequence, as the values. + * + * @param inputMap the map to invert + * @return the inverted map + */ + def invert[K, V](inputMap: Map[K, V]): Map[V, Seq[K]] = { + val invertedMapWithBuilderValues = mutable.Map.empty[V, mutable.Builder[K, Seq[K]]] + for ((k, v) <- inputMap) { + val valueBuilder = invertedMapWithBuilderValues.getOrElseUpdate(v, Seq.newBuilder[K]) + valueBuilder += k + } + + val invertedMap = immutable.Map.newBuilder[V, Seq[K]] + for ((k, valueBuilder) <- invertedMapWithBuilderValues) { + invertedMap += (k -> valueBuilder.result) + } + + invertedMap.result() + } + + /** + * Inverts the map so that every input map value becomes the key and the + * input map key becomes the value. + * + * @param inputMap the map to invert + * @return the inverted map + */ + def invertSingleValue[K, V](inputMap: Map[K, V]): Map[V, K] = { + inputMap map { _.swap } + } + + /** + * Filters the pairs in the map which values satisfy the predicate + * represented by `func`. + * + * @param inputMap the map which will be filtered + * @param func the predicate that needs to be satisfied to + * select a key-value pair + * @return the filtered map + */ + def filterValues[K, V](inputMap: Map[K, V], func: V => Boolean): Map[K, V] = { + inputMap filter { + case (_, value) => + func(value) + } + } + + /** + * Filters the pairs in the map which values do NOT satisfy the + * predicate represented by `func`. + * + * @param inputMap the map which will be filtered + * @param func the predicate that needs to be satisfied to NOT + * select a key-value pair + * @return the filtered map + */ + def filterNotValues[K, V](inputMap: Map[K, V], func: V => Boolean): Map[K, V] = + filterValues(inputMap, (v: V) => !func(v)) + + /** + * Filters the pairs in the map which keys do NOT satisfy the + * predicate represented by `func`. + * + * @param inputMap the map which will be filtered + * @param func the predicate that needs to be satisfied to NOT + * select a key-value pair + * @return the filtered map + */ + def filterNotKeys[K, V](inputMap: Map[K, V], func: K => Boolean): Map[K, V] = { + // use filter instead of filterKeys(deprecated since 2.13.0) for cross-building. + inputMap.filter { case (key, _) => !func(key) } + } + + /** + * Sorts the map by the keys and returns a SortedMap. + * + * @param inputMap the map to sort + * @param ordering the order in which to sort the map + * @return a SortedMap + */ + def toSortedMap[K, V](inputMap: Map[K, V])(implicit ordering: Ordering[K]): SortedMap[K, V] = { + SortedMap[K, V](inputMap.toSeq: _*) + } + +} diff --git a/util-core/src/test/scala/com/twitter/conversions/MapOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/MapOpsTest.scala new file mode 100644 index 0000000000..6370367e7c --- /dev/null +++ b/util-core/src/test/scala/com/twitter/conversions/MapOpsTest.scala @@ -0,0 +1,49 @@ +package com.twitter.conversions + +import com.twitter.conversions.MapOps._ +import org.scalatest.funsuite.AnyFunSuite +import scala.collection.SortedMap + +class MapOpsTest extends AnyFunSuite { + + test("RichMap#mapKeys") { + val mappedKeys = Map(1 -> "a") mapKeys { _.toString } + assert(mappedKeys == Map("1" -> "a")) + } + + test("RichMap#invert simple") { + val invertedMap = Map(1 -> "a").invert + assert(invertedMap == Map("a" -> Iterable(1))) + } + + test("RichMap#invert complex") { + val complexInverted = Map(1 -> "a", 2 -> "a", 3 -> "b").invert + assert(complexInverted == Map("a" -> Iterable(1, 2), "b" -> Iterable(3))) + } + + test("RichMap#invertSingleValue") { + val singleValueInverted = Map(1 -> "a", 2 -> "a", 3 -> "b").invertSingleValue + assert(singleValueInverted == Map("a" -> 2, "b" -> 3)) + } + + test("RichMap#filterValues") { + val filteredMap = Map(1 -> "a", 2 -> "a", 3 -> "b") filterValues { _ == "b" } + assert(filteredMap == Map(3 -> "b")) + } + + test("RichMap#filterNotValues") { + val filteredMap = Map(1 -> "a", 2 -> "a", 3 -> "b") filterNotValues { _ == "b" } + assert(filteredMap == Map(1 -> "a", 2 -> "a")) + } + + test("RichMap#filterNotKeys") { + val filteredMap = Map(1 -> "a", 2 -> "a", 3 -> "b") filterNotKeys { _ == 3 } + assert(filteredMap == Map(1 -> "a", 2 -> "a")) + } + + test("RichMap#toSortedMap") { + val sortedMap = Map(6 -> "f", 2 -> "b", 1 -> "a").toSortedMap + assert(sortedMap == SortedMap(1 -> "a", 2 -> "b", 6 -> "f")) + } + +} From e95799b8079e018bd3d806e84191990633d751ed Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 20 Nov 2020 16:16:02 +0000 Subject: [PATCH 409/796] util-core: remove com.twitter.util.Config Problem: util-core's Config has been deprecated for several years and it blocks cross-building against Scala 3. It is still difficult to remove Config altogether. Solution: remove Config from util-internal JIRA Issues: CSL-10378 Differential Revision: https://phabricator.twitter.biz/D580444 --- CHANGELOG.rst | 4 + .../main/scala/com/twitter/util/Config.scala | 187 ------------------ .../scala/com/twitter/util/ConfigTest.scala | 89 --------- 3 files changed, 4 insertions(+), 276 deletions(-) delete mode 100644 util-core/src/main/scala/com/twitter/util/Config.scala delete mode 100644 util-core/src/test/scala/com/twitter/util/ConfigTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b43215839b..8cdd37180e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ +* util-core: removed `com.twitter.util.Config.` ``PHAB_ID=D580444`` + New Features ~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/util/Config.scala b/util-core/src/main/scala/com/twitter/util/Config.scala deleted file mode 100644 index fbd834ba98..0000000000 --- a/util-core/src/main/scala/com/twitter/util/Config.scala +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2011 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.twitter.util - -import java.io.Serializable -import scala.collection.mutable -import scala.language.implicitConversions - -/** - * You can import Config._ if you want the auto-conversions in a class - * that does not inherit from trait Config. - */ -@deprecated("use a Plain Old Scala Object", "") -object Config { - sealed trait Required[+A] extends Serializable { - def value: A - def isSpecified: Boolean - def isEmpty: Boolean = !isSpecified - } - - object Specified { - def unapply[A](specified: Specified[A]): Some[A] = Some(specified.value) - } - - class Specified[+A](f: => A) extends Required[A] { - lazy val value: A = f - def isSpecified: Boolean = true - } - - case object Unspecified extends Required[scala.Nothing] { - def value: scala.Nothing = throw new NoSuchElementException - def isSpecified: Boolean = false - } - - class RequiredValuesMissing(names: Seq[String]) extends Exception(names.mkString(",")) - - /** - * Config classes that don't produce anything via an apply() method - * can extends Config.Nothing, and still get access to the Required - * classes and methods. - */ - class Nothing extends Config[scala.Nothing] { - def apply(): scala.Nothing = throw new UnsupportedOperationException - } - - implicit def toSpecified[A](value: => A): Specified[A] = new Specified(value) - implicit def toSpecifiedOption[A](value: => A): Specified[Some[A]] = - new Specified(Some(value)) - implicit def fromRequired[A](req: Required[A]): A = req.value - implicit def intoOption[A](item: A): Option[A] = Some(item) - implicit def intoList[A](item: A): List[A] = List(item) -} - -/** - * A config object contains vars for configuring an object of type T, and an apply() method - * which turns this config into an object of type T. - * - * The trait also contains a few useful implicits. - * - * You can define fields that are required but that don't have a default value with: - * class BotConfig extends Config[Bot] { - * var botName = required[String] - * var botPort = required[Int] - * .... - * } - * - * Optional fields can be defined with: - * var something = optional[Duration] - * - * Fields that are dependent on other fields and have a default value computed - * from an expression should be marked as computed: - * - * var level = required[Int] - * var nextLevel = computed { level + 1 } - */ -@deprecated("use a Plain Old Scala Object", "") -trait Config[T] extends (() => T) { - import Config.{Required, Specified, Unspecified, RequiredValuesMissing} - - /** - * a lazy singleton of the configured class. Allows one to pass - * the Config around to multiple classes without producing multiple - * configured instances - */ - lazy val memoized: T = apply() - - def required[A]: Required[A] = Unspecified - def required[A](default: => A): Required[A] = new Specified(default) - def optional[A]: Required[Option[A]] = new Specified(None) - def optional[A](default: => A): Required[Option[A]] = new Specified(Some(default)) - - /** - * The same as specifying required[A] with a default value, but the intent to the - * use is slightly different. - */ - def computed[A](f: => A): Required[A] = new Specified(f) - - /** a non-lazy way to create a Specified that can be called from java */ - def specified[A](value: A): Specified[A] = new Specified(value) - - implicit def toSpecified[A](value: => A): Specified[A] = new Specified(value) - implicit def toSpecifiedOption[A](value: => A): Specified[Some[A]] = - new Specified(Some(value)) - implicit def fromRequired[A](req: Required[A]): A = req.value - implicit def intoOption[A](item: A): Option[A] = Some(item) - implicit def intoList[A](item: A): List[A] = List(item) - - /** - * Looks for any fields that are Required but currently Unspecified, in - * this object and in any child Config objects, and returns the names - * of those missing values. For missing values in child Config objects, - * the name is the dot-separated path down to that value. - * - * @note as of Scala 2.12 the returned values are not always correct. - * See `ConfigTest` for an example. - */ - def missingValues: Seq[String] = { - val alreadyVisited = mutable.Set[AnyRef]() - val interestingReturnTypes = Seq(classOf[Required[_]], classOf[Config[_]], classOf[Option[_]]) - val buf = new mutable.ListBuffer[String] - val mirror = scala.reflect.runtime.universe.runtimeMirror(getClass.getClassLoader) - def collect(prefix: String, config: Config[_]): Unit = { - if (!alreadyVisited.contains(config)) { - alreadyVisited += config - val nullaryMethods = config.getClass.getMethods.toSeq filter { _.getParameterTypes.isEmpty } - val syntheticTermNames: Set[String] = { - val configSymbol = mirror.reflect(config).symbol - val configMembers = configSymbol.toType.members - configMembers.iterator.collect { - case symbol if symbol.isTerm && symbol.isSynthetic => - symbol.name.decodedName.toString - }.toSet - } - for (method <- nullaryMethods) { - val name = method.getName - val rt = method.getReturnType - if (name != "required" && - name != "optional" && - !syntheticTermNames.contains(name) && // no loops, no compiler generated methods! - interestingReturnTypes.exists(_.isAssignableFrom(rt))) { - method.invoke(config) match { - case Unspecified => - buf += (prefix + name) - case specified: Specified[_] => - specified match { - case Specified(sub: Config[_]) => - collect(prefix + name + ".", sub) - case Specified(Some(sub: Config[_])) => - collect(prefix + name + ".", sub) - case _ => - } - case sub: Config[_] => - collect(prefix + name + ".", sub) - case Some(sub: Config[_]) => - collect(prefix + name + ".", sub) - case _ => - } - } - } - } - } - collect("", this) - buf.toList - } - - /** - * @note Throws `RequiredValuesMissing` if there are any missing values. - */ - def validate(): Unit = { - val missing = missingValues - if (missing.nonEmpty) throw new RequiredValuesMissing(missing) - } -} diff --git a/util-core/src/test/scala/com/twitter/util/ConfigTest.scala b/util-core/src/test/scala/com/twitter/util/ConfigTest.scala deleted file mode 100644 index 60f4f1040c..0000000000 --- a/util-core/src/test/scala/com/twitter/util/ConfigTest.scala +++ /dev/null @@ -1,89 +0,0 @@ -package com.twitter.util - -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -class ConfigTest extends AnyWordSpec with Matchers { - import Config._ - - "Config" should { - "computed should delay evaluation" in { - class Foo extends Config.Nothing { - var didIt = false - var x = 10 - var y = computed { - didIt = true - x * 2 + 5 - } - } - - val foo = new Foo - assert(!foo.didIt) - assert((foo.y: Int) == 25) // use type annotation to force implicit conversion - assert(foo.didIt) - } - - "subclass can override indepedent var for use in dependent var" in { - class Foo extends Config.Nothing { - var x = 10 - var y = computed(x * 2 + 5) - } - val bar = new Foo { - x = 20 - } - assert((bar.y: Int) == 45) // use type annotation to force implicit conversion - } - - class Bar extends Config.Nothing { - var z = required[Int] - } - class Baz extends Config.Nothing { - var w = required[Int] - } - class Foo extends Config.Nothing { - var x = required[Int] - var y = 3 - var bar = required[Bar] - var baz = optional[Baz] - } - - "missingValues" should { - "must return empty Seq when no values are missing" in { - val foo = new Foo { - x = 42 - bar = new Bar { - z = 10 - } - } - assert(foo.missingValues.isEmpty) - } - - "must find top-level missing values" in { - val foo = new Foo - assert(foo.missingValues.sorted == Seq("x", "bar").sorted) - } - - "must find top-level and nested missing values" in { - val foo = new Foo { - bar = new Bar - } - assert(foo.missingValues.sorted == Seq("x", "bar.z").sorted) - } - - "must find nested missing values in optional sub-configs" in { - val foo = new Foo { - x = 3 - bar = new Bar { - z = 1 - } - baz = new Baz - } - // we can't do an exact match here since in scala 2.12.x - // this looks something like `List("$anonfun$new$16.w", "baz.w")`. - // this code is deprecated and we want to stop supporting it. - // so we simplify the test. - assert(foo.missingValues.contains("baz.w")) - } - } - } -} From 22fedfd1eb17938f6d730b024db414051bcb9f2f Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 23 Nov 2020 22:05:44 +0000 Subject: [PATCH 410/796] util-core: Add FuturePool.numPendingTasks Problem FuturePools export some stats about the task they run. There is a way to get how many tasks are currently running and how many tasks were run overall. There is no way to get how many tasks are pending. Solution Expose `numPendingTasks` that, if supposed by the underlying scheduler, reports how many tasks are sitting in the queue, waiting to be executed. JIRA Issues: CSL-10316 Differential Revision: https://phabricator.twitter.biz/D583030 --- CHANGELOG.rst | 3 +++ .../src/main/scala/com/twitter/util/FuturePool.scala | 11 +++++++++++ .../test/scala/com/twitter/util/FuturePoolTest.scala | 9 +++++++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8cdd37180e..3e2aaf2e7a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -48,6 +48,9 @@ New Features * util-stats: `c.t.finagle.stats.DenylistStatsReceiver` now includes methods for creating `DenyListStatsReceiver` from partial functions. ``PHAB_ID=D576833`` +* util-core: `c.t.util.FuturePool` now supports exporting the number of its pending tasks via + `numPendingTasks`. ``PHAB_ID=D582979`` + 20.10.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index 9ba549dd6e..52ed0d464b 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -39,6 +39,13 @@ trait FuturePool { * @return -1 if this [[FuturePool]] implementation doesn't support this statistic. */ def numCompletedTasks: Long = -1L + + /** + * The approximate number of tasks that are pending (typically waiting in the queue). + * + * @return -1 if this [[FuturePool]] implementation doesn't support this statistic. + */ + def numPendingTasks: Long = -1 } /** @@ -189,4 +196,8 @@ class ExecutorServiceFuturePool protected[this] ( case _ => super.numCompletedTasks } + override def numPendingTasks: Long = executor match { + case tpe: ThreadPoolExecutor => tpe.getQueue.size() + case _ => super.numPendingTasks + } } diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index a7ccc7cc7d..f8bf342179 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -200,12 +200,13 @@ class FuturePoolTest extends AnyFunSuite with Eventually { // But we want to make sure it will have the correct behavior. // We compromise by roughly creating an ExecutorService/FuturePool // that behaves the same. - val executor = Executors.newCachedThreadPool() + val executor = Executors.newFixedThreadPool(1) val pool = new ExecutorServiceFuturePool(executor) // verify the initial state assert(pool.poolSize == 0) assert(pool.numActiveTasks == 0) assert(pool.numCompletedTasks == 0) + assert(pool.numPendingTasks == 0) // execute a task we can control val latch = new CountDownLatch(1) @@ -213,16 +214,20 @@ class FuturePoolTest extends AnyFunSuite with Eventually { latch.await(10, TimeUnit.SECONDS) true } + pool(()) // pending task + eventually { assert(pool.poolSize == 1) } eventually { assert(pool.numActiveTasks == 1) } eventually { assert(pool.numCompletedTasks == 0) } + eventually { assert(pool.numPendingTasks == 1) } // let the task complete latch.countDown() Await.ready(future, 5.seconds) eventually { assert(pool.poolSize == 1) } eventually { assert(pool.numActiveTasks == 0) } - eventually { assert(pool.numCompletedTasks == 1) } + eventually { assert(pool.numCompletedTasks == 2) } + eventually { assert(pool.numPendingTasks == 0) } // cleanup. executor.shutdown() From 5a6837e5055db88abb705390a06c5e75c1898136 Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 9 Dec 2020 20:24:08 +0000 Subject: [PATCH 411/796] util, finagle, finatra, scrooge: Bump sbt-jmh plugin version to 0.4.0 Problem: The latest version of the sbt-jmh plugin for running JMH microbenchmarking is 0.4.0. This version supports sbt 1.3.0+ and JMH 1.25. One feature of JMH 1.25 that we can take advantage of is the async profiler for profiling CPU cycles, cache misses, and stack traces. We are currently on 0.3.4. Solution: Bump the sbt-jmh plugin version to 0.4.0 JIRA Issues: CSL-10421 Differential Revision: https://phabricator.twitter.biz/D588819 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index f7e8c4cd8b..b7c24e916b 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,4 +4,4 @@ resolvers += Resolver.sonatypeRepo("snapshots") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.0") addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC13") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") -addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.4") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.0") From b97abfdd45dba1a6d181e0b097adff4dbdceb63a Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 10 Dec 2020 18:25:11 +0000 Subject: [PATCH 412/796] Twitter-oss: Prepare OSS libraries for release 20.12.0 Problem We want to release the next versions of our Twitter OSS libraries 20.12.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-10389 Differential Revision: https://phabricator.twitter.biz/D589851 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3e2aaf2e7a..4d9bb88e82 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +20.12.0 +------- + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ * util-core: removed `com.twitter.util.Config.` ``PHAB_ID=D580444`` diff --git a/README.md b/README.md index 94ce6b9c25..b52e297dd4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.10.0" +val utilCore = "com.twitter" %% "util-core" % "20.12.0" ``` # Units diff --git a/build.sbt b/build.sbt index 0623b79eda..4fbd4e8205 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.11.0-SNAPSHOT" +val releaseVersion = "20.12.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From ad1c88a96a27aff5688b880e3aef1c8c4debfccb Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 10 Dec 2020 21:07:12 +0000 Subject: [PATCH 413/796] Twitter-oss: update OSS libraries post-release to 21.1.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10389 Differential Revision: https://phabricator.twitter.biz/D591241 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b52e297dd4..c8e8b86363 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 4fbd4e8205..b0a2700816 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "20.12.0" +val releaseVersion = "21.1.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 2985dba7c81c2b451fe64c4241a8f072add7dbba Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 11 Dec 2020 17:34:34 +0000 Subject: [PATCH 414/796] util: Fix the phab id for one of the changelog entries Problem / Solution One of the phab ids is incorrect. Fix it Differential Revision: https://phabricator.twitter.biz/D591919 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4d9bb88e82..10f83e1aba 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -52,7 +52,7 @@ New Features `DenyListStatsReceiver` from partial functions. ``PHAB_ID=D576833`` * util-core: `c.t.util.FuturePool` now supports exporting the number of its pending tasks via - `numPendingTasks`. ``PHAB_ID=D582979`` + `numPendingTasks`. ``PHAB_ID=D583030`` 20.10.0 ------- From d405c0064a7a0e14a09322641520b8e6a8fc1982 Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 17 Dec 2020 20:17:11 +0000 Subject: [PATCH 415/796] util-benchmark: add unit, void, filter and map benchmarks Problem: We don't have benchmark tests for some common operators in ConstFuture. We also don't benchmark `.map` and `.filter` for satisfying Promises. (There's `timeMap()`, but it only accounts for the creation of a Promise and registering it on the wait queue). This makes it hard to measure optimization changes. Solution: Add benchmarks for ConstFuture.unit, ConstFuture.void, ConstFuture.filter, Promise.filter, Promise.map JIRA Issues: CSL-10459 Differential Revision: https://phabricator.twitter.biz/D595080 --- .../com/twitter/util/FutureBenchmark.scala | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala index 4095af37ba..65ebf75e35 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala @@ -12,6 +12,14 @@ class FutureBenchmark extends StdBenchAnnotations { def timePromise(): Future[Unit] = new Promise[Unit] + @Benchmark + def timeConstUnit(): Future[Unit] = + Future.Done.unit + + @Benchmark + def timeConstVoid(): Future[Void] = + Future.Done.voided + @Benchmark def timeBy(state: ByState): Future[Unit] = { import state._ @@ -89,6 +97,10 @@ class FutureBenchmark extends StdBenchAnnotations { def fromConstTransformToConst: Future[Unit] = Future.Done.transform(TransformFn) + @Benchmark + def timeConstFilter(): Future[Unit] = + Future.Done.filter(FilterFn) + @Benchmark def timeSelect(state: SelectState): Future[(Try[Unit], Seq[Future[Unit]])] = { import state._ @@ -218,11 +230,15 @@ class FutureBenchmark extends StdBenchAnnotations { val fmap = { i: Int => Future.value(i + 1) } val respond = { _: Try[Int] => sum += 1 } + val map = { i: Int => i + 1 } + val filter = { _: Int => true } while (i < state.depth) { - next = i % 2 match { + next = i % 4 match { case 0 => next.flatMap(fmap) case 1 => next.respond(respond) + case 2 => next.map(map) + case 3 => next.filter(filter) } i += 1 } @@ -260,6 +276,7 @@ object FutureBenchmark { private val FlatMapFn: Unit => Future[Unit] = { _: Unit => Future.Unit } private val MapFn: Unit => Unit = { _: Unit => () } private val TransformFn: Try[Unit] => Future[Unit] = { _: Try[Unit] => Future.Done } + private val FilterFn: Unit => Boolean = { _ => true } @State(Scope.Benchmark) class ByState { From f326ea126c17167fa2c983b4b7311565f71daba5 Mon Sep 17 00:00:00 2001 From: Bonnie Eisenman Date: Tue, 5 Jan 2021 18:43:10 +0000 Subject: [PATCH 416/796] [misc] Add self to PROJECT files Adding myself to various PROJECT files based on where I've made changes. Differential Revision: https://phabricator.twitter.biz/D598443 --- util-stats/PROJECT | 1 + 1 file changed, 1 insertion(+) diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 244fc53688..41b2a610e9 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -2,6 +2,7 @@ owners: - banderson + - beisenman - koliver - mnakamura - roanta From 593da6f666e156ef08abf2cb23ed36b7ad654250 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 5 Jan 2021 18:46:07 +0000 Subject: [PATCH 417/796] util-*: Fix dependency to use scalatestplus mockito-3-3 ## Problem mockitoVersion is already 3.3.3, but scalatestplus currently incorrectly uses mockito-3-2, which transitively depends on Mockito 3.2 as the name suggests. This seems to present no issue on Pants or sbt since Coursier would be able to resolve to the later version of Mockito, and apparently ScalaTestPlus is ok with having a different Mockito version on the classapth. However, this presents an issue for Bazel using bazel-multiversion where we'd either have to choose between using the latest Mockito version in Source, or declare that Mockito uses pvp versionScheme, at which point we would actually have both Mockito 3.2 and Mockito 3.3 on the classpath. ## Solution The solution is to upgrade scalatestplus to the correct mockito-3-3 module. Differential Revision: https://phabricator.twitter.biz/D598104 --- build.sbt | 14 +++++++------- util-app/src/test/scala/BUILD | 2 +- util-cache/src/test/scala/com/twitter/cache/BUILD | 2 +- .../test/scala/com/twitter/cache/caffeine/BUILD | 2 +- .../src/test/scala/com/twitter/concurrent/BUILD | 2 +- util-core/src/test/scala/com/twitter/io/BUILD | 2 +- util-core/src/test/scala/com/twitter/util/BUILD | 2 +- util-jvm/src/test/scala/BUILD | 2 +- .../src/test/scala/com/twitter/util/lint/BUILD | 2 +- util-registry/src/test/scala/BUILD | 2 +- .../src/test/scala/com/twitter/util/routing/BUILD | 2 +- util-zk/src/test/scala/com/twitter/zk/BUILD | 2 +- .../test/scala/com/twitter/zk/coordination/BUILD | 2 +- 13 files changed, 19 insertions(+), 19 deletions(-) diff --git a/build.sbt b/build.sbt index b0a2700816..121c2e29a4 100644 --- a/build.sbt +++ b/build.sbt @@ -226,7 +226,7 @@ lazy val utilApp = Project( name := "util-app", libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" ) ).dependsOn(utilAppLifecycle, utilCore, utilRegistry) @@ -261,7 +261,7 @@ lazy val utilCache = Project( caffeineLib, jsr305Lib, "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" ) ).dependsOn(utilCore) @@ -309,7 +309,7 @@ lazy val utilCore = Project( "org.scala-lang" % "scala-reflect" % scalaVersion.value, "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2", "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test", + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ), resourceGenerators in Compile += Def.task { @@ -398,7 +398,7 @@ lazy val utilJvm = Project( name := "util-jvm", libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" ) ).dependsOn(utilApp, utilCore, utilStats) @@ -443,7 +443,7 @@ lazy val utilRegistry = Project( name := "util-registry", libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" ) ).dependsOn(utilCore) @@ -514,7 +514,7 @@ lazy val utilStats = Project( jsr305Lib, scalacheckLib, "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test", + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ) ++ { CrossVersion.partialVersion(scalaVersion.value) match { @@ -580,7 +580,7 @@ lazy val utilZk = Project( libraryDependencies ++= Seq( zkDependency, "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-2" % "3.1.2.0" % "test" + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" ) ).dependsOn(utilCore, utilLogging) diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index 37ce6c89b8..e0a7f5da4d 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "util/util-app/src/main/scala", "util/util-app/src/test/resources", "util/util-core:util-core-util", diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index e9d74512bd..ce5e374b2e 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -13,7 +13,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-core:util-core-util", diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index c940903901..928759b970 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -9,7 +9,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index 8463984478..10d8afc5cf 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -8,7 +8,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 65f1debc2f..f66980dee1 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -8,7 +8,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index 3c133f5d43..53c653d3e2 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -8,7 +8,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index 83848efb47..7a9cd28710 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -2,7 +2,7 @@ COMMON_DEPS = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "util/util-app/src/main/scala", "util/util-core:util-core-util", "util/util-jvm/src/main/scala/com/twitter/jvm", diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index 59d9cbb8fa..6ff2a42d8f 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "util/util-lint/src/main/scala/com/twitter/util/lint", ], ) diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index 7c28e3d942..e23721ba6f 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index f4bb6e901c..065cec5c54 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -7,7 +7,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "util/util-routing/src/main/scala/com/twitter/util/routing", "util/util-routing/src/test/scala/com/twitter/util/routing/dynamic", "util/util-routing/src/test/scala/com/twitter/util/routing/simple", diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index 059184e0f2..ac938e70b0 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -8,7 +8,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-logging/src/main/scala/com/twitter/logging", diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index 4234b79a12..3c07ce463a 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -8,7 +8,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-2", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-zk/src/main/scala/com/twitter/zk", From f70930ea94ce9be15a94a54e48378d14a891071e Mon Sep 17 00:00:00 2001 From: Lily He Date: Fri, 8 Jan 2021 21:03:02 +0000 Subject: [PATCH 418/796] util-core: Add comment on why transformTry uses fewer allocations Problem / Solution: Previous experimentation of removing transformTry showed that there would be an increase in allocations if transformTry is replaced with transform. Record learnings in comments for future record. JIRA Issues: CSL-10479 Differential Revision: https://phabricator.twitter.biz/D597678 --- util-core/src/main/scala/com/twitter/util/Future.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 20a2350d1b..2aa48fe612 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1773,9 +1773,10 @@ abstract class Future[+A] extends Awaitable[A] { self => * When this future completes, run `f` on that completed result * whether or not this computation was successful. * - * This method is similar to `transform`, but the transformation is - * applied without introducing an intermediate future, which leads - * to better performance. + * @note This method is similar to `transform`, but the transformation is + * applied without introducing an intermediate future, which leads + * to fewer allocations. The promise is satisfied directly instead + * of wrapping a Future around a Try. * * The returned `Future` will be satisfied when `this`, * the original future, is done. From c33d193259e7867e1f3c31a2add6449d4633c41a Mon Sep 17 00:00:00 2001 From: Igor Daniel Servulo Date: Mon, 11 Jan 2021 21:12:20 +0000 Subject: [PATCH 419/796] Bumping zookeeper version due to security issues with Netty Problem The most recent release from finagle is using org.apache.zookeeper/zookeeper on version 3.5.0-alpha. This zookeeper version is using io.netty/netty on version 3.7.0 which introduces several vulnerable scenarios on the code. Solution The solution for this issue was bumping zkVersion on the project, to an a compatible version of zookeeper that is using netty in a newer version (most precisely, zookeeper was bumped to version 3.5.6). Result The result of this pull-request will be finagle be free from these vulnerable scenarios from this outdated dependency. Closes #887. Signed-off-by: Yufan Gong JIRA Issues: CSL-10537 Differential Revision: https://phabricator.twitter.biz/D600938 --- build.sbt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 121c2e29a4..96a82c5f89 100644 --- a/build.sbt +++ b/build.sbt @@ -8,7 +8,7 @@ val jacksonVersion = "2.11.2" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" -val zkVersion = "3.5.0-alpha" +val zkVersion = "3.5.6" val zkClientVersion = "0.0.81" val zkGroupVersion = "0.0.92" val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll ( @@ -491,16 +491,16 @@ lazy val utilSecurity = Project( scalacheckLib, "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ) -).dependsOn(utilCore, utilLogging, utilSecurityTestCerts % "test") + ).dependsOn(utilCore, utilLogging, utilSecurityTestCerts % "test") lazy val utilSecurityTestCerts = Project( id = "util-security-test-certs", base = file("util-security-test-certs") ).settings( - sharedSettings -).settings( - name := "util-security-test-certs" -) + sharedSettings + ).settings( + name := "util-security-test-certs" + ) lazy val utilStats = Project( id = "util-stats", From 5effa2a622700995a7c099687865429d947970c8 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 13 Jan 2021 21:45:46 +0000 Subject: [PATCH 420/796] Twitter-oss: Prepare OSS libraries for release 21.1.0 Problem We want to release the next versions of our Twitter OSS libraries 21.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D602794 --- CHANGELOG.rst | 5 +++++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 10f83e1aba..4a5b03f6ca 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.1.0 +------ + +No Changes + 20.12.0 ------- diff --git a/README.md b/README.md index c8e8b86363..0e924ceac4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "20.12.0" +val utilCore = "com.twitter" %% "util-core" % "21.1.0" ``` # Units diff --git a/build.sbt b/build.sbt index 96a82c5f89..b535db1bbd 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.1.0-SNAPSHOT" +val releaseVersion = "21.1.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From cc4b6a83edfff155edcea1ff41af6dd5ace6bf0c Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Thu, 14 Jan 2021 16:21:53 +0000 Subject: [PATCH 421/796] [util] Remove util/util-thrift/src/test/java/BUILD Problem / Solution It's unused. Remove it Differential Revision: https://phabricator.twitter.biz/D603165 --- util-thrift/src/test/java/BUILD | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 util-thrift/src/test/java/BUILD diff --git a/util-thrift/src/test/java/BUILD b/util-thrift/src/test/java/BUILD deleted file mode 100644 index 60c095c87f..0000000000 --- a/util-thrift/src/test/java/BUILD +++ /dev/null @@ -1,5 +0,0 @@ -target( - dependencies = [ - "util/util-thrift/src/test/java/com/twitter/util", - ], -) From e974d04e771b7ca790e1f9edd2a71b406b3c819e Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 14 Jan 2021 20:51:54 +0000 Subject: [PATCH 422/796] util-core: fix typo in Scaladoc example for Reader trait Problem: There is an extra close parenthesis ) in the example code for Reader Solution: Remove the typo JIRA Issues: CSL-10547 Differential Revision: https://phabricator.twitter.biz/D602854 --- util-core/src/main/scala/com/twitter/io/Reader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index d2c0681750..4b308ad20b 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -12,7 +12,7 @@ import java.io.{File, FileInputStream, InputStream} * typical code pattern to consume a reader is to use a read-loop: * * {{{ - * def consume[A](r: Reader[A]))(process: A => Future[Unit]): Future[Unit] = + * def consume[A](r: Reader[A])(process: A => Future[Unit]): Future[Unit] = * r.read().flatMap { * case Some(a) => process(a).before(consume(r)(process)) * case None => Future.Done // reached the end of the stream; no need to discard From e185c659de9474704e41f58c065dfed3e97d8835 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 14 Jan 2021 23:34:19 +0000 Subject: [PATCH 423/796] Twitter-oss: update OSS libraries post-release to 20.2.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 20.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D603517 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0e924ceac4..551eb18094 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b535db1bbd..e0dbf042b2 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.1.0" +val releaseVersion = "21.2.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 411cef442a0641b77d1ddbbbe7668536cc38984e Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 23 Jan 2021 02:47:31 +0000 Subject: [PATCH 424/796] oss: Update README copyright year to 2021 Problem We've crossed into a new year and our copyright ranges are now out of date. Solution Update them to 2021. JIRA Issues: CSL-10577 Differential Revision: https://phabricator.twitter.biz/D606603 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 551eb18094..354b71835a 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,6 @@ val duration: Duration = elapsed() ## License -Copyright 2010-2020 Twitter, Inc. +Copyright 2010-2021 Twitter, Inc. Licensed under the Apache License, Version 2.0: https://www.apache.org/licenses/LICENSE-2.0 From a5fe3497fd5c10a29703c9add3424d75f0b3a208 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Mon, 25 Jan 2021 05:58:59 +0000 Subject: [PATCH 425/796] [bazel prep] Convert `target` to `tests_suite` for those that point to tests Problem / Solution Because bazel cannot test an alias (https://docs.bazel.build/versions/2.2.0/be/general.html#alias), we need to correctly and explicitly identify for bazel which `target` target of Pants points to a set of tests, and they should be using the native bazel `test_suite` concept. JIRA Issues: DPB-13054 Differential Revision: https://phabricator.twitter.biz/D603931 --- util-app/BUILD | 2 +- util-cache-guava/src/test/java/BUILD | 2 +- util-cache-guava/src/test/scala/BUILD | 2 +- util-cache/BUILD | 2 +- util-cache/src/test/scala/BUILD | 2 +- util-codec/BUILD | 2 +- util-codec/src/test/scala/BUILD | 2 +- util-core/BUILD | 2 +- util-core/src/test/java/BUILD | 2 +- util-core/src/test/scala/BUILD | 2 +- util-hashing/BUILD | 2 +- util-hashing/src/test/scala/BUILD | 2 +- util-jvm/BUILD | 2 +- util-lint/BUILD | 2 +- util-lint/src/test/scala/BUILD | 2 +- util-logging/BUILD | 2 +- util-reflect/BUILD | 2 +- util-reflect/src/test/scala/BUILD | 2 +- util-registry/BUILD | 2 +- util-security/src/test/java/BUILD | 2 +- util-security/src/test/scala/BUILD | 2 +- util-slf4j-api/BUILD | 2 +- util-slf4j-api/src/test/java/BUILD | 2 +- util-slf4j-api/src/test/scala/BUILD | 2 +- util-stats/BUILD | 2 +- util-test/BUILD | 2 +- util-test/src/test/scala/BUILD | 2 +- util-thrift/BUILD | 2 +- util-thrift/src/test/scala/BUILD | 2 +- util-tunable/src/test/scala/BUILD | 2 +- util-zk-test/BUILD | 2 +- util-zk/BUILD | 2 +- util-zk/src/test/scala/BUILD | 2 +- 33 files changed, 33 insertions(+), 33 deletions(-) diff --git a/util-app/BUILD b/util-app/BUILD index e26545d608..6fd9f53aa0 100644 --- a/util-app/BUILD +++ b/util-app/BUILD @@ -5,7 +5,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-app/src/test/java", diff --git a/util-cache-guava/src/test/java/BUILD b/util-cache-guava/src/test/java/BUILD index 76b5264c8b..e4a98fbfe1 100644 --- a/util-cache-guava/src/test/java/BUILD +++ b/util-cache-guava/src/test/java/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-cache-guava/src/test/java/com/twitter/cache/guava", ], diff --git a/util-cache-guava/src/test/scala/BUILD b/util-cache-guava/src/test/scala/BUILD index 2f54421711..247847df14 100644 --- a/util-cache-guava/src/test/scala/BUILD +++ b/util-cache-guava/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-cache-guava/src/test/scala/com/twitter/cache/guava", ], diff --git a/util-cache/BUILD b/util-cache/BUILD index ce5424c315..924027a1a8 100644 --- a/util-cache/BUILD +++ b/util-cache/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-cache/src/test/java", diff --git a/util-cache/src/test/scala/BUILD b/util-cache/src/test/scala/BUILD index dee129b504..b368a969e4 100644 --- a/util-cache/src/test/scala/BUILD +++ b/util-cache/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-cache/src/test/scala/com/twitter/cache", "util/util-cache/src/test/scala/com/twitter/cache/caffeine", diff --git a/util-codec/BUILD b/util-codec/BUILD index 9a12f6e0e4..fe7f171d97 100644 --- a/util-codec/BUILD +++ b/util-codec/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-codec/src/test/scala", diff --git a/util-codec/src/test/scala/BUILD b/util-codec/src/test/scala/BUILD index b4c871f06b..9d2bcbe66e 100644 --- a/util-codec/src/test/scala/BUILD +++ b/util-codec/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-codec/src/test/scala/com/twitter/util", ], diff --git a/util-core/BUILD b/util-core/BUILD index 460d03d2e0..4cbea79d7c 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -5,7 +5,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-core/src/test/java", diff --git a/util-core/src/test/java/BUILD b/util-core/src/test/java/BUILD index c99528cb20..cf8f19a631 100644 --- a/util-core/src/test/java/BUILD +++ b/util-core/src/test/java/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-core/src/test/java/com/twitter/concurrent", "util/util-core/src/test/java/com/twitter/io", diff --git a/util-core/src/test/scala/BUILD b/util-core/src/test/scala/BUILD index 4ca1cf0944..1919c2211a 100644 --- a/util-core/src/test/scala/BUILD +++ b/util-core/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-core/src/test/scala/com/twitter/concurrent", "util/util-core/src/test/scala/com/twitter/conversions", diff --git a/util-hashing/BUILD b/util-hashing/BUILD index cc89f2ca64..afce834f44 100644 --- a/util-hashing/BUILD +++ b/util-hashing/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-hashing/src/test/scala", diff --git a/util-hashing/src/test/scala/BUILD b/util-hashing/src/test/scala/BUILD index 50c8534f02..c62ca977cc 100644 --- a/util-hashing/src/test/scala/BUILD +++ b/util-hashing/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-hashing/src/test/scala/com/twitter/hashing", ], diff --git a/util-jvm/BUILD b/util-jvm/BUILD index 5b41ae4712..43bac99308 100644 --- a/util-jvm/BUILD +++ b/util-jvm/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-jvm/src/test/java", diff --git a/util-lint/BUILD b/util-lint/BUILD index 0caeec0478..109f5946d3 100644 --- a/util-lint/BUILD +++ b/util-lint/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-lint/src/test/scala", diff --git a/util-lint/src/test/scala/BUILD b/util-lint/src/test/scala/BUILD index 22b1a0076a..f683daf038 100644 --- a/util-lint/src/test/scala/BUILD +++ b/util-lint/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-lint/src/test/scala/com/twitter/util/lint", ], diff --git a/util-logging/BUILD b/util-logging/BUILD index 6b4c5c6451..08133e0c3a 100644 --- a/util-logging/BUILD +++ b/util-logging/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-logging/src/test/java", diff --git a/util-reflect/BUILD b/util-reflect/BUILD index 6aa8fcd0cf..27d6eb2eb0 100644 --- a/util-reflect/BUILD +++ b/util-reflect/BUILD @@ -8,7 +8,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-reflect/src/test/scala", diff --git a/util-reflect/src/test/scala/BUILD b/util-reflect/src/test/scala/BUILD index e5ef76f364..f0543a5893 100644 --- a/util-reflect/src/test/scala/BUILD +++ b/util-reflect/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-reflect/src/test/scala/com/twitter/util/reflect", ], diff --git a/util-registry/BUILD b/util-registry/BUILD index 0901ca475e..c1361d7722 100644 --- a/util-registry/BUILD +++ b/util-registry/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-registry/src/test/java", diff --git a/util-security/src/test/java/BUILD b/util-security/src/test/java/BUILD index f2370e4dd8..61076e5d29 100644 --- a/util-security/src/test/java/BUILD +++ b/util-security/src/test/java/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-security/src/test/java/com/twitter/util/security", ], diff --git a/util-security/src/test/scala/BUILD b/util-security/src/test/scala/BUILD index 224798f5bf..3293572245 100644 --- a/util-security/src/test/scala/BUILD +++ b/util-security/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-security/src/test/scala/com/twitter/util/security", ], diff --git a/util-slf4j-api/BUILD b/util-slf4j-api/BUILD index b123da30a3..22da9a28ad 100644 --- a/util-slf4j-api/BUILD +++ b/util-slf4j-api/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-slf4j-api/src/test/scala", diff --git a/util-slf4j-api/src/test/java/BUILD b/util-slf4j-api/src/test/java/BUILD index fcd4c2c567..8ecefa1c4b 100644 --- a/util-slf4j-api/src/test/java/BUILD +++ b/util-slf4j-api/src/test/java/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-slf4j-api/src/test/java/com/twitter/util/logging", ], diff --git a/util-slf4j-api/src/test/scala/BUILD b/util-slf4j-api/src/test/scala/BUILD index 34bc6295e6..c9645b170e 100644 --- a/util-slf4j-api/src/test/scala/BUILD +++ b/util-slf4j-api/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-slf4j-api/src/test/scala/com/twitter/util/logging", ], diff --git a/util-stats/BUILD b/util-stats/BUILD index 0de6ae6acf..a266afa24b 100644 --- a/util-stats/BUILD +++ b/util-stats/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-stats/src/test/java", diff --git a/util-test/BUILD b/util-test/BUILD index edd568c335..90c6c818d6 100644 --- a/util-test/BUILD +++ b/util-test/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-test/src/test/scala", diff --git a/util-test/src/test/scala/BUILD b/util-test/src/test/scala/BUILD index ede2bb7120..2cfefa131f 100644 --- a/util-test/src/test/scala/BUILD +++ b/util-test/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-test/src/test/scala/com/twitter/util/testing", ], diff --git a/util-thrift/BUILD b/util-thrift/BUILD index ce55cabe80..3943e49d22 100644 --- a/util-thrift/BUILD +++ b/util-thrift/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-thrift/src/test/scala", diff --git a/util-thrift/src/test/scala/BUILD b/util-thrift/src/test/scala/BUILD index 92308690a9..29ff5c188e 100644 --- a/util-thrift/src/test/scala/BUILD +++ b/util-thrift/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-thrift/src/test/scala/com/twitter/util", ], diff --git a/util-tunable/src/test/scala/BUILD b/util-tunable/src/test/scala/BUILD index 5a6a16dd85..c1b6083c5c 100644 --- a/util-tunable/src/test/scala/BUILD +++ b/util-tunable/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-tunable/src/test/scala/com/twitter/util/tunable", ], diff --git a/util-zk-test/BUILD b/util-zk-test/BUILD index 88fa6a0060..72454b3faf 100644 --- a/util-zk-test/BUILD +++ b/util-zk-test/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-zk-test/src/test/java", diff --git a/util-zk/BUILD b/util-zk/BUILD index e9907b6656..e3ed022b4b 100644 --- a/util-zk/BUILD +++ b/util-zk/BUILD @@ -4,7 +4,7 @@ target( ], ) -target( +test_suite( name = "tests", dependencies = [ "util/util-zk/src/test/scala", diff --git a/util-zk/src/test/scala/BUILD b/util-zk/src/test/scala/BUILD index de068cd977..eb0e0371a3 100644 --- a/util-zk/src/test/scala/BUILD +++ b/util-zk/src/test/scala/BUILD @@ -1,4 +1,4 @@ -target( +test_suite( dependencies = [ "util/util-zk/src/test/scala/com/twitter/zk", "util/util-zk/src/test/scala/com/twitter/zk/coordination", From 08ad99b50c3abf72e4fd6ddde46ffaa24a633a51 Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Fri, 29 Jan 2021 03:37:15 +0000 Subject: [PATCH 426/796] [util-security] throw MatchError when case match is non-exhaustive Problem Supporting latest https://github.com/scala/scala/releases/tag/v2.12.13 includes a feature change in the scala compiler that will fail to compile when case matching is not exhaustive on tuples. Issues - https://github.com/scala/bug/issues/10680 - https://github.com/scala/bug/issues/10373 - https://github.com/scala/bug/issues/10019 Resolved - https://github.com/scala/scala/pull/9163 - https://github.com/scala/scala/pull/9147 Solution Add `case _ => throw new MatchError` to case matching statement that is non-exhaustive JIRA Issues: SCALA-25 Differential Revision: https://phabricator.twitter.biz/D609046 --- .../main/scala/com/twitter/util/security/Credentials.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala index 0623a238e5..5525033717 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala @@ -37,7 +37,10 @@ object Credentials { private[this] val key = "[\\w-]+".r private[this] val value = ".+".r - def auth: Parser[Tuple2[String, String]] = key ~ ":" ~ value ^^ { case k ~ ":" ~ v => (k, v) } + def auth: Parser[Tuple2[String, String]] = key ~ ":" ~ value ^^ { + case k ~ ":" ~ v => (k, v) + case _ => throw new MatchError + } def content: Parser[Map[String, String]] = rep(auth) ^^ { auths => Map(auths: _*) } def apply(in: String): Map[String, String] = { From 47512f21eeb350a52da721409f6cd85941a4f2f9 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Fri, 29 Jan 2021 20:32:36 +0000 Subject: [PATCH 427/796] util-routing: evolve routing result beyond Option[Route] and simplify Router generation Problem A `c.t.util.routing.Router` represents the result of finding a `Route` as a `Option[Route]`. As we implement new use cases around the `Router`, we have found a need to pass along the routing `Input` as part of the result. Solution Introduce a new `c.t.util.routing.Result` type for representing a `Router` result, which can be represented by different states: `Found` and `NotFound`. As a side-effect, we have the ability to return a result that represents a router being closed instead of being forced to throw an exception for states that do not match Some or None. This also simplifies our Java-friendly story and allows us to remove `AbstractRouter`. It was also found that the `Generator` should use an encapsulating type for its input instead of a Tuple, which has also been reflected in these changes. Result The result of routing via a `Router` can pass on more information than was previously possible. Generating a `Router` is simplified. JIRA Issues: CSL-10591 Differential Revision: https://phabricator.twitter.biz/D608227 --- .../twitter/util/routing/AbstractRouter.scala | 37 --------- .../com/twitter/util/routing/Generator.scala | 18 +---- .../com/twitter/util/routing/Result.scala | 35 +++++++++ .../com/twitter/util/routing/Router.scala | 62 +++++++-------- .../twitter/util/routing/RouterBuilder.scala | 2 +- .../com/twitter/util/routing/RouterInfo.scala | 14 ++++ .../com/twitter/util/routing/Validator.scala | 4 +- .../util/routing/RouterCompilationTest.java | 46 +++++++---- .../PolymorphicRouterCompilationTest.scala | 69 +++++++++-------- .../util/routing/RouterBuilderTest.scala | 14 ++-- .../com/twitter/util/routing/RouterTest.scala | 77 +++++++++---------- .../util/routing/dynamic/DynamicRoute.scala | 4 +- .../util/routing/dynamic/DynamicRouter.scala | 30 +++++--- .../util/routing/dynamic/Response.scala | 8 ++ .../twitter/util/routing/dynamic/Result.scala | 8 -- .../util/routing/simple/SimpleRouter.scala | 13 ++-- 16 files changed, 229 insertions(+), 212 deletions(-) delete mode 100644 util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/Result.scala create mode 100644 util-routing/src/main/scala/com/twitter/util/routing/RouterInfo.scala create mode 100644 util-routing/src/test/scala/com/twitter/util/routing/dynamic/Response.scala delete mode 100644 util-routing/src/test/scala/com/twitter/util/routing/dynamic/Result.scala diff --git a/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala b/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala deleted file mode 100644 index 49c64282df..0000000000 --- a/util-routing/src/main/scala/com/twitter/util/routing/AbstractRouter.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.util.routing - -import com.twitter.util.Try -import java.lang.{Iterable => JIterable} -import java.util.Optional -import scala.jdk.CollectionConverters._ - -/** - * Java-friendly version of [[Router]] - * - * @param label A label used for identifying this Router (i.e. for distinguishing between [[Router]] - * instances in error messages or for StatsReceiver scope). - * @param jRoutes All of the `Route` routes contained within this [[Router]], represented as - * a Java `Iterable`. - * @tparam Input The Input type used to determine a route destination - * @tparam Route The output/resulting route type - */ -abstract class AbstractRouter[Input, Route](val label: String, jRoutes: JIterable[Route]) - extends Router[Input, Route] { - - /** Java-friendly version of [[apply]] */ - final def route(input: Input): Optional[Route] = apply(input) match { - case Some(route) => Optional.of(route) - case _ => Optional.empty() - } - - /** Java-friendly version of [[find]] */ - protected def findAny(input: Input): Optional[Route] - - override final protected def find(input: Input): Option[Route] = { - val found = findAny(input) - Try(found.get()).toOption - } - - override final val routes: Iterable[Route] = jRoutes.asScala - -} diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala b/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala index 774bc60c70..8383c0c05c 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/Generator.scala @@ -1,24 +1,8 @@ package com.twitter.util.routing -import java.lang.{Iterable => JIterable} -import java.util.function.BiFunction -import scala.jdk.CollectionConverters._ - -object Generator { - - /** Java-friendly creation of a [[Generator]] */ - def create[Input, Route, RouterType <: Router[Input, Route]]( - fn: BiFunction[String, JIterable[Route], RouterType] - ): Generator[Input, Route, RouterType] = new Generator[Input, Route, RouterType] { - override def apply(label: String, routes: Iterable[Route]): RouterType = - fn(label, routes.asJava) - } - -} - /** * Functional alias for creating a [[Router router]] given a label and * all of the defined routes for the router to be generated. */ abstract class Generator[Input, Route, +RouterType <: Router[Input, Route]] - extends ((String, Iterable[Route]) => RouterType) + extends (RouterInfo[Route] => RouterType) diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Result.scala b/util-routing/src/main/scala/com/twitter/util/routing/Result.scala new file mode 100644 index 0000000000..c0c9521a2d --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/Result.scala @@ -0,0 +1,35 @@ +package com.twitter.util.routing + +/** A result from attempting to find a route via a [[Router]]. */ +sealed abstract class Result + +/** + * A result that represents that the [[Router router]] could not determine a defined route + * for a given input. + */ +case object NotFound extends Result { + + /** Java-friendly Singleton accessor */ + def get(): Result = this +} + +/** + * A result that represents that the [[Router router]] could determine a defined [[Route route]] + * for a given [[Input input]]. + * + * @param input The [[Input input]] when attempting to find a route. + * @param route The [[Route route]] that was found. + * @tparam Input The input type for the [[Router]]. + * @tparam Route The route type for the [[Router]]. + */ +final case class Found[Input, Route](input: Input, route: Route) extends Result + +/** + * A result that represents that the [[Router router]] has been closed and will no longer attempt + * to find routes for any inputs. + */ +case object RouterClosed extends Result { + + /** Java-friendly Singleton accessor */ + def get(): Result = this +} diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Router.scala b/util-routing/src/main/scala/com/twitter/util/routing/Router.scala index 24346dd272..3244174e43 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/Router.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/Router.scala @@ -2,11 +2,22 @@ package com.twitter.util.routing import com.twitter.util.logging.Logging import com.twitter.util.{Closable, ClosableOnce, Future, Time} +import java.lang.{Iterable => JIterable} +import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal /** * A generic interface for routing an input to an optional matching route. * + * @param label A label used for identifying this Router (i.e. for distinguishing between [[Router]] + * instances in error messages or for StatsReceiver scope). + * @param routes All of the `Route` routes contained within this [[Router]] + * + * This property is used to linearly determine the routes when `close()` is called. + * It may also be used as a way to determine what routes are defined within a [[Router]] + * (i.e. for generating meaningful error messages). This property does not imply + * any relationship with [[apply]] or [[find]] or that a [[Router]]'s runtime behavior + * needs to be linear. * @tparam Input The Input type used when determining a route destination * @tparam Route The resulting route type. A `Route` may have dynamic * properties and may be found to satisfy multiple `Input` @@ -22,43 +33,34 @@ import scala.util.control.NonFatal * should be aware of the method, path, and and the destination/logic responsible * for handling an `Input` that matches. * - * Another property to consider for a `Route` is "uniqueness". It may or may not - * be desirable for a [[Router]] to contain multiple routes that correspond to - * the same `Input`. + * Another property to consider for a `Route` is "uniqueness". A [[Router]] should be + * able to determine either: + * a) a single `Route` for an `Input` + * b) no `Route` for an `Input` + * A [[Router]] should NOT be expected to return multiple routes for an `Input`. * * @note A [[Router]] should be considered immutable unless explicitly noted. */ -trait Router[Input, +Route] extends (Input => Option[Route]) with ClosableOnce with Logging { - - /** - * A label used for identifying this Router (i.e. for distinguishing between [[Router]] instances - * in error messages or for StatsReceiver scope). - */ - def label: String +abstract class Router[Input, +Route](val label: String, val routes: Iterable[Route]) + extends (Input => Result) + with ClosableOnce + with Logging { - /** - * All of the `Route` routes contained within this [[Router]] - * - * @note This property is used to linearly determine the routes when `close()` is called. - * It may also be used as a way to determine what routes are defined within a [[Router]] - * (i.e. for generating meaningful error messages). This property does not imply - * any relationship with [[apply]] or [[find]] or that a [[Router]]'s runtime behavior - * needs to be linear. - */ - def routes: Iterable[Route] + /** Java-friendly constructor */ + def this(label: String, routes: JIterable[Route]) = this(label, routes.asScala) /** * Attempt to route an `Input` to a `Route` route known by this Router. * * @param input The `Input` to use for determining an available route * - * @return `Some(Route)` if a route is found to match, `None` otherwise - * - * @note A [[ClosedRouterException]] will be thrown if `close()` - * has been initiated on this [[Router]] and subsequent routing attempts are received. + * @return [[Found `Found(input: Input, route: Route)`]] if a route is found to match, + * [[NotFound]] if there is no match, or + * [[RouterClosed]] if `close()` has been initiated on this [[Router]] and subsequent + * routing attempts are received. */ - final def apply(input: Input): Option[Route] = - if (isClosed) throw ClosedRouterException(this) + final def apply(input: Input): Result = + if (isClosed) RouterClosed else find(input) /** @@ -67,19 +69,19 @@ trait Router[Input, +Route] extends (Input => Option[Route]) with ClosableOnce w * * @param input The `Input` to determine a route for. * - * @return `Some(Route)` if a matching route is defined, `None` if a route destination is - * not defined for the `Input`. + * @return [[Found `Found(input: Input, route: Route)`]] if a matching route is defined, + * [[NotFound]] if a route destination is not defined for the `Input`. * * @note This method is only meant to be called within the [[Router]]'s `apply`, which * handles lifecycle concerns. It should not be accessed directly. */ - protected def find(input: Input): Option[Route] + protected def find(input: Input): Result /** * @note NonFatal exceptions encountered when calling `close()` on a `Route` will be * suppressed and Fatal exceptions will take on the same exception behavior of a `Future.join`. */ - override protected def closeOnce(deadline: Time): Future[Unit] = + protected def closeOnce(deadline: Time): Future[Unit] = Future.join(routes.map(closeIfClosable(_, deadline)).toSeq) private[this] def closeIfClosable(route: Route, deadline: Time): Future[Unit] = route match { diff --git a/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala b/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala index 7249657ba2..8056c37290 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/RouterBuilder.scala @@ -53,7 +53,7 @@ case class RouterBuilder[Input, Route, +RouterType <: Router[Input, Route]] priv val failures = validator(routes) if (failures.nonEmpty) throw ValidationException(failures) - generator(label, routes) + generator(RouterInfo(label, routes)) } } diff --git a/util-routing/src/main/scala/com/twitter/util/routing/RouterInfo.scala b/util-routing/src/main/scala/com/twitter/util/routing/RouterInfo.scala new file mode 100644 index 0000000000..bc89050fe3 --- /dev/null +++ b/util-routing/src/main/scala/com/twitter/util/routing/RouterInfo.scala @@ -0,0 +1,14 @@ +package com.twitter.util.routing + +import java.lang.{Iterable => JIterable} +import scala.jdk.CollectionConverters._ + +/** The information needed to generate a [[Router]] via a [[Generator]]. */ +case class RouterInfo[+Route](label: String, routes: Iterable[Route]) { + + /** Java-friendly constructor. */ + def this(label: String, jRoutes: JIterable[Route]) = this(label, jRoutes.asScala) + + /** Java-friendly accessor to the `Iterable[Route]` routes. */ + def jRoutes(): JIterable[_ <: Route] = routes.asJava +} diff --git a/util-routing/src/main/scala/com/twitter/util/routing/Validator.scala b/util-routing/src/main/scala/com/twitter/util/routing/Validator.scala index acb6f60173..bf39407f71 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/Validator.scala +++ b/util-routing/src/main/scala/com/twitter/util/routing/Validator.scala @@ -8,13 +8,13 @@ object Validator { /** [[Validator]] that does no validation and always returns empty error results */ val None: Validator[Any] = new Validator[Any] { - override def apply(routes: Iterable[Any]): Iterable[ValidationError] = Iterable.empty + def apply(routes: Iterable[Any]): Iterable[ValidationError] = Iterable.empty } /** Java-friendly creation of a [[Validator]] */ def create[T](fn: Function[JIterable[T], JIterable[ValidationError]]): Validator[T] = new Validator[T] { - override def apply(routes: Iterable[T]): Iterable[ValidationError] = + def apply(routes: Iterable[T]): Iterable[ValidationError] = fn(routes.asJava).asScala } diff --git a/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java b/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java index c079172a51..7488298dd9 100644 --- a/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java +++ b/util-routing/src/test/java/com/twitter/util/routing/RouterCompilationTest.java @@ -5,7 +5,6 @@ import java.util.Map; import java.util.HashMap; -import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -21,13 +20,25 @@ private static final class StringRoute { } } - private static final class StringRouter extends AbstractRouter { + private static final class StringRouterGenerator extends Generator { + private static final StringRouterGenerator instance = new StringRouterGenerator(); + static final StringRouterGenerator getInstance() { + return instance; + } + + private StringRouterGenerator() {} + + @Override + public StringRouter apply(RouterInfo routerInfo) { + Map routeMap = new HashMap<>(); + routerInfo.jRoutes().forEach(r -> routeMap.put(r.input, r)); + return new StringRouter(routeMap); + } + } + + private static final class StringRouter extends Router { static RouterBuilder newBuilder() { - return RouterBuilder.newBuilder(Generator.create((label, routes) -> { - Map routeMap = new HashMap<>(); - routes.forEach(r -> routeMap.put(r.input, r)); - return new StringRouter(routeMap); - })); + return RouterBuilder.newBuilder(StringRouterGenerator.getInstance()); } private final Map routeMap; @@ -38,9 +49,10 @@ public StringRouter(Map routeMap) { } @Override - public Optional findAny(String input) { + public Result find(String input) { StringRoute found = routeMap.get(input); - return Optional.ofNullable(found); + if (found == null) return NotFound.get(); + return Found.apply(input, found); } } @@ -64,10 +76,10 @@ public void testRouter() { routes.put(hello.input, hello); routes.put(goodbye.input, goodbye); - AbstractRouter router = new StringRouter(routes); - Assert.assertEquals(router.route("hello"), Optional.of(hello)); - Assert.assertEquals(router.route("goodbye"), Optional.of(goodbye)); - Assert.assertEquals(router.route("oh-no"), Optional.empty()); + Router router = new StringRouter(routes); + Assert.assertEquals(router.apply("hello"), Found.apply("hello", hello)); + Assert.assertEquals(router.apply("goodbye"), Found.apply("goodbye", goodbye)); + Assert.assertEquals(router.apply("oh-no"), NotFound.get()); } @Test @@ -75,14 +87,14 @@ public void testRouterBuilder() { StringRoute hello = new StringRoute("hello", "abc"); StringRoute goodbye = new StringRoute("goodbye", "123"); - AbstractRouter router = StringRouter.newBuilder() + Router router = StringRouter.newBuilder() .withRoute(hello) .withRoute(goodbye) .newRouter(); - Assert.assertEquals(router.route("hello"), Optional.of(hello)); - Assert.assertEquals(router.route("goodbye"), Optional.of(goodbye)); - Assert.assertEquals(router.route("oh-no"), Optional.empty()); + Assert.assertEquals(router.apply("hello"), Found.apply("hello", hello)); + Assert.assertEquals(router.apply("goodbye"), Found.apply("goodbye", goodbye)); + Assert.assertEquals(router.apply("oh-no"), NotFound.get()); } @Test(expected = ValidationException.class) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala index 96e2db4214..5ffe6f8adc 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/PolymorphicRouterCompilationTest.scala @@ -21,14 +21,19 @@ private object PolymorphicRouterCompilationTest { def newBuilder: RouterBuilder[String, StatusHandler, StatusRouter] = RouterBuilder.newBuilder[String, StatusHandler, StatusRouter]( new Generator[String, StatusHandler, StatusRouter] { - override def apply(label: String, routes: Iterable[StatusHandler]): StatusRouter = - StatusRouter(label, routes) + def apply(labelAndRoutes: RouterInfo[StatusHandler]): StatusRouter = + new StatusRouter(labelAndRoutes.label, labelAndRoutes.routes) }) } - case class StatusRouter(label: String, routes: Iterable[StatusHandler]) - extends Router[String, StatusHandler] { - override protected def find(input: String): Option[StatusHandler] = routes.find(_.str == input) + class StatusRouter(label: String, routes: Iterable[StatusHandler]) + extends Router[String, StatusHandler](label, routes) { + protected def find(input: String): Result = { + routes.find(_.str == input) match { + case Some(route) => Found(input, route) + case _ => NotFound + } + } } val statusRouter: StatusRouter = StatusRouter.newBuilder @@ -38,11 +43,10 @@ private object PolymorphicRouterCompilationTest { def checkStatusRouterType: Router[String, StatusHandler] = statusRouter - val covariantStatusRouter: Router[String, StatusHandler] = new Router[String, StatusHandlerA] { - override def label: String = "covariant-status" - override def routes: Iterable[StatusHandlerA] = Iterable.empty - override protected def find(input: String): Option[StatusHandlerA] = None - } + val covariantStatusRouter: Router[String, StatusHandler] = + new Router[String, StatusHandlerA]("covariant-status", Iterable.empty) { + protected def find(input: String): Result = NotFound + } /* polymorphic input type router */ @@ -56,15 +60,18 @@ private object PolymorphicRouterCompilationTest { object CommandRouter { def newBuilder: RouterBuilder[Command, CommandHandler, CommandRouter] = RouterBuilder.newBuilder(new Generator[Command, CommandHandler, CommandRouter] { - override def apply(label: String, routes: Iterable[CommandHandler]): CommandRouter = - CommandRouter(label, routes) + def apply(labelAndRoutes: RouterInfo[CommandHandler]): CommandRouter = + new CommandRouter(labelAndRoutes.label, labelAndRoutes.routes) }) } - case class CommandRouter(label: String, routes: Iterable[CommandHandler]) - extends Router[Command, CommandHandler] { - override protected def find(input: Command): Option[CommandHandler] = - routes.find(_.canHandle(input)) + class CommandRouter(label: String, routes: Iterable[CommandHandler]) + extends Router[Command, CommandHandler](label, routes) { + protected def find(input: Command): Result = + routes.find(_.canHandle(input)) match { + case Some(r) => Found(input, r) + case _ => NotFound + } } val CommandHandlerA: CommandHandler = CommandHandler(_.isInstanceOf[IntCommand], Success) @@ -84,24 +91,22 @@ class PolymorphicRouterCompilationTest extends AnyFunSuite { import PolymorphicRouterCompilationTest._ test("Supports routing with polymorphic destinations") { - assert(statusRouter("success") == Some(StatusHandlerA("success", Success))) - assert(statusRouter("fail") == Some(StatusHandlerB("fail", Failure))) - assert(statusRouter("meh") == None) + assert(statusRouter("success") == Found("success", StatusHandlerA("success", Success))) + assert(statusRouter("fail") == Found("fail", StatusHandlerB("fail", Failure))) + assert(statusRouter("meh") == NotFound) } test("Supports routing with polymorphic inputs") { - assert(commandRouter(IntCommand(1)) == Some(CommandHandlerA)) - assert(commandRouter(StringCommand("1")) == Some(CommandHandlerB)) - assert(commandRouter(NullCommand) == None) + assert(commandRouter(IntCommand(1)) == Found(IntCommand(1), CommandHandlerA)) + assert(commandRouter(StringCommand("1")) == Found(StringCommand("1"), CommandHandlerB)) + assert(commandRouter(NullCommand) == NotFound) } test("Input type cannot be covariant") { assertDoesNotCompile { """ - | val contravariantInputRouter: Router[Command, CommandHandler] = new Router[IntCommand, CommandHandler] { - | override def label: String = "int-command-router" - | override def routes: Iterable[CommandHandler] = Iterable.empty - | override protected def find(input: IntCommand): Option[CommandHandler] = None + | val contravariantInputRouter: Router[Command, CommandHandler] = new Router[IntCommand, CommandHandler]("int-command-router", Iterable.empty) { + | protected def find(input: IntCommand): Result = NotFound | } |""".stripMargin } @@ -110,10 +115,8 @@ class PolymorphicRouterCompilationTest extends AnyFunSuite { test("Input type cannot be contravariant") { assertDoesNotCompile { """ - | val contravariantInputRouter: Router[IntCommand, CommandHandler] = new Router[Command, CommandHandler] { - | override def label: String = "int-command-router" - | override def routes: Iterable[CommandHandler] = Iterable.empty - | override protected def find(input: Command): Option[CommandHandler] = None + | val contravariantInputRouter: Router[IntCommand, CommandHandler] = new Router[Command, CommandHandler]("int-command-router", Iterable.empty) { + | protected def find(input: Command): Result = NotFound | } |""".stripMargin } @@ -122,10 +125,8 @@ class PolymorphicRouterCompilationTest extends AnyFunSuite { test("Destination type cannot be contravariant") { assertDoesNotCompile { """ - | val contravariantStatusRouter: Router[String, StatusHandlerA] = new Router[String, StatusHandler] { - | override def label: String = "contravariant-status" - | override def routes: Iterable[StatusHandler] = Iterable.empty - | override protected def find(input: String): Option[StatusHandler] = None + | val contravariantStatusRouter: Router[String, StatusHandlerA] = new Router[String, StatusHandler]("contravariant-status", Iterable.empty[StatusHandler]) { + | protected def find(input: String): Result = NotFound | } |""".stripMargin } diff --git a/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala index d975ef2ea3..ba5d07d10b 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala @@ -8,8 +8,8 @@ class RouterBuilderTest extends AnyFunSuite { private object TestRouter { def newBuilder(): RouterBuilder[String, SimpleRoute, SimpleRouter] = RouterBuilder.newBuilder(new Generator[String, SimpleRoute, SimpleRouter] { - override def apply(label: String, routes: Iterable[SimpleRoute]): SimpleRouter = - new SimpleRouter(routes.map { r => r.in -> r }.toMap) + def apply(labelAndRoutes: RouterInfo[SimpleRoute]): SimpleRouter = + new SimpleRouter(labelAndRoutes.routes.map { r => r.in -> r }.toMap) }) } @@ -17,7 +17,7 @@ class RouterBuilderTest extends AnyFunSuite { def newBuilder(): RouterBuilder[String, SimpleRoute, SimpleRouter] = TestRouter.newBuilder.withValidator( new Validator[SimpleRoute] { - override def apply(routes: Iterable[SimpleRoute]): Iterable[ValidationError] = + def apply(routes: Iterable[SimpleRoute]): Iterable[ValidationError] = routes.collect { case r if r.in == "invalid" => ValidationError(s"INVALID @ $r") } @@ -33,10 +33,10 @@ class RouterBuilderTest extends AnyFunSuite { .withRoute(SimpleRoute("z", true)) .newRouter() - assert(router("x") == Some(SimpleRoute("x", true))) - assert(router("y") == Some(SimpleRoute("y", false))) - assert(router("z") == Some(SimpleRoute("z", true))) - assert(router("a") == None) + assert(router("x") == Found("x", SimpleRoute("x", true))) + assert(router("y") == Found("y", SimpleRoute("y", false))) + assert(router("z") == Found("z", SimpleRoute("z", true))) + assert(router("a") == NotFound) } test("throws when invalid route is passed") { diff --git a/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala index 9d85f1ddb0..3bf19c1604 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/RouterTest.scala @@ -1,7 +1,7 @@ package com.twitter.util.routing import com.twitter.conversions.DurationOps._ -import com.twitter.util.routing.dynamic.{DynamicRouter, DynamicRoute, Method, Request, Result} +import com.twitter.util.routing.dynamic.{DynamicRouter, DynamicRoute, Method, Request, Response} import com.twitter.util.routing.simple.{SimpleRoute, SimpleRouter} import com.twitter.util.{Await, Awaitable, Closable, Future, Time} import java.util.concurrent.atomic.AtomicBoolean @@ -18,20 +18,20 @@ class RouterTest extends AnyFunSuite { val router = new SimpleRouter(Map("hello" -> helloRoute, "goodbye" -> goodbyeRoute)) - assert(router("hello") == Some(helloRoute)) - assert(router("goodbye") == Some(goodbyeRoute)) - assert(router("something else") == None) + assert(router("hello") == Found("hello", helloRoute)) + assert(router("goodbye") == Found("goodbye", goodbyeRoute)) + assert(router("something else") == NotFound) } test("test a router with dynamic routes") { val abcReadHandler = - DynamicRoute(Method.Read, _.query.startsWith("abc/"), _ => Future.value(Result.Success)) + DynamicRoute(Method.Read, _.query.startsWith("abc/"), _ => Future.value(Response.Success)) val xyzReadHandler = - DynamicRoute(Method.Read, _.query.startsWith("xyz/"), _ => Future.value(Result.Failure)) + DynamicRoute(Method.Read, _.query.startsWith("xyz/"), _ => Future.value(Response.Failure)) val abcWriteHandler = - DynamicRoute(Method.Write, _.query.startsWith("abc/"), _ => Future.value(Result.Success)) + DynamicRoute(Method.Write, _.query.startsWith("abc/"), _ => Future.value(Response.Success)) val defWriteHandler = - DynamicRoute(Method.Write, _.query == "def", _ => Future.value(Result.Failure)) + DynamicRoute(Method.Write, _.query == "def", _ => Future.value(Response.Failure)) val router: DynamicRouter = DynamicRouter .newBuilder() @@ -41,13 +41,23 @@ class RouterTest extends AnyFunSuite { .withRoute(defWriteHandler) .newRouter() - assert(router(Request(Method.Read, "abc/123/456")) == Some(abcReadHandler)) - assert(router(Request(Method.Read, "def/123/456")) == None) - assert(router(Request(Method.Read, "xyz/123/456")) == Some(xyzReadHandler)) - assert(router(Request(Method.Write, "abc/123/456")) == Some(abcWriteHandler)) - assert(router(Request(Method.Write, "def")) == Some(defWriteHandler)) - assert(router(Request(Method.Write, "def/123/456")) == None) - assert(router(Request(Method.Write, "xyz/123/456")) == None) + assert( + router(Request(Method.Read, "abc/123/456")) == Found( + Request(Method.Read, "abc/123/456"), + abcReadHandler)) + assert(router(Request(Method.Read, "def/123/456")) == NotFound) + assert( + router(Request(Method.Read, "xyz/123/456")) == Found( + Request(Method.Read, "xyz/123/456"), + xyzReadHandler)) + assert( + router(Request(Method.Write, "abc/123/456")) == Found( + Request(Method.Write, "abc/123/456"), + abcWriteHandler)) + assert( + router(Request(Method.Write, "def")) == Found(Request(Method.Write, "def"), defWriteHandler)) + assert(router(Request(Method.Write, "def/123/456")) == NotFound) + assert(router(Request(Method.Write, "xyz/123/456")) == NotFound) await(router.close()) assert(router.isClosed) @@ -66,18 +76,16 @@ class RouterTest extends AnyFunSuite { assert(router.closedTimes.get() == 1) } - test("routing to closed router throws") { + test("routing to closed router returns a RouterClosed result") { val router = new SimpleRouter(Map.empty) await(router.close()) - intercept[ClosedRouterException] { - router("hello") - } + assert(router("hello") == RouterClosed) } test("router closes routes that are closable") { class AtomicClosable extends Closable { val isClosed = new AtomicBoolean(false); - override def close(deadline: Time): Future[Unit] = { + def close(deadline: Time): Future[Unit] = { isClosed.compareAndSet(false, true) Future.Done } @@ -87,10 +95,9 @@ class RouterTest extends AnyFunSuite { val closableB = new AtomicClosable val closableC = new AtomicClosable - class ClosableRouter extends Router[Boolean, Closable] { - override def label: String = "closable" - override def routes: Iterable[Closable] = Seq(closableA, closableB, closableC) - override protected def find(input: Boolean): Option[Closable] = None + class ClosableRouter + extends Router[Boolean, Closable]("closable", Seq(closableA, closableB, closableC)) { + protected def find(input: Boolean): Result = NotFound } val router = new ClosableRouter @@ -111,10 +118,8 @@ class RouterTest extends AnyFunSuite { val closableA = Closable.make(_ => Future.never) val closableB = Closable.make(_ => Future.Done) - class ClosableRouter extends Router[Boolean, Closable] { - override def label: String = "closable" - override def routes: Iterable[Closable] = Seq(closableA, closableB) - override protected def find(input: Boolean): Option[Closable] = None + class ClosableRouter extends Router[Boolean, Closable]("closable", Seq(closableA, closableB)) { + protected def find(input: Boolean): Result = NotFound } val router = new ClosableRouter @@ -124,19 +129,15 @@ class RouterTest extends AnyFunSuite { val closeFuture = router.close() // close the router, any further requests will fail assert(closeFuture.isDefined == false) - intercept[ClosedRouterException] { - router(true) - } + assert(router(true) == RouterClosed) assert(closeFuture.isDefined == false) // the future never completes } test("a router suppresses non-fatal exceptions on close") { val closableA = Closable.make(_ => throw new IllegalStateException("BOOM")) - class ClosableRouter extends Router[Boolean, Closable] { - override def label: String = "closable" - override def routes: Iterable[Closable] = Seq(closableA) - override protected def find(input: Boolean): Option[Closable] = None + class ClosableRouter extends Router[Boolean, Closable]("closable", Seq(closableA)) { + protected def find(input: Boolean): Result = NotFound } val router = new ClosableRouter @@ -147,10 +148,8 @@ class RouterTest extends AnyFunSuite { val closableA = Closable.make(_ => throw new OutOfMemoryError("BOOM")) val closableB = Closable.make(_ => throw new OutOfMemoryError("BAM")) - class ClosableRouter extends Router[Boolean, Closable] { - override def label: String = "closable" - override def routes: Iterable[Closable] = Seq(closableA, closableB) - override protected def find(input: Boolean): Option[Closable] = None + class ClosableRouter extends Router[Boolean, Closable]("closable", Seq(closableA, closableB)) { + protected def find(input: Boolean): Result = NotFound } val router = new ClosableRouter diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRoute.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRoute.scala index 8ce2567805..eb98472bb0 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRoute.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRoute.scala @@ -6,7 +6,7 @@ import com.twitter.util.{ClosableOnce, Future, Time} private[routing] case class DynamicRoute( method: Method, canHandle: Request => Boolean, - handle: Request => Future[Result]) + handle: Request => Future[Response]) extends ClosableOnce { - override protected def closeOnce(deadline: Time): Future[Unit] = Future.Done + protected def closeOnce(deadline: Time): Future[Unit] = Future.Done } diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala index 9cd946d163..705ff106bf 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/DynamicRouter.scala @@ -1,29 +1,37 @@ package com.twitter.util.routing.dynamic -import com.twitter.util.routing.{Generator, Router, RouterBuilder} +import com.twitter.util.routing.{ + Found, + Generator, + RouterInfo, + NotFound, + Result, + Router, + RouterBuilder +} private[routing] object DynamicRouter { def newBuilder(): RouterBuilder[Request, DynamicRoute, DynamicRouter] = RouterBuilder.newBuilder(new Generator[Request, DynamicRoute, DynamicRouter] { - override def apply(label: String, routes: Iterable[DynamicRoute]): DynamicRouter = - new DynamicRouter(routes.toSeq) + def apply(labelAndRoutes: RouterInfo[DynamicRoute]): DynamicRouter = + new DynamicRouter(labelAndRoutes.routes.toSeq) }) } // example to show how a router might be built using dynamic inputs private[routing] class DynamicRouter(handlers: Seq[DynamicRoute]) - extends Router[Request, DynamicRoute] { + extends Router[Request, DynamicRoute]("dynamic-test-router", handlers) { // example showing how we can optimize dynamic routes to lookup by a property of the input private[this] val handlersByMethod: Map[Method, Seq[DynamicRoute]] = handlers.groupBy(_.method) - override def label: String = "dynamic-test-router" - - override def routes: Iterable[DynamicRoute] = handlers - - override protected def find(input: Request): Option[DynamicRoute] = + protected def find(input: Request): Result = handlersByMethod.get(input.method) match { - case Some(handlers) => handlers.find(_.canHandle(input)) - case _ => None + case Some(handlers) => + handlers.find(_.canHandle(input)) match { + case Some(r) => Found(input, r) + case _ => NotFound + } + case _ => NotFound } } diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Response.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Response.scala new file mode 100644 index 0000000000..cdc9004c38 --- /dev/null +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Response.scala @@ -0,0 +1,8 @@ +package com.twitter.util.routing.dynamic + +private[routing] sealed trait Response + +private[routing] object Response { + case object Success extends Response + case object Failure extends Response +} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Result.scala b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Result.scala deleted file mode 100644 index 67f5ef9ff5..0000000000 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/Result.scala +++ /dev/null @@ -1,8 +0,0 @@ -package com.twitter.util.routing.dynamic - -private[routing] sealed trait Result - -private[routing] object Result { - case object Success extends Result - case object Failure extends Result -} diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRouter.scala b/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRouter.scala index 232452eb93..f290a2a583 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRouter.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/SimpleRouter.scala @@ -1,23 +1,22 @@ package com.twitter.util.routing.simple -import com.twitter.util.routing.Router +import com.twitter.util.routing.{Found, NotFound, Result, Router} import com.twitter.util.{Future, Time} import java.util.concurrent.atomic.AtomicInteger private[routing] class SimpleRouter(routeMap: Map[String, SimpleRoute]) - extends Router[String, SimpleRoute] { - - override def label: String = "test-router" + extends Router[String, SimpleRoute]("test-router", routeMap.values) { val closedTimes: AtomicInteger = new AtomicInteger(0) - def find(req: String): Option[SimpleRoute] = routeMap.get(req) + protected def find(req: String): Result = routeMap.get(req) match { + case Some(r) => Found(req, r) + case _ => NotFound + } override protected def closeOnce(deadline: Time): Future[Unit] = { closedTimes.incrementAndGet() Future.Done } - override def routes: Iterable[SimpleRoute] = routeMap.values - } From 45a9b3488086eb71e304906c37a93be2bb811c71 Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Fri, 29 Jan 2021 22:32:12 +0000 Subject: [PATCH 428/796] [util-security] scala 2.11 MatchError empty parameter causes build warning Problem `MatchError` on scala 2.11 will print out warnings of adaptation when no argument is supplied. 2.11 considers this a leaky abstraction. scala 2.12 has annotated the parameter with `@transient` leading to this warning not printed. - https://github.com/scala/scala/blob/v2.11.0/src/library/scala/MatchError.scala - https://github.com/scala/scala/blob/2.12.x/src/library/scala/MatchError.scala Solution Provide a string message as argument to `MatchError` JIRA Issues: SCALA-25 Differential Revision: https://phabricator.twitter.biz/D610196 --- .../src/main/scala/com/twitter/util/security/Credentials.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala index 5525033717..24cb8a7685 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala @@ -39,7 +39,7 @@ object Credentials { def auth: Parser[Tuple2[String, String]] = key ~ ":" ~ value ^^ { case k ~ ":" ~ v => (k, v) - case _ => throw new MatchError + case _ => throw new MatchError("Failed case matching on auth parser") } def content: Parser[Map[String, String]] = rep(auth) ^^ { auths => Map(auths: _*) } From 1aad454300cf8650cd97680a451c576ea3c6766e Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Mon, 1 Feb 2021 19:45:09 +0000 Subject: [PATCH 429/796] csl: Remove ryano from Owners/Project Files Differential Revision: https://phabricator.twitter.biz/D610849 --- PROJECT.yml | 1 - util-app/PROJECT | 1 - util-benchmark/PROJECT | 1 - util-codec/PROJECT | 1 - util-core/PROJECT | 1 - util-inject/PROJECT | 1 - util-intellij/PROJECT | 1 - util-logging/PROJECT | 1 - util-mock/PROJECT | 1 - util-slf4j-api/PROJECT | 1 - util-slf4j-jul-bridge/PROJECT | 1 - util-stats/PROJECT | 1 - util-zk/PROJECT | 1 - 13 files changed, 13 deletions(-) diff --git a/PROJECT.yml b/PROJECT.yml index bc21dcd32a..44c4a877f0 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -5,7 +5,6 @@ owners: - koliver - mnakamura - roanta - - ryano - vkostyukov watchers: - csl-team:ldap diff --git a/util-app/PROJECT b/util-app/PROJECT index 01c397156f..31678e4ba3 100644 --- a/util-app/PROJECT +++ b/util-app/PROJECT @@ -7,5 +7,4 @@ owners: - koliver - mnakamura - roanta - - ryano - vkostyukov diff --git a/util-benchmark/PROJECT b/util-benchmark/PROJECT index 8a8fabfcdc..eb8a36aae8 100644 --- a/util-benchmark/PROJECT +++ b/util-benchmark/PROJECT @@ -5,6 +5,5 @@ owners: - koliver - mnakamura - roanta - - ryano - vkostyukov - yufang diff --git a/util-codec/PROJECT b/util-codec/PROJECT index b15a45699b..85f4cb0288 100644 --- a/util-codec/PROJECT +++ b/util-codec/PROJECT @@ -4,6 +4,5 @@ owners: - banderson - mnakamura - roanta - - ryano - vkostyukov - yufang diff --git a/util-core/PROJECT b/util-core/PROJECT index 8a8fabfcdc..eb8a36aae8 100644 --- a/util-core/PROJECT +++ b/util-core/PROJECT @@ -5,6 +5,5 @@ owners: - koliver - mnakamura - roanta - - ryano - vkostyukov - yufang diff --git a/util-inject/PROJECT b/util-inject/PROJECT index 791a30f039..5010c7f26a 100644 --- a/util-inject/PROJECT +++ b/util-inject/PROJECT @@ -5,7 +5,6 @@ owners: - koliver - mnakamura - roanta - - ryano - vkostyukov watchers: - csl-team:ldap diff --git a/util-intellij/PROJECT b/util-intellij/PROJECT index 141b6376e6..24faddeed5 100644 --- a/util-intellij/PROJECT +++ b/util-intellij/PROJECT @@ -6,5 +6,4 @@ owners: - ibennett - mnakamura - roanta - - ryano - vkostyukov diff --git a/util-logging/PROJECT b/util-logging/PROJECT index 141b6376e6..24faddeed5 100644 --- a/util-logging/PROJECT +++ b/util-logging/PROJECT @@ -6,5 +6,4 @@ owners: - ibennett - mnakamura - roanta - - ryano - vkostyukov diff --git a/util-mock/PROJECT b/util-mock/PROJECT index 791a30f039..5010c7f26a 100644 --- a/util-mock/PROJECT +++ b/util-mock/PROJECT @@ -5,7 +5,6 @@ owners: - koliver - mnakamura - roanta - - ryano - vkostyukov watchers: - csl-team:ldap diff --git a/util-slf4j-api/PROJECT b/util-slf4j-api/PROJECT index 32d03a0ee9..ec3a2d20d5 100644 --- a/util-slf4j-api/PROJECT +++ b/util-slf4j-api/PROJECT @@ -6,4 +6,3 @@ owners: - ibennett - mnakamura - roanta - - ryano diff --git a/util-slf4j-jul-bridge/PROJECT b/util-slf4j-jul-bridge/PROJECT index 32d03a0ee9..ec3a2d20d5 100644 --- a/util-slf4j-jul-bridge/PROJECT +++ b/util-slf4j-jul-bridge/PROJECT @@ -6,4 +6,3 @@ owners: - ibennett - mnakamura - roanta - - ryano diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 41b2a610e9..22a3c9c5f6 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -6,5 +6,4 @@ owners: - koliver - mnakamura - roanta - - ryano - vkostyukov diff --git a/util-zk/PROJECT b/util-zk/PROJECT index e930d5fcfc..50aa1a2a50 100644 --- a/util-zk/PROJECT +++ b/util-zk/PROJECT @@ -4,7 +4,6 @@ owners: - banderson - mnakamura - roanta - - ryano - vkostyukov watchers: - coord-team@twitter.com From 23039c66936853d33caea82eb8bc9d53142ed8f3 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 2 Feb 2021 00:16:58 +0000 Subject: [PATCH 430/796] util-core: Remove Future.isDone Problem We have a method on Future named `isDone` which tells you whether the future has completed successfully, which is not obviously from its name. Solution Remove the method since the vast majority of use cases are actually interested in the `isDefined` method. Differential Revision: https://phabricator.twitter.biz/D602762 --- CHANGELOG.rst | 4 ++ .../scala/com/twitter/util/ConstFuture.scala | 2 - .../main/scala/com/twitter/util/Future.scala | 9 --- .../src/main/scala/com/twitter/util/Var.scala | 2 +- .../twitter/concurrent/AsyncMeterTest.scala | 4 +- .../twitter/concurrent/SchedulerTest.scala | 3 +- .../test/scala/com/twitter/io/PipeTest.scala | 66 +++++++++---------- .../scala/com/twitter/util/ClosableTest.scala | 6 +- .../test/scala/com/twitter/util/VarTest.scala | 2 +- 9 files changed, 45 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4a5b03f6ca..a80adb869c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,10 @@ Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ * util-core: removed `com.twitter.util.Config.` ``PHAB_ID=D580444`` +* util-core: removed `com.twitter.util.Future.isDone` method. The semantics of this method + are surprising in that `Future.exception(throwable).isDone == false`. Replace usages with + `Future.isDefined` or `Future.poll` ``PHAB_ID=D585700`` + New Features ~~~~~~~~~~~~ diff --git a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala index 97097ba746..8817d2a015 100644 --- a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala @@ -73,8 +73,6 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { override def isDefined: Boolean = true - override def isDone(implicit ev: this.type <:< Future[Unit]): Boolean = result.isReturn - override def toString: String = s"ConstFuture($result)" // Awaitable diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 2aa48fe612..301ca3ec35 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -57,7 +57,6 @@ object Future { /** A successfully satisfied constant `Future` of `false` */ val False: Future[Boolean] = new ConstFuture(Return.False) - private val SomeReturnUnit = Some(Return.Unit) private val NotApplied: Future[Nothing] = new NoFuture private val AlwaysNotApplied: Any => Future[Nothing] = scala.Function.const(NotApplied) private val tryToUnit: Try[Any] => Try[Unit] = { @@ -1540,14 +1539,6 @@ abstract class Future[+A] extends Awaitable[A] { self => */ def isDefined: Boolean = poll.isDefined - /** - * Checks whether a Unit-typed Future is done. By - * convention, futures of type Future[Unit] are used - * for signalling. - */ - def isDone(implicit ev: this.type <:< Future[Unit]): Boolean = - ev(this).poll == Future.SomeReturnUnit - /** * Polls for an available result. If the Future has been * satisfied, returns Some(result), otherwise None. diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 3968ef43ab..0dd17eb98e 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -123,7 +123,7 @@ trait Var[+T] { self => // combinators have been modified to evaluate their respective // Futures eagerly. val done = toClose.close() - assert(done.isDone) + assert(done.isDefined) } def close(deadline: Time): Future[Unit] = { diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala index 625d62ac13..a05c6b310d 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncMeterTest.scala @@ -22,7 +22,7 @@ class AsyncMeterTest extends AnyFunSuite { // an interpolated string. We get around it by evaluating first and passing the result to // `assert`. private[this] def isDone(f: Future[Unit]): Boolean = - f.isDone + f.isDefined private[this] def assertIsDone(f: Future[Unit]): Unit = assert(isDone(f)) @@ -85,7 +85,7 @@ class AsyncMeterTest extends AnyFunSuite { ctl.advance(1.second) timer.tick() - waiter.isDone + assertIsDone(waiter) } } diff --git a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala index 86b7e638e1..ee2a5a60c0 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala @@ -165,8 +165,7 @@ class ThreadPoolSchedulerTest extends AnyFunSuite with Eventually with Integrati def run(): Unit = p.setDone() }) - eventually { p.isDone } - + Await.result(p, 5.seconds) scheduler.shutdown() } } diff --git a/util-core/src/test/scala/com/twitter/io/PipeTest.scala b/util-core/src/test/scala/com/twitter/io/PipeTest.scala index 6b4b6cbb2a..373f4818e6 100644 --- a/util-core/src/test/scala/com/twitter/io/PipeTest.scala +++ b/util-core/src/test/scala/com/twitter/io/PipeTest.scala @@ -27,14 +27,14 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh // Passing that directly to `assert` causes problems because the `$conforms` is also seen as // an interpolated string. We get around it by evaluating first and passing the result to // `assert`. - private def isDone(f: Future[Unit]): Boolean = - f.isDone + private def IsSuccess(f: Future[Unit]): Boolean = + f.poll.exists(_.isReturn) - private def assertIsDone(f: Future[Unit]): Unit = - assert(isDone(f)) + private def assertIsSuccess(f: Future[Unit]): Unit = + assert(IsSuccess(f)) - private def assertIsNotDone(f: Future[Unit]): Unit = - assert(!isDone(f)) + private def assertIsNotSuccess(f: Future[Unit]): Unit = + assert(!IsSuccess(f)) private def assertRead(r: Reader[Buf], i: Int, j: Int): Unit = { val f = r.read() @@ -70,7 +70,7 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh private def assertReadEofAndClosed(rw: Pipe[Buf]): Unit = { assertReadNone(rw) - assertIsDone(rw.close()) + assertIsSuccess(rw.close()) } private def assertReadNone(r: Reader[Buf]): Unit = @@ -170,10 +170,10 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh val rw = new Pipe[Buf] val wf = rw.write(buf(0, 6)) - assertIsNotDone(wf) + assertIsNotSuccess(wf) assertRead(rw, 0, 6) - assertIsDone(wf) - assertIsNotDone(rw.close()) + assertIsSuccess(wf) + assertIsNotSuccess(rw.close()) assertReadEofAndClosed(rw) } @@ -185,10 +185,10 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh assert(!rf.isDefined) val wf = rw.write(buf(0, 6)) - assertIsDone(wf) + assertIsSuccess(wf) assertRead(rf, 0, 6) - assertIsNotDone(rw.close()) + assertIsNotSuccess(rw.close()) assertReadEofAndClosed(rw) } @@ -198,7 +198,7 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh assertFailedEx(rw.write(buf(0, 6))) val cf = rw.close() - assertIsNotDone(cf) + assertIsNotSuccess(cf) assertFailedEx(rw.read()) assertFailedEx(cf) @@ -207,9 +207,9 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh test("write after close") { val rw = new Pipe[Buf] val cf = rw.close() - assertIsNotDone(cf) + assertIsNotSuccess(cf) assertReadEofAndClosed(rw) - assertIsDone(cf) + assertIsSuccess(cf) intercept[IllegalStateException] { await(rw.write(buf(0, 1))) @@ -221,7 +221,7 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh var closed = false rw.onClose.ensure { closed = true } val wf = rw.write(buf(0, 1)) - assertIsNotDone(wf) + assertIsNotSuccess(wf) intercept[IllegalStateException] { await(rw.write(buf(0, 1))) @@ -240,7 +240,7 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh def readAfterCloseNoPendingReads() = { val rw = new Pipe[Buf] - assertIsNotDone(rw.close()) + assertIsNotSuccess(rw.close()) assertReadEofAndClosed(rw) } test("read after close with no pending reads") { @@ -251,10 +251,10 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh val rw = new Pipe[Buf] val wf = rw.write(buf(0, 1)) - assertIsNotDone(wf) + assertIsNotSuccess(wf) // close before the write is satisfied wipes the pending write - assertIsNotDone(rw.close()) + assertIsNotSuccess(rw.close()) intercept[IllegalStateException] { await(wf) } @@ -302,10 +302,10 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh test("close not satisfied until writes are read") { val rw = new Pipe[Buf] val cf = rw.write(buf(0, 6)).before(rw.close()) - assertIsNotDone(cf) + assertIsNotSuccess(cf) assertRead(rw, 0, 6) - assertIsNotDone(cf) + assertIsNotSuccess(cf) assertReadEofAndClosed(rw) } @@ -314,12 +314,12 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh val rf = rw.read() val cf = rf.flatMap { _ => rw.close() } assert(!rf.isDefined) - assertIsNotDone(cf) + assertIsNotSuccess(cf) - assertIsDone(rw.write(buf(0, 3))) + assertIsSuccess(rw.write(buf(0, 3))) assertRead(rf, 0, 3) - assertIsNotDone(cf) + assertIsNotSuccess(cf) assertReadEofAndClosed(rw) } test("close not satisfied until reads are fulfilled")(closeNotSatisfiedUntillAllReadsDone()) @@ -329,16 +329,16 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh val rf = rw.read() assert(!rf.isDefined) - assertIsDone(rw.close()) + assertIsSuccess(rw.close()) assert(rf.isDefined) } test("close while read pending")(closeWhileReadPending) def closeTwice() = { val rw = new Pipe[Buf] - assertIsNotDone(rw.close()) + assertIsNotSuccess(rw.close()) assertReadEofAndClosed(rw) - assertIsDone(rw.close()) + assertIsSuccess(rw.close()) assertReadEofAndClosed(rw) } test("close then close")(closeTwice()) @@ -347,7 +347,7 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh val rw = new Pipe[Buf] rw.fail(failedEx) val cf = rw.close() - assertIsNotDone(cf) + assertIsNotSuccess(cf) assertFailedEx(rw.read()) assertFailedEx(cf) @@ -358,7 +358,7 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh Time.withCurrentTimeFrozen { ctrl => val rw = new Pipe[Buf](timer) val cf = rw.close(1.second) - assertIsNotDone(cf) + assertIsNotSuccess(cf) ctrl.advance(1.second) timer.tick() @@ -374,10 +374,10 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh Time.withCurrentTimeFrozen { _ => val rw = new Pipe[Buf](timer) val cf = rw.close(1.second) - assertIsNotDone(cf) + assertIsNotSuccess(cf) rw.fail(failedEx) - assertIsNotDone(cf) + assertIsNotSuccess(cf) assertFailedEx(rw.read()) assertFailedEx(cf) @@ -387,9 +387,9 @@ class PipeTest extends AnyFunSuite with Matchers with ScalaCheckDrivenPropertyCh test("close while write pending") { val rw = new Pipe[Buf] val wf = rw.write(buf(0, 1)) - assertIsNotDone(wf) + assertIsNotSuccess(wf) val cf = rw.close() - assertIsNotDone(cf) + assertIsNotSuccess(cf) intercept[IllegalStateException] { await(wf) } diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index d2f07ba1a5..a485a75941 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -21,7 +21,7 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience // an interpolated string. We get around it by evaluating first and passing the result to // `assert`. private[this] def isDone(f: Future[Unit]): Boolean = - f.isDone + f.isDefined private[this] def assertIsDone(f: Future[Unit]): Unit = assert(isDone(f)) @@ -98,7 +98,7 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience assert( Future .value(1) - .map(_ => Closable.all(Closable.nop, Closable.nop).close().isDone) + .map(_ => Closable.all(Closable.nop, Closable.nop).close().isDefined) .poll .contains(Return.True) ) @@ -194,7 +194,7 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience assert( Future .value(1) - .map(_ => Closable.sequence(Closable.nop, Closable.nop).close().isDone) + .map(_ => Closable.sequence(Closable.nop, Closable.nop).close().isDefined) .poll .contains(Return.True) ) diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 06603bb8d7..d3eb6b8dd9 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -22,7 +22,7 @@ class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { // an interpolated string. We get around it by evaluating first and passing the result to // `assert`. private[this] def isDone(f: Future[Unit]): Boolean = - f.isDone + f.isDefined private[this] def assertIsDone(f: Future[Unit]): Unit = assert(isDone(f)) From e8464006d46bdd645f563025811919fec9b4db90 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 4 Feb 2021 22:14:09 +0000 Subject: [PATCH 431/796] oss: Remove Date Ranges from Copyright in READMEs Problem/Solution: The copyright lines in the project README files, do not need to be date ranges but can simply be the original (first) year the code was copyrighted. JIRA Issues: CSL-10588 Differential Revision: https://phabricator.twitter.biz/D613522 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 354b71835a..92c7bbda39 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,6 @@ val duration: Duration = elapsed() ## License -Copyright 2010-2021 Twitter, Inc. +Copyright 2010 Twitter, Inc. Licensed under the Apache License, Version 2.0: https://www.apache.org/licenses/LICENSE-2.0 From 1a7605efea3964509775aa2cfbc3c3adb2986b0a Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 5 Feb 2021 01:24:24 +0000 Subject: [PATCH 432/796] util-core: Java APIs use toStream when converting streams Problem One streaming test in finatra-http java failed in travis-ci scala 2.13 indicates it hits an infinite loop when converting an infinite stream from java to scala. Solution Scala 2.13 iterator `toSeq` longer redirect to `toStream` but construct an `immutable.Seq`. Explicitly call `toStream` when we are converting streams. Testing found another toString format difference for 2.13, fixed it. JIRA Issues: CSL-10630 Differential Revision: https://phabricator.twitter.biz/D613001 --- util-core/src/main/java/com/twitter/io/Readers.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java index fa2bd2e958..4e684ce5e3 100644 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ b/util-core/src/main/java/com/twitter/io/Readers.java @@ -85,7 +85,7 @@ public static Reader fromSeq(List list) { */ public static Reader fromSeq(Stream stream) { return Reader$.MODULE$.fromSeq(JavaConverters.asScalaIteratorConverter(stream.iterator()) - .asScala().toSeq()); + .asScala().toStream()); } /** From 076551e3c3207a0e4d22f6af94af91c878f77c92 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 10 Feb 2021 22:33:04 +0000 Subject: [PATCH 433/796] Twitter-oss: Prepare OSS libraries for release 21.2.0 Problem We want to release the next versions of our Twitter OSS libraries 21.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D615824 --- CHANGELOG.rst | 5 +++++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a80adb869c..df0e830826 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.2.0 +------ + +No Changes + 21.1.0 ------ diff --git a/README.md b/README.md index 92c7bbda39..b8a5d05000 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.1.0" +val utilCore = "com.twitter" %% "util-core" % "21.2.0" ``` # Units diff --git a/build.sbt b/build.sbt index e0dbf042b2..6de87a3943 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.2.0-SNAPSHOT" +val releaseVersion = "21.2.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From de22a82eb74d5b4d403bd88bcbe204a4ca96459f Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 10 Feb 2021 23:48:22 +0000 Subject: [PATCH 434/796] Twitter-oss: update OSS libraries post-release to 21.3.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.3.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D617097 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8a5d05000..d4e00fe9f0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=release)](https://travis-ci.com/twitter/util) +[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 6de87a3943..b3637707f6 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.2.0" +val releaseVersion = "21.3.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 4e3cd8a562f0ee03a9edd7a3a230a06c830af9fc Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Wed, 17 Feb 2021 20:09:17 +0000 Subject: [PATCH 435/796] [finagle/util/twitter-server] add hallam as owner to projects Problem: hallam should own some projects. Solution / Result: Add hallam. Add banderson to finagle-thriftmux Differential Revision: https://phabricator.twitter.biz/D619360 --- util-core/PROJECT | 1 + 1 file changed, 1 insertion(+) diff --git a/util-core/PROJECT b/util-core/PROJECT index eb8a36aae8..f6d911c574 100644 --- a/util-core/PROJECT +++ b/util-core/PROJECT @@ -2,6 +2,7 @@ owners: - banderson + - hallam - koliver - mnakamura - roanta From 715ee205f1d50cc2e1b47fb95af87f0777c9362b Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 19 Feb 2021 04:30:47 +0000 Subject: [PATCH 436/796] util-logging: Add Formatter#formatMessageLines variant Problem/Solution There's currently no way to format the message lines on an already processed message String. Update to add a `formatMessageLines` variant which takes a (possibly already truncated and argument formatted) message String and any Throwable from the LogRecord to create the array of message lines. Result A way to pass in a differently formatted message String potentially bypassing the `formatText` and truncation call in the version that only takes a LogRecord. Differential Revision: https://phabricator.twitter.biz/D619932 --- .../scala/com/twitter/logging/Formatter.scala | 40 ++++++++++++++++--- .../com/twitter/logging/FormatterTest.scala | 36 +++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala index 874672f0bc..3a1769df7b 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala @@ -143,22 +143,52 @@ class Formatter( */ def lineTerminator: String = "\n" + /** + * Truncates and formats the message of the given LogRecord. Formatting entails + * inserting any LogRecord parameters into the LogRecord message string. Truncation of the + * message text is controlled by the [[truncateAt]] parameter. If it is greater than 0, + * messages will be truncated at that length and ellipses appended. Messages are then returned + * as an array of lines where each entry is the line of text between newlines. + * + * @param record a [[javalog.LogRecord]] + * @return an Array representing the formatted lines of a messages. + * @see [[javalog.LogRecord#getParameters]] + */ def formatMessageLines(record: javalog.LogRecord): Array[String] = { val message = truncateText(formatText(record)) + formatMessageLines(message, record.getThrown) + } + /** + * Assumes the given message is the processed LogRecord message (see: [[formatText(record: javalog.LogRecord)]]), + * that is, any LogRecord parameters have been properly applied to the LogRecord message string as + * well as any desired truncation or trimming. The method does not further change the incoming message, + * it only splits the message on newlines and handles formatting of any given Throwable. + * + * The passed Throwable represents the LogRecord captured exception (if any). Processes the inputs + * as described in [[formatMessageLines(record: javalog.LogRecord)]]. + * + * @param message a formatted LogRecord message + * @param thrown the LogRecord captured exception (if any). Can be null. + * @see [[javalog.LogRecord#getThrown]] + */ + private[twitter] def formatMessageLines( + message: String, + thrown: Throwable + ): Array[String] = { val containsNewLine = message.indexOf('\n') >= 0 - if (!containsNewLine && record.getThrown == null) { + if (!containsNewLine && thrown == null) { Array(message) } else { val splitOnNewlines = message.split("\n") - val numThrowLines = if (record.getThrown == null) 0 else 20 + val numThrowLines = if (thrown == null) 0 else 20 val lines = new mutable.ArrayBuffer[String](splitOnNewlines.length + numThrowLines) lines ++= splitOnNewlines - if (record.getThrown ne null) { - val traceLines = Formatter.formatStackTrace(record.getThrown, truncateStackTracesAt) - lines += record.getThrown.toString + if (thrown ne null) { + val traceLines = Formatter.formatStackTrace(thrown, truncateStackTracesAt) + lines += thrown.toString if (traceLines.nonEmpty) lines ++= traceLines } diff --git a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala index 53f5ada361..e3f8892322 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala @@ -95,6 +95,42 @@ class FormatterTest extends AnyWordSpec { assert(basicFormatter.formatText(record) == "error 123") } + "format message lines" in { + val record = new LogRecord(Level.ERROR, "how now brown cow") + assert(basicFormatter.formatMessageLines(record) sameElements Array("how now brown cow")) + } + + "format message lines multiple values" in { + val record = new LogRecord(Level.ERROR, "how now brown cow\nthe quick brown fox") + assert(basicFormatter + .formatMessageLines(record) sameElements Array("how now brown cow", "the quick brown fox")) + } + + "format message lines with message" in { + val record = new LogRecord(Level.ERROR, "how now brown cow") + assert( + basicFormatter.formatMessageLines("how now brown cow", record.getThrown) sameElements Array( + "how now brown cow")) + } + + "format message lines with message and multiple values" in { + val record = new LogRecord(Level.ERROR, "how now brown cow\nthe quick brown fox") + // ignores the message from the record + assert( + basicFormatter.formatMessageLines("how now brown cow", record.getThrown) sameElements Array( + "how now brown cow")) + } + + "format message lines with message and multiple values again" in { + val record = new LogRecord(Level.ERROR, "how now brown cow\nthe quick brown fox") + // ignores the message from the record + assert( + basicFormatter + .formatMessageLines( + "how now brown cow\nthe quick brown fox", + record.getThrown) sameElements Array("how now brown cow", "the quick brown fox")) + } + "format a timestamp" in { assert(utcFormatter.format(record1) == "ERR [20080329-05:53:16.722] jobs: boo.\n") } From ccbb3b2bc7f235c2294f80a0fa26e804b05a4528 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 20 Feb 2021 01:06:07 +0000 Subject: [PATCH 437/796] util: Add GitHub Action continuous integration Problem/Solution Add continuous integration with a GitHub Action. Differential Revision: https://phabricator.twitter.biz/D621157 --- .github/workflows/continuous-build.yml | 62 ++++++++++++++++++++++++++ README.md | 2 +- 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/continuous-build.yml diff --git a/.github/workflows/continuous-build.yml b/.github/workflows/continuous-build.yml new file mode 100644 index 0000000000..78d6772300 --- /dev/null +++ b/.github/workflows/continuous-build.yml @@ -0,0 +1,62 @@ +name: continuous integration + +env: + JAVA_OPTS: "-Dsbt.log.noformat=true" + TRAVIS: "true" # pretend we're TravisCI + +defaults: + run: + shell: bash +on: + push: + branches: + - develop + - release + pull_request: + +jobs: + build: + strategy: + fail-fast: false + matrix: + scala: [2.11.12, 2.12.8, 2.13.1] + java: ['1.8', '1.11'] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.3.4 + - uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + - name: echo java version + run: java -Xmx32m -version + - name: echo javac version + run: javac -J-Xmx32m -version + - name: cache build dependencies + uses: actions/cache@v2 + env: + cache-name: cache-build-deps + with: + path: | + ~/.ivy2/cache + ~/.ivy2/local/com.twitter + ~/.sbt + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.scala }}-${{ matrix.java }} + - name: update cache + run: | + if [ -f ~/.ivy2/cache ]; then + find ~/.ivy2/cache -name "ivydata-*.properties" -delete + fi + if [ -f ~/.sbt ]; then + find ~/.sbt -name "*.lock" -delete + fi + - name: test + run: ${{ format('./sbt ++{0} clean coverage test coverageReport', matrix.scala) }} + - name: test util-intellij with scala 2.11 + if: ${{ startsWith(matrix.scala, '2.11') }} + run: ${{ format('./sbt ++{0} "project util-intellij" coverage test coverageReport', matrix.scala) }} + - name: coverage + if: ${{ success() }} + run: ${{ format('./sbt ++{0} coverageAggregate', matrix.scala) }} + - name: upload coverage + if: ${{ success() }} + run: bash <(curl -s https://codecov.io/bash) diff --git a/README.md b/README.md index d4e00fe9f0..114881c792 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build status](https://travis-ci.com/twitter/util.svg?branch=develop)](https://travis-ci.com/twitter/util) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) From 955754d422b7a477113841c6eb2fa77996c30244 Mon Sep 17 00:00:00 2001 From: Muxian Wu Date: Wed, 24 Feb 2021 04:06:18 +0000 Subject: [PATCH 438/796] util-core: Support creating and deconstructing unsigned 128-bit buffers in Buf Problem: There's a lack of support for creating and deconstructing unsigned 128-bit big endian/little endian encoded buffers in c.t.io. Solution: Add U128BE and U128LE objects, for creating and deconstructing unsigned 128-bit big endian/little endian encoded buffers. Result: Implementation and test cases added. We could represent IPv6 IP addresses using two 64-bit longs, some operations required conversion between IPv6 IP addresses represented in bytes and the two 64-bits longs. With U128BE and U128LE added, we are able to do conversions like: ``` val buf = Buf.U128BE(highBits, lowerBits) val ipInByteArray = Buf.ByteArray.Owned.extract(buf) ``` and ``` val buf = Buf.ByteArray.apply(ipInByteArray: _*) val byteBuf = Buf.ByteBuffer.Owned.extract(buf) val highBits = byteBuf.getLong val lowBits = byteBuf.getLong ``` JIRA Issues: TRAFFIC-1212 Differential Revision: https://phabricator.twitter.biz/D606905 --- CHANGELOG.rst | 4 + .../src/main/scala/com/twitter/io/Buf.scala | 118 ++++++++++++++++++ .../test/scala/com/twitter/io/BufTest.scala | 43 ++++++- 3 files changed, 163 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index df0e830826..335569f2a3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +* util-core: `c.t.io` now supports creating and deconstructing unsigned 128-bit buffers + in Buf. ``PHAB_ID=D606905`` + 21.2.0 ------ diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index e1dc9080b4..cc54e985f1 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -1243,6 +1243,65 @@ object Buf { } } + /** + * Create and deconstruct unsigned 128-bit + * big endian encoded buffers. + * + * Deconstructing will return the value + * as well as the remaining buffer. + */ + object U128BE { + def apply(highBits: Long, lowBits: Long): Buf = { + val arr = new Array[Byte](16) + arr(0) = ((highBits >> 56) & 0xff).toByte + arr(1) = ((highBits >> 48) & 0xff).toByte + arr(2) = ((highBits >> 40) & 0xff).toByte + arr(3) = ((highBits >> 32) & 0xff).toByte + arr(4) = ((highBits >> 24) & 0xff).toByte + arr(5) = ((highBits >> 16) & 0xff).toByte + arr(6) = ((highBits >> 8) & 0xff).toByte + arr(7) = (highBits & 0xff).toByte + arr(8) = ((lowBits >> 56) & 0xff).toByte + arr(9) = ((lowBits >> 48) & 0xff).toByte + arr(10) = ((lowBits >> 40) & 0xff).toByte + arr(11) = ((lowBits >> 32) & 0xff).toByte + arr(12) = ((lowBits >> 24) & 0xff).toByte + arr(13) = ((lowBits >> 16) & 0xff).toByte + arr(14) = ((lowBits >> 8) & 0xff).toByte + arr(15) = (lowBits & 0xff).toByte + ByteArray.Owned(arr) + } + + // returns (highBits, lowBits, Buf) + def unapply(buf: Buf): Option[(Long, Long, Buf)] = + if (buf.length < 16) None + else { + val arr = new Array[Byte](16) + buf.slice(0, 16).write(arr, 0) + val rem = buf.slice(16, buf.length) + + val highBits = + ((arr(0) & 0xff).toLong << 56) | + ((arr(1) & 0xff).toLong << 48) | + ((arr(2) & 0xff).toLong << 40) | + ((arr(3) & 0xff).toLong << 32) | + ((arr(4) & 0xff).toLong << 24) | + ((arr(5) & 0xff).toLong << 16) | + ((arr(6) & 0xff).toLong << 8) | + (arr(7) & 0xff).toLong + val lowBits = + ((arr(8) & 0xff).toLong << 56) | + ((arr(9) & 0xff).toLong << 48) | + ((arr(10) & 0xff).toLong << 40) | + ((arr(11) & 0xff).toLong << 32) | + ((arr(12) & 0xff).toLong << 24) | + ((arr(13) & 0xff).toLong << 16) | + ((arr(14) & 0xff).toLong << 8) | + (arr(15) & 0xff).toLong + Some((highBits, lowBits, rem)) + } + } + /** * Create and deconstruct unsigned 32-bit * little endian encoded buffers. @@ -1317,6 +1376,65 @@ object Buf { } } + /** + * Create and deconstruct unsigned 128-bit + * little endian encoded buffers. + * + * Deconstructing will return the value + * as well as the remaining buffer. + */ + object U128LE { + def apply(highBits: Long, lowBits: Long): Buf = { + val arr = new Array[Byte](16) + arr(0) = (lowBits & 0xff).toByte + arr(1) = ((lowBits >> 8) & 0xff).toByte + arr(2) = ((lowBits >> 16) & 0xff).toByte + arr(3) = ((lowBits >> 24) & 0xff).toByte + arr(4) = ((lowBits >> 32) & 0xff).toByte + arr(5) = ((lowBits >> 40) & 0xff).toByte + arr(6) = ((lowBits >> 48) & 0xff).toByte + arr(7) = ((lowBits >> 56) & 0xff).toByte + arr(8) = (highBits & 0xff).toByte + arr(9) = ((highBits >> 8) & 0xff).toByte + arr(10) = ((highBits >> 16) & 0xff).toByte + arr(11) = ((highBits >> 24) & 0xff).toByte + arr(12) = ((highBits >> 32) & 0xff).toByte + arr(13) = ((highBits >> 40) & 0xff).toByte + arr(14) = ((highBits >> 48) & 0xff).toByte + arr(15) = ((highBits >> 56) & 0xff).toByte + ByteArray.Owned(arr) + } + + // returns (higherBits, lowerBits, Buf) + def unapply(buf: Buf): Option[(Long, Long, Buf)] = + if (buf.length < 16) None + else { + val arr = new Array[Byte](16) + buf.slice(0, 16).write(arr, 0) + val rem = buf.slice(16, buf.length) + + val lowBits = + (arr(0) & 0xff).toLong | + ((arr(1) & 0xff).toLong << 8) | + ((arr(2) & 0xff).toLong << 16) | + ((arr(3) & 0xff).toLong << 24) | + ((arr(4) & 0xff).toLong << 32) | + ((arr(5) & 0xff).toLong << 40) | + ((arr(6) & 0xff).toLong << 48) | + ((arr(7) & 0xff).toLong << 56) + val highBits = + (arr(8) & 0xff).toLong | + ((arr(9) & 0xff).toLong << 8) | + ((arr(10) & 0xff).toLong << 16) | + ((arr(11) & 0xff).toLong << 24) | + ((arr(12) & 0xff).toLong << 32) | + ((arr(13) & 0xff).toLong << 40) | + ((arr(14) & 0xff).toLong << 48) | + ((arr(15) & 0xff).toLong << 56) + Some((highBits, lowBits, rem)) + } + } + // Modeled after the Vector.++ operator, but use indexes instead of `for` // comprehensions for the optimized paths to save allocations and time. // See Scala ticket SI-7725 for details. diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index 41c42582ab..c5156a3e3e 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -872,6 +872,16 @@ class BufTest out == in && outByteBuf.getLong == in }) + check(Prop.forAll { (in1: Long, in2: Long) => + val buf = Buf.U128BE(in1, in2) + val Buf.U128BE(out1, out2, _) = buf + + val outByteBuf = java.nio.ByteBuffer.wrap(Buf.ByteArray.Owned.extract(buf)) + outByteBuf.order(java.nio.ByteOrder.BIG_ENDIAN) + + out1 == in1 && out2 == in2 && outByteBuf.getLong == in1 && outByteBuf.getLong == in2 + }) + check(Prop.forAll { (in: Long) => val buf = Buf.U64LE(in) val Buf.U64LE(out, _) = buf @@ -882,28 +892,57 @@ class BufTest out == in && outByteBuf.getLong == in }) + check(Prop.forAll { (in1: Long, in2: Long) => + val buf = Buf.U128LE(in1, in2) + val Buf.U128LE(out1, out2, _) = buf + + val outByteBuf = java.nio.ByteBuffer.wrap(Buf.ByteArray.Owned.extract(buf)) + outByteBuf.order(java.nio.ByteOrder.LITTLE_ENDIAN) + + out1 == in1 && out2 == in2 && outByteBuf.getLong == in2 && outByteBuf.getLong == in1 + }) + test("Buf num matching") { val hasMatch = Buf.Empty match { case Buf.U32BE(_, _) => true case Buf.U64BE(_, _) => true + case Buf.U128BE(_, _, _) => true case Buf.U32LE(_, _) => true case Buf.U64LE(_, _) => true + case Buf.U128LE(_, _, _) => true case _ => false } assert(!hasMatch) + // val uBigInt128 = BigInt("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 16) + val buf = Buf.Empty .concat(Buf.U32BE(Int.MaxValue)) .concat(Buf.U64BE(Long.MaxValue)) + .concat(Buf.U128BE(Long.MaxValue, Long.MaxValue)) .concat(Buf.U32LE(Int.MinValue)) .concat(Buf.U64LE(Long.MinValue)) - - val Buf.U32BE(be32, Buf.U64BE(be64, Buf.U32LE(le32, Buf.U64LE(le64, rem)))) = buf + .concat(Buf.U128LE(Long.MaxValue, Long.MaxValue)) + + val Buf.U32BE( + be32, + Buf + .U64BE( + be64, + Buf.U128BE( + be1281, + be1282, + Buf.U32LE(le32, Buf.U64LE(le64, Buf.U128LE(le1281, le1282, rem)))))) = + buf assert(be32 == Int.MaxValue) assert(be64 == Long.MaxValue) + assert(be1281 == Long.MaxValue) + assert(be1282 == Long.MaxValue) assert(le32 == Int.MinValue) assert(le64 == Long.MinValue) + assert(le1281 == Long.MaxValue) + assert(le1282 == Long.MaxValue) assert(rem == Buf.Empty) } From 35abecad4622efb8c13ef5607dbb09f70c023e4a Mon Sep 17 00:00:00 2001 From: Ryan O'Neill Date: Wed, 24 Feb 2021 20:23:21 +0000 Subject: [PATCH 439/796] util: Make `ProxyByteReader` and `ProxyByteWriter` Public Problem `ProxyByteReader` is incredibly useful for reading specialized information out of a `Buf` while still benefitting from all the already existing goodness that comes for free from the `ByteReader` implementation. However, it is not currently accessible to code outside of the `com.twitter` namespace. Solution Make `ProxyByteReader` and `ProxyByteWriter` both public, and modify `ProxyByteReader`s structure so that it aligns with `ProxyByteWriter`. Differential Revision: https://phabricator.twitter.biz/D622705 --- CHANGELOG.rst | 6 ++ .../scala/com/twitter/io/ByteReader.scala | 68 ----------------- .../com/twitter/io/ProxyByteReader.scala | 74 +++++++++++++++++++ .../com/twitter/io/ProxyByteWriter.scala | 6 +- 4 files changed, 83 insertions(+), 71 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/io/ProxyByteReader.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 335569f2a3..ce82a8e29e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,9 +8,15 @@ Unreleased ---------- New Features +~~~~~~~~~~~~ + * util-core: `c.t.io` now supports creating and deconstructing unsigned 128-bit buffers in Buf. ``PHAB_ID=D606905`` +* util-core: `c.t.io.ProxyByteReader` and `c.t.io.ProxyByteWriter` are now public. They are + useful for wrapping an existing `ByteReader` or `ByteWriter` and extending its functionality + without modifying the underlying instance. ``PHAB_ID=D622705`` + 21.2.0 ------ diff --git a/util-core/src/main/scala/com/twitter/io/ByteReader.scala b/util-core/src/main/scala/com/twitter/io/ByteReader.scala index eb6f5ae3df..7db3419546 100644 --- a/util-core/src/main/scala/com/twitter/io/ByteReader.scala +++ b/util-core/src/main/scala/com/twitter/io/ByteReader.scala @@ -203,74 +203,6 @@ trait ByteReader extends AutoCloseable { def readAll(): Buf } -private[twitter] trait ProxyByteReader extends ByteReader { - protected def reader: ByteReader - - def remaining: Int = reader.remaining - - def remainingUntil(byte: Byte): Int = reader.remainingUntil(byte) - - def readByte(): Byte = reader.readByte() - - def readUnsignedByte(): Short = reader.readUnsignedByte() - - def readShortBE(): Short = reader.readShortBE() - - def readShortLE(): Short = reader.readShortLE() - - def readUnsignedShortBE(): Int = reader.readUnsignedShortBE() - - def readUnsignedShortLE(): Int = reader.readUnsignedShortLE() - - def readMediumBE(): Int = reader.readMediumBE() - - def readMediumLE(): Int = reader.readMediumLE() - - def readUnsignedMediumBE(): Int = reader.readUnsignedMediumBE() - - def readUnsignedMediumLE(): Int = reader.readUnsignedMediumLE() - - def readIntBE(): Int = reader.readIntBE() - - def readIntLE(): Int = reader.readIntLE() - - def readUnsignedIntBE(): Long = reader.readUnsignedIntBE() - - def readUnsignedIntLE(): Long = reader.readUnsignedIntLE() - - def readLongBE(): Long = reader.readLongBE() - - def readLongLE(): Long = reader.readLongLE() - - def readUnsignedLongBE(): BigInt = reader.readUnsignedLongBE() - - def readUnsignedLongLE(): BigInt = reader.readUnsignedLongLE() - - def readFloatBE(): Float = reader.readFloatBE() - - def readFloatLE(): Float = reader.readFloatLE() - - def readDoubleBE(): Double = reader.readDoubleBE() - - def readDoubleLE(): Double = reader.readDoubleLE() - - def readBytes(n: Int): Buf = reader.readBytes(n) - - def readString(bytes: Int, charset: Charset): String = reader.readString(bytes, charset) - - def skip(n: Int): Unit = reader.skip(n) - - def readAll(): Buf = reader.readAll() - - def process(from: Int, until: Int, processor: Buf.Processor): Int = - reader.process(from, until, processor) - - def process(processor: Buf.Processor): Int = - reader.process(processor) - - def close(): Unit = reader.close() -} - object ByteReader { /** diff --git a/util-core/src/main/scala/com/twitter/io/ProxyByteReader.scala b/util-core/src/main/scala/com/twitter/io/ProxyByteReader.scala new file mode 100644 index 0000000000..a07b001b60 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/ProxyByteReader.scala @@ -0,0 +1,74 @@ +package com.twitter.io + +import java.nio.charset.Charset + +/** + * A proxy [[ByteReader]] that forwards all calls to another [[ByteReader]]. + * This is useful if you want to wrap-but-modify an existing [[ByteReader]]. + */ +abstract class ProxyByteReader(underlying: ByteReader) extends ByteReader { + + def remaining: Int = underlying.remaining + + def remainingUntil(byte: Byte): Int = underlying.remainingUntil(byte) + + def readByte(): Byte = underlying.readByte() + + def readUnsignedByte(): Short = underlying.readUnsignedByte() + + def readShortBE(): Short = underlying.readShortBE() + + def readShortLE(): Short = underlying.readShortLE() + + def readUnsignedShortBE(): Int = underlying.readUnsignedShortBE() + + def readUnsignedShortLE(): Int = underlying.readUnsignedShortLE() + + def readMediumBE(): Int = underlying.readMediumBE() + + def readMediumLE(): Int = underlying.readMediumLE() + + def readUnsignedMediumBE(): Int = underlying.readUnsignedMediumBE() + + def readUnsignedMediumLE(): Int = underlying.readUnsignedMediumLE() + + def readIntBE(): Int = underlying.readIntBE() + + def readIntLE(): Int = underlying.readIntLE() + + def readUnsignedIntBE(): Long = underlying.readUnsignedIntBE() + + def readUnsignedIntLE(): Long = underlying.readUnsignedIntLE() + + def readLongBE(): Long = underlying.readLongBE() + + def readLongLE(): Long = underlying.readLongLE() + + def readUnsignedLongBE(): BigInt = underlying.readUnsignedLongBE() + + def readUnsignedLongLE(): BigInt = underlying.readUnsignedLongLE() + + def readFloatBE(): Float = underlying.readFloatBE() + + def readFloatLE(): Float = underlying.readFloatLE() + + def readDoubleBE(): Double = underlying.readDoubleBE() + + def readDoubleLE(): Double = underlying.readDoubleLE() + + def readBytes(n: Int): Buf = underlying.readBytes(n) + + def readString(bytes: Int, charset: Charset): String = underlying.readString(bytes, charset) + + def skip(n: Int): Unit = underlying.skip(n) + + def readAll(): Buf = underlying.readAll() + + def process(from: Int, until: Int, processor: Buf.Processor): Int = + underlying.process(from, until, processor) + + def process(processor: Buf.Processor): Int = + underlying.process(processor) + + def close(): Unit = underlying.close() +} diff --git a/util-core/src/main/scala/com/twitter/io/ProxyByteWriter.scala b/util-core/src/main/scala/com/twitter/io/ProxyByteWriter.scala index 2ea6d669a4..8d5a713d28 100644 --- a/util-core/src/main/scala/com/twitter/io/ProxyByteWriter.scala +++ b/util-core/src/main/scala/com/twitter/io/ProxyByteWriter.scala @@ -3,10 +3,10 @@ package com.twitter.io import java.nio.charset.Charset /** - * A simple proxy ByteWriter that forwards all calls to another ByteWriter. - * This is useful if you want to wrap-but-modify an existing ByteWriter. + * A proxy [[ByteWriter]] that forwards all calls to another [[ByteWriter]]. + * This is useful if you want to wrap-but-modify an existing [[ByteWriter]]. */ -private[twitter] abstract class ProxyByteWriter(underlying: ByteWriter) extends AbstractByteWriter { +abstract class ProxyByteWriter(underlying: ByteWriter) extends AbstractByteWriter { def writeString(string: CharSequence, charset: Charset): ProxyByteWriter.this.type = { underlying.writeString(string, charset) From cddcbf823acfb8f39efac3099ef2226df9139cb3 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 24 Feb 2021 21:09:42 +0000 Subject: [PATCH 440/796] util-stats: Improve java compat for MetricBuilder Problem There are no docs for how to use custom percentiles, and it's also tough for Java users to construct Seqs. Solution Add some docs for custom percentiles, and some nice facilities for Java users to use them. JIRA Issues: CSL-10686 Differential Revision: https://phabricator.twitter.biz/D620425 --- CHANGELOG.rst | 8 ++++ doc/src/sphinx/util-stats/user_guide.rst | 14 +++++- .../finagle/stats/InMemoryStatsReceiver.scala | 4 +- .../twitter/finagle/stats/MetricBuilder.scala | 34 +++++++++++++-- .../twitter/finagle/stats/MetricUnit.scala | 8 +++- .../stats/NameTranslatingStatsReceiver.scala | 7 +-- .../RelativeNameMarkingStatsReceiver.scala | 6 +-- .../finagle/stats/RollupStatsReceiver.scala | 6 +-- .../twitter/finagle/stats/StatsReceiver.scala | 8 ++-- .../stats/MetricBuilderCompilationTest.java | 43 +++++++++++++++++++ .../stats/StatsReceiverCompilationTest.java | 2 +- 11 files changed, 118 insertions(+), 22 deletions(-) create mode 100644 util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ce82a8e29e..93f8537b12 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -38,6 +38,14 @@ Breaking API Changes are surprising in that `Future.exception(throwable).isDone == false`. Replace usages with `Future.isDefined` or `Future.poll` ``PHAB_ID=D585700`` +* util-stats: Changed `com.twitter.finagle.stats.MethodBuilder#withName`, + `com.twitter.finagle.stats.MethodBuilder#withRelativeName`, + `com.twitter.finagle.stats.MethodBuilder#withPercentiles`, + `com.twitter.finagle.stats.MethodBuilder#counter`, and + `com.twitter.finagle.stats.MethodBuilder#histogram`, to accept varargs as parameters, + rather than a `Seq`. Added a Java-friendly `com.twitter.finagle.stats.MethodBuilder#gauge`. + ``PHAB_ID=D620425`` + New Features ~~~~~~~~~~~~ diff --git a/doc/src/sphinx/util-stats/user_guide.rst b/doc/src/sphinx/util-stats/user_guide.rst index 559bd12172..68ec17dbbb 100644 --- a/doc/src/sphinx/util-stats/user_guide.rst +++ b/doc/src/sphinx/util-stats/user_guide.rst @@ -137,7 +137,6 @@ function **must also** be thread-safe. Leveraging Verbosity Levels --------------------------- - Introducing a new application metric (i.e., a histogram or a counter) is always a trade-off between its operational value and its cost within observability. Verbosity levels for StatsReceivers are aiming to reduce the observability cost of Finagle-based services by allowing for a granular control @@ -152,6 +151,19 @@ cost. However taking this to extremes may drastically affect operability of your recommend using your judgment to make sure denylisting a given metric will not reduce a process' visibility. +Custom Histogram Percentiles +---------------------------- +If you need a custom histogram percentile, you can use the `MetricBuilder` interface, and populate +it via `MetricBuilder#withPercentiles`. + +.. code-block:: scala + + import com.twitter.finagle.stats.{DefaultStatsReceiver, StatsReceiver, MetricBuilder} + + val sr: StatsReceiver = DefaultStatsReceiver + val mb: MetricBuilder = sr.metricBuilder().withPercentiles(Seq(0.99, 0.999, 0.9999, 0.90, 0.6)) + val stat = mb.histogram("my", "cool", "histo") + Access needed to a StatsReceiver in an inconvenient place --------------------------------------------------------- Ideally classes would be passed a properly scoped ``StatsReceiver`` in their constructor but this diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 934ebbb4a4..fabbf95908 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -70,7 +70,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { new ConcurrentHashMap[Seq[String], MetricSchema]().asScala override def counter(name: String*): ReadableCounter = - counter(CounterSchema(this.metricBuilder().withName(name))) + counter(CounterSchema(this.metricBuilder().withName(name: _*))) /** * Creates a [[ReadableCounter]] of the given `name`. @@ -100,7 +100,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } override def stat(name: String*): ReadableStat = - stat(HistogramSchema(this.metricBuilder().withName(name))) + stat(HistogramSchema(this.metricBuilder().withName(name: _*))) /** * Creates a [[ReadableStat]] of the given `name`. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 7975542ad9..e2a602a753 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -1,5 +1,8 @@ package com.twitter.finagle.stats +import java.util.function.Supplier +import scala.annotation.varargs + /** * Represents the "role" this service plays with respect to this metric. * @@ -7,7 +10,13 @@ package com.twitter.finagle.stats * request to another service). In many cases, there is no relevant role for a metric, in which * case NoRole should be used. */ -sealed trait SourceRole +sealed trait SourceRole { + + /** + * Java-friendly helper for accessing the object itself. + */ + def getInstance(): SourceRole = this +} case object NoRoleSpecified extends SourceRole case object Client extends SourceRole case object Server extends SourceRole @@ -107,12 +116,16 @@ class MetricBuilder( def withRole(role: SourceRole): MetricBuilder = this.copy(role = role) - def withName(name: Seq[String]): MetricBuilder = this.copy(name = name) + @varargs + def withName(name: String*): MetricBuilder = this.copy(name = name) - def withRelativeName(relativeName: Seq[String]): MetricBuilder = + @varargs + def withRelativeName(relativeName: String*): MetricBuilder = if (this.relativeName == Seq.empty) this.copy(relativeName = relativeName) else this - def withPercentiles(percentiles: IndexedSeq[Double]) = this.copy(percentiles = percentiles) + @varargs + def withPercentiles(percentiles: Double*): MetricBuilder = + this.copy(percentiles = percentiles.toIndexedSeq) /** * Generates a CounterSchema which can be used to create a counter in a StatsReceiver. @@ -139,6 +152,7 @@ class MetricBuilder( * Produce a counter as described by the builder inside the underlying StatsReceiver. * @return the counter created. */ + @varargs def counter(name: String*): Counter = { val schema = this.copy(name = name).counterSchema this.statsReceiver.counter(schema) @@ -153,10 +167,22 @@ class MetricBuilder( this.statsReceiver.addGauge(schema)(f) } + /** + * Produce a gauge as described by the builder inside the underlying StatsReceiver. + * This API is for Java compatibility + * @return the gauge created. + */ + @varargs + def gauge(f: Supplier[Float], name: String*): Gauge = { + val schema = this.copy(name = name).gaugeSchema + this.statsReceiver.addGauge(schema)(f.get()) + } + /** * Produce a histogram as described by the builder inside the underlying StatsReceiver. * @return the histogram created. */ + @varargs def histogram(name: String*): Stat = { val schema = this.copy(name = name).histogramSchema this.statsReceiver.stat(schema) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala index 2cf1557fc9..b52c9e5946 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUnit.scala @@ -9,7 +9,13 @@ package com.twitter.finagle.stats * Requests (for successes, failures, and requests) * Percentage (for CPU Util, Memory Usage) */ -sealed trait MetricUnit +sealed trait MetricUnit { + + /** + * Java-friendly helper for accessing the object itself. + */ + def getInstance(): MetricUnit = this +} case object Unspecified extends MetricUnit case object Bytes extends MetricUnit case object Kilobytes extends MetricUnit diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index 7992ee9013..0c64344227 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -20,18 +20,19 @@ abstract class NameTranslatingStatsReceiver( override def counter(counterSchema: CounterSchema): Counter = { self.counter( CounterSchema( - counterSchema.metricBuilder.withName(translate(counterSchema.metricBuilder.name)))) + counterSchema.metricBuilder.withName(translate(counterSchema.metricBuilder.name): _*))) } override def stat(histogramSchema: HistogramSchema): Stat = { self.stat( HistogramSchema( - histogramSchema.metricBuilder.withName(translate(histogramSchema.metricBuilder.name)))) + histogramSchema.metricBuilder.withName(translate(histogramSchema.metricBuilder.name): _*))) } override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { self.addGauge( - GaugeSchema(gaugeSchema.metricBuilder.withName(translate(gaugeSchema.metricBuilder.name))))(f) + GaugeSchema( + gaugeSchema.metricBuilder.withName(translate(gaugeSchema.metricBuilder.name): _*)))(f) } override def toString: String = s"$self/$namespacePrefix" diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala index c4924120f3..dcaf9423c4 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala @@ -13,7 +13,7 @@ class RelativeNameMarkingStatsReceiver( override def counter(counterSchema: CounterSchema): Counter = { val schema = CounterSchema( counterSchema.metricBuilder - .withRelativeName(counterSchema.metricBuilder.name)) + .withRelativeName(counterSchema.metricBuilder.name: _*)) self.counter(schema) } @@ -21,7 +21,7 @@ class RelativeNameMarkingStatsReceiver( val schema = HistogramSchema( histogramSchema.metricBuilder - .withRelativeName(histogramSchema.metricBuilder.name)) + .withRelativeName(histogramSchema.metricBuilder.name: _*)) self.stat(schema) } @@ -29,7 +29,7 @@ class RelativeNameMarkingStatsReceiver( val schema = GaugeSchema( gaugeSchema.metricBuilder - .withRelativeName(gaugeSchema.metricBuilder.name)) + .withRelativeName(gaugeSchema.metricBuilder.name: _*)) self.addGauge(schema)(f) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala index e932203353..c630a99cea 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala @@ -25,14 +25,14 @@ class RollupStatsReceiver(protected val self: StatsReceiver) extends StatsReceiv override def counter(schema: CounterSchema) = new Counter { private[this] val allCounters = BroadcastCounter( tails(schema.metricBuilder.name).map(n => - self.counter(CounterSchema(schema.metricBuilder.withName(n)))) + self.counter(CounterSchema(schema.metricBuilder.withName(n: _*)))) ) def incr(delta: Long): Unit = allCounters.incr(delta) } override def stat(schema: HistogramSchema) = new Stat { private[this] val allStats = BroadcastStat( tails(schema.metricBuilder.name).map(n => - self.stat(HistogramSchema(schema.metricBuilder.withName(n)))) + self.stat(HistogramSchema(schema.metricBuilder.withName(n: _*)))) ) def add(value: Float): Unit = allStats.add(value) } @@ -40,7 +40,7 @@ class RollupStatsReceiver(protected val self: StatsReceiver) extends StatsReceiv override def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { private[this] val underlying = tails(schema.metricBuilder.name).map(n => - self.addGauge(GaugeSchema(schema.metricBuilder.withName(n)))(f)) + self.addGauge(GaugeSchema(schema.metricBuilder.withName(n: _*)))(f)) def remove(): Unit = underlying.foreach(_.remove()) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index b7687f0fbc..ed70f083f5 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -120,7 +120,7 @@ trait StatsReceiver { */ @varargs def counter(verbosity: Verbosity, name: String*): Counter = - counter(CounterSchema(this.metricBuilder().withVerbosity(verbosity).withName(name))) + counter(CounterSchema(this.metricBuilder().withVerbosity(verbosity).withName(name: _*))) /** * Get a [[Counter counter]] with the given schema. @@ -138,7 +138,7 @@ trait StatsReceiver { */ @varargs def stat(verbosity: Verbosity, name: String*): Stat = - stat(HistogramSchema(this.metricBuilder().withVerbosity(verbosity).withName(name))) + stat(HistogramSchema(this.metricBuilder().withVerbosity(verbosity).withName(name: _*))) /** * Get a [[Stat stat]] with the given schema. @@ -213,7 +213,7 @@ trait StatsReceiver { * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = - addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name)))(f) + addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name: _*)))(f) /** * Just like $AddGaugeScaladocLink but optimized for better Java experience. @@ -226,7 +226,7 @@ trait StatsReceiver { */ @varargs def addGauge(f: Supplier[Float], verbosity: Verbosity, name: String*): Gauge = - addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name)))(f.get()) + addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name: _*)))(f.get()) /** * Add the function `f` as a [[Gauge gauge]] with the given name. diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java b/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java new file mode 100644 index 0000000000..6261e249bc --- /dev/null +++ b/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java @@ -0,0 +1,43 @@ +package com.twitter.finagle.stats; + +import org.junit.Test; + +import java.util.Arrays; + +import scala.Some; + +/** + * Java compatibility test for {@link com.twitter.finagle.stats.MetricBuilder}. + */ +public final class MetricBuilderCompilationTest { + @Test + public void testMetricBuilderConstruction() { + StatsReceiver sr = new InMemoryStatsReceiver(); + MetricBuilder mb = sr.metricBuilder(); + } + + @Test + public void testWithMethods() { + StatsReceiver sr = new InMemoryStatsReceiver(); + MetricBuilder mb = sr.metricBuilder() + .withKeyIndicator(true) + .withDescription("my cool metric") + .withVerbosity(Verbosity.Debug()) + .withSourceClass(new Some<>("com.twitter.finagle.AwesomeClass")) + .withIdentifier(new Some<>("/p/foo/bar")) + .withUnits(Microseconds.getInstance()) + .withRole(NoRoleSpecified.getInstance()) + .withName("my", "very", "cool", "name") + .withRelativeName("cool", "name") + .withPercentiles(0.99, 0.88, 0.77); + } + + @Test + public void testConstructingMetrics() { + StatsReceiver sr = new InMemoryStatsReceiver(); + MetricBuilder mb = sr.metricBuilder(); + Gauge g = mb.gauge(() -> 3.0f, "my", "cool", "gauge"); + Stat s = mb.histogram("my", "cool", "histogram"); + Counter c = mb.counter("my", "cool", "counter"); + } +} diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java index f06579c87b..b8ef5b3c3d 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java @@ -12,7 +12,7 @@ import scala.collection.immutable.Seq; /** - * Java compatibility layer for {@link com.twitter.finagle.stats.StatsReceiver}. + * Java compatibility test for {@link com.twitter.finagle.stats.StatsReceiver}. */ public final class StatsReceiverCompilationTest { From 47d815de960d1da54f949eff17c4ea2db8461811 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 4 Mar 2021 22:09:45 +0000 Subject: [PATCH 441/796] Twitter-oss: Remove TravisCI configuration for projects Problem/Solution We have updated to using GitHub workflows for continuous integration. Let's remove the TravisCI configuration. JIRA Issues: CSL-8182 Differential Revision: https://phabricator.twitter.biz/D628740 --- .travis.yml | 50 -------------------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f932960712..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,50 +0,0 @@ -dist: trusty -sudo: required - -language: scala - -git: - depth: 3 - quiet: true - -env: - - JAVA_OPTS="-Dsbt.log.noformat=true" - -# These directories are cached to S3 at the end of the build -cache: - directories: - - $HOME/.ivy2/cache - - $HOME/.ivy2/local/com.twitter - - $HOME/.sbt - - $HOME/.dodo - -before_cache: - # Cleanup the cached directories to avoid unnecessary cache updates - - find $HOME/.ivy2/cache -name "ivydata-*.properties" -delete - - find $HOME/.sbt -name "*.lock" -delete - -scala: - - 2.11.12 - - 2.12.8 - - 2.13.1 - -jdk: - - openjdk8 - - openjdk11 - -notifications: - slack: - secure: BFX2aB0sq5cMxmxlWIwgHwJOjI5W6WBn3YSPqHYYz6VAK6fzlX2VUKsfyQJTUnqMYox/0qGzPUB4TFhVMH3c78ocNZNj/9tPOPLkMfHjAqobnq/89+hp4Q6fmWCMKVAGTzVNq83OCykYkzITcYWxo0Pyc2eguW6tHaHIgqnbCRY= - on_failure: change - on_success: change - -before_script: - - unset SBT_OPTS # default $SBT_OPTS is irrelevant to sbt launcher - -script: - - travis_retry ./sbt ++$TRAVIS_SCALA_VERSION clean coverage test coverageReport - - if [[ "$TRAVIS_SCALA_VERSION" == 2.11* ]]; then travis_retry ./sbt ++$TRAVIS_SCALA_VERSION "project util-intellij" coverage test coverageReport; fi - -after_success: - - ./sbt ++$TRAVIS_SCALA_VERSION coverageAggregate - - bash <(curl -s https://codecov.io/bash) From 35aa206d1ffea9f784db7e40e6f1989a6519ff8d Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 8 Mar 2021 20:10:08 +0000 Subject: [PATCH 442/796] {finagle, util, twitter-server}: Metric Expression prototype Problem We want to ship our prototype of Metric Expression to start the development, this prototype confirms that end-to-end the expression instrument and exports work as we expected. Solution The current prototype code is sitting in experiment packages, the exported expression json document is under-development. JIRA Issues: CSL-10616 Differential Revision: https://phabricator.twitter.biz/D615453 --- .../scala/com/twitter/finagle/stats/BUILD | 1 + .../stats/BroadcastStatsReceiver.scala | 10 +++ .../finagle/stats/ExceptionStatsHandler.scala | 2 +- .../finagle/stats/InMemoryStatsReceiver.scala | 7 +++ .../stats/RoleConfiguredStatsReceiver.scala | 2 +- .../finagle/stats/SchemaRegistry.scala | 5 ++ .../twitter/finagle/stats/StatsReceiver.scala | 3 + .../finagle/stats/StatsReceiverProxy.scala | 5 ++ .../twitter/finagle/stats/exp/Bounds.scala | 15 +++++ .../finagle/stats/exp/Expression.scala | 44 +++++++++++++ .../finagle/stats/exp/ExpressionLabels.scala | 11 ++++ .../finagle/stats/exp/ExpressionSchema.scala | 34 ++++++++++ .../twitter/finagle/stats/exp/Operator.scala | 15 +++++ .../stats/InMemoryStatsReceiverTest.scala | 11 +++- .../finagle/stats/exp/ExprEndToEndTest.scala | 63 +++++++++++++++++++ 15 files changed, 225 insertions(+), 3 deletions(-) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/exp/Bounds.scala create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/exp/Operator.scala create mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD index 0ba5e8cbc2..fbd53de60c 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD @@ -1,4 +1,5 @@ scala_library( + sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], provides = scala_artifact( org = "com.twitter", diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala index fd5ba7043e..29c4561bf8 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala @@ -1,5 +1,7 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.exp.ExpressionSchema + /** * BroadcastStatsReceiver is a helper object that create a StatsReceiver wrapper around multiple * StatsReceivers (n). @@ -36,6 +38,11 @@ object BroadcastStatsReceiver { } } + override protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = { + first.registerExpression(expressionSchema) + second.registerExpression(expressionSchema) + } + def underlying: Seq[StatsReceiver] = Seq(first, second) override def toString: String = @@ -56,6 +63,9 @@ object BroadcastStatsReceiver { def remove(): Unit = gauges.foreach { _.remove() } } + override protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = + srs.map(_.registerExpression(expressionSchema)) + def underlying: Seq[StatsReceiver] = srs override def toString: String = diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala index 53a8b05bda..46da54d10a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala @@ -10,7 +10,7 @@ import com.twitter.util.Throwables * @see [[Null]] for a no-op handler. */ object ExceptionStatsHandler { - private[stats] val Failures = "failures" + private[finagle] val Failures = "failures" private[stats] val SourcedFailures = "sourcedfailures" /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index fabbf95908..75437a2ed7 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -1,5 +1,6 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.exp.ExpressionSchema import java.io.PrintStream import java.util.Locale import java.util.concurrent.ConcurrentHashMap @@ -69,6 +70,9 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { val schemas: mutable.Map[Seq[String], MetricSchema] = new ConcurrentHashMap[Seq[String], MetricSchema]().asScala + val expressions: mutable.Map[String, ExpressionSchema] = + new ConcurrentHashMap[String, ExpressionSchema]().asScala + override def counter(name: String*): ReadableCounter = counter(CounterSchema(this.metricBuilder().withName(name: _*))) @@ -241,6 +245,9 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { def histogramDetails: Map[String, HistogramDetail] = stats.toMap.map { case (k, v) => (k.mkString("/"), toHistogramDetail(v)) } + + override protected[finagle] def registerExpression(schema: ExpressionSchema): Unit = + expressions(schema.name) = schema } /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala index 65d9d76f52..99a7e99870 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala @@ -9,7 +9,7 @@ package com.twitter.finagle.stats */ case class RoleConfiguredStatsReceiver( protected val self: StatsReceiver, - val role: SourceRole) + role: SourceRole) extends StatsReceiverProxy { override def counter(counterSchema: CounterSchema): Counter = { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala index cb0344cde7..2f3a1ba64a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala @@ -1,5 +1,7 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.exp.ExpressionSchema + /** * Interface used via the LoadService mechanism to obtain an * efficient mechanism to sample stats. @@ -10,4 +12,7 @@ private[twitter] trait SchemaRegistry { def hasLatchedCounters: Boolean def schemas(): Map[String, MetricSchema] + + def expressions(): Map[String, ExpressionSchema] + } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index ed70f083f5..92986e7aa0 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -1,5 +1,6 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.exp.ExpressionSchema import java.lang.{Float => JFloat} import java.util.concurrent.Callable import java.util.function.Supplier @@ -312,6 +313,8 @@ trait StatsReceiver { final def scope(namespaces: String*): StatsReceiver = namespaces.foldLeft(this)((statsReceiver, name) => statsReceiver.scope(name)) + protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = {} + /** * Prepend a suffix value to the next scope. * diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala index d801ea09d2..a4311b72fd 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala @@ -1,5 +1,7 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.exp.ExpressionSchema + /** * A proxy [[StatsReceiver]] that delegates all calls to the `self` stats receiver. */ @@ -13,6 +15,9 @@ trait StatsReceiverProxy extends StatsReceiver with DelegatingStatsReceiver { def stat(schema: HistogramSchema): Stat = self.stat(schema) def addGauge(schema: GaugeSchema)(f: => Float): Gauge = self.addGauge(schema)(f) + override protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = + self.registerExpression(expressionSchema) + def underlying: Seq[StatsReceiver] = Seq(self) override def isNull: Boolean = self.isNull diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Bounds.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Bounds.scala new file mode 100644 index 0000000000..396b85b023 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Bounds.scala @@ -0,0 +1,15 @@ +package com.twitter.finagle.stats.exp + +sealed trait Bounds + +case class AcceptableRange(lowerBoundInclusive: Double, upperBoundExclusive: Double) extends Bounds + +case class MonotoneThreshold( + operator: Operator, + badThreshold: Double, + goodThreshold: Double, + lowerBoundInclusive: Option[Double] = None, + upperBoundExclusive: Option[Double] = None) + extends Bounds + +case object Unbounded extends Bounds diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala new file mode 100644 index 0000000000..985f270e8d --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -0,0 +1,44 @@ +package com.twitter.finagle.stats.exp + +import com.twitter.finagle.stats.{MetricSchema, StatsReceiver} +import scala.annotation.varargs + +private[twitter] sealed trait Expression { + def plus(other: Expression): Expression = FunctionExpression("plus", Seq(this, other)) + + def minus(other: Expression): Expression = FunctionExpression("minus", Seq(this, other)) + + def divide(other: Expression): Expression = FunctionExpression("divide", Seq(this, other)) + + def multiply(other: Expression): Expression = + FunctionExpression("multiply", Seq(this, other)) + + @varargs + def func(name: String, rest: Expression*): Expression = + FunctionExpression(name, this +: rest) +} + +private[twitter] object Expression { + def getStatsReceivers(expr: Expression): Set[StatsReceiver] = expr match { + case FunctionExpression(_, exprs) => + exprs.foldLeft(Set.empty[StatsReceiver]) { + case (acc, expr) => + getStatsReceivers(expr) ++ acc + } + case MetricExpression(schema) => Set(schema.metricBuilder.statsReceiver) + case ConstantExpression(_, statsReceiver) => Set(statsReceiver) + } + def apply(num: Double, statsReceiver: StatsReceiver): Expression = + ConstantExpression(num.toString, statsReceiver) + def apply(schema: MetricSchema): Expression = MetricExpression(schema) +} + +private[twitter] case class ConstantExpression(repr: String, statsReceiver: StatsReceiver) + extends Expression + +private[twitter] case class FunctionExpression(fnName: String, exprs: Seq[Expression]) + extends Expression { + require(exprs.size != 0, "Functions must have at least 1 argument") +} + +private[twitter] case class MetricExpression(schema: MetricSchema) extends Expression diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala new file mode 100644 index 0000000000..f176cc1c30 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala @@ -0,0 +1,11 @@ +package com.twitter.finagle.stats.exp + +import com.twitter.finagle.stats.{NoRoleSpecified, SourceRole} + +private[exp] object ExpressionLabels { + val empty = ExpressionLabels(None, None, NoRoleSpecified) +} +private[exp] case class ExpressionLabels( + processPath: Option[String], + serviceName: Option[String], + role: SourceRole) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala new file mode 100644 index 0000000000..c1d7a4b0a9 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -0,0 +1,34 @@ +package com.twitter.finagle.stats.exp + +import com.twitter.finagle.stats.{MetricUnit, SourceRole, StatsReceiver, Unspecified} + +case class ExpressionSchema private ( + name: String, + labels: ExpressionLabels, + expr: Expression, + bounds: Bounds, + description: String, + unit: MetricUnit) { + def withBounds(bounds: Bounds): ExpressionSchema = copy(bounds = bounds) + + def withDescription(description: String): ExpressionSchema = copy(description = description) + + def withUnit(unit: MetricUnit): ExpressionSchema = copy(unit = unit) + + private[finagle] def withRole(role: SourceRole): ExpressionSchema = + copy(labels = labels.copy(role = role)) + + def register(): Unit = { + Expression.getStatsReceivers(expr).toSeq match { + case Seq(sr) => sr.registerExpression(this) + case srs: Seq[StatsReceiver] if srs.nonEmpty => srs.map(_.registerExpression(this)) + case _ => // should not happen + } + } +} + +// expose for testing in twitter-server +private[twitter] object ExpressionSchema { + def apply(name: String, expr: Expression): ExpressionSchema = + ExpressionSchema(name, ExpressionLabels.empty, expr, Unbounded, "Unspecified", Unspecified) +} diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Operator.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Operator.scala new file mode 100644 index 0000000000..ff23068b09 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Operator.scala @@ -0,0 +1,15 @@ +package com.twitter.finagle.stats.exp + +sealed trait Operator { + def isIncreasing: Boolean +} + +case object GreaterThan extends Operator { + override def toString: String = ">" + val isIncreasing: Boolean = true +} + +case object LessThan extends Operator { + override def toString: String = "<" + val isIncreasing: Boolean = false +} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 7f6201d8d4..b143c6bdcb 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -1,10 +1,11 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema} import java.io.{ByteArrayOutputStream, PrintStream} import java.nio.charset.StandardCharsets import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import scala.collection.parallel.immutable.ParRange import org.scalatest.funsuite.AnyFunSuite +import scala.collection.parallel.immutable.ParRange class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with IntegrationPatience { @@ -119,6 +120,14 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(stats.verbosity(Seq("baz")) == Verbosity.Debug) } + test("register expressions") { + val stats = new InMemoryStatsReceiver() + ExpressionSchema( + "a", + Expression(CounterSchema(new MetricBuilder(name = Seq("counter"), statsReceiver = stats)))) + stats.expressions.contains("a") + } + test("print") { val baos = new ByteArrayOutputStream() val ps = new PrintStream(baos, true, "utf-8") diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala new file mode 100644 index 0000000000..5be66171ac --- /dev/null +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala @@ -0,0 +1,63 @@ +package com.twitter.finagle.stats.exp + +import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats._ +import com.twitter.util.{Stopwatch, Time, TimeControl} +import org.scalatest.FunSuite +import scala.util.control.NonFatal + +class ExprEndToEndTest extends FunSuite { + def mySlowQuery(ctl: TimeControl, succeed: Boolean): Unit = { + ctl.advance(50.milliseconds) + if (!succeed) { + throw new Exception("boom!") + } + } + + test("Demonstrate an end to end example of using metrics") { + val sr = new InMemoryStatsReceiver + val successMb = CounterSchema(new MetricBuilder(name = Seq("success"), statsReceiver = sr)) + val failuresMb = + CounterSchema(new MetricBuilder(name = Seq("failures"), statsReceiver = sr)) + val latencyMb = HistogramSchema(new MetricBuilder(name = Seq("latency"), statsReceiver = sr)) + val sum = Expression(successMb).plus(Expression(failuresMb)) + val successCounter = sr.counter(successMb) + val failuresCounter = sr.counter(failuresMb) + val latencyStat = sr.stat(latencyMb) + val successRate = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) + .withBounds(MonotoneThreshold(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query") + + val latency = ExpressionSchema("latency", Expression(latencyMb)) + .withUnit(Milliseconds) + .withDescription("The latency of the slow query") + + successRate.register() + latency.register() + + def runTheQuery(succeed: Boolean): Unit = { + Time.withCurrentTimeFrozen { ctl => + val elapsed = Stopwatch.start() + try { + mySlowQuery(ctl, succeed) + successCounter.incr() + } catch { + case NonFatal(exn) => + failuresCounter.incr() + } finally { + latencyStat.add(elapsed().inMilliseconds) + } + } + } + runTheQuery(true) + runTheQuery(false) + + assert(sr.expressions("success_rate") == successRate) + assert(sr.expressions("latency") == latency) + + assert(sr.counters(Seq("success")) == 1) + assert(sr.counters(Seq("failures")) == 1) + assert(sr.stats(Seq("latency")) == Seq(50, 50)) + } +} From 31eb5fa75c2694718c35a5dcb6c80535339843a4 Mon Sep 17 00:00:00 2001 From: Joseph Boyd Date: Tue, 9 Mar 2021 01:49:22 +0000 Subject: [PATCH 443/796] Scrooge: Fix a typo in 'asigned' in scrooge gen Problem The scrooge generator templates have a comment typo them Solution Correct the templates and corresponding 'gold' test files Differential Revision: https://phabricator.twitter.biz/D625392 --- .../src/test/java/com/twitter/util/TestThriftStructure.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util-thrift/src/test/java/com/twitter/util/TestThriftStructure.java b/util-thrift/src/test/java/com/twitter/util/TestThriftStructure.java index 5decf718a7..1ec5e84e87 100644 --- a/util-thrift/src/test/java/com/twitter/util/TestThriftStructure.java +++ b/util-thrift/src/test/java/com/twitter/util/TestThriftStructure.java @@ -159,7 +159,7 @@ public void unsetAString() { this.aString = null; } - /** Returns true if field aString is set (has been asigned a value) and false otherwise */ + /** Returns true if field aString is set (has been assigned a value) and false otherwise */ public boolean isSetAString() { return this.aString != null; } @@ -184,7 +184,7 @@ public void unsetANumber() { __isset_bit_vector.clear(__ANUMBER_ISSET_ID); } - /** Returns true if field aNumber is set (has been asigned a value) and false otherwise */ + /** Returns true if field aNumber is set (has been assigned a value) and false otherwise */ public boolean isSetANumber() { return __isset_bit_vector.get(__ANUMBER_ISSET_ID); } @@ -226,7 +226,7 @@ public Object getFieldValue(_Fields field) { throw new IllegalStateException(); } - /** Returns true if field corresponding to fieldID is set (has been asigned a value) and false otherwise */ + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ public boolean isSet(_Fields field) { if (field == null) { throw new IllegalArgumentException(); From 62a3169da0107aeec6205d0e1bbe49addc22d7cd Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 9 Mar 2021 23:49:43 +0000 Subject: [PATCH 444/796] util-reflect: Remove deprecated `c.t.u.reflect.Proxy` Problem/Solution Remove deprecated `c.t.util.reflect.Proxy`. There is no library replacement. Differential Revision: https://phabricator.twitter.biz/D630143 --- CHANGELOG.rst | 8 +- build.sbt | 8 +- util-reflect/BUILD | 5 - .../main/scala/com/twitter/util/reflect/BUILD | 4 - .../com/twitter/util/reflect/Proxy.scala | 126 ---------- util-reflect/src/test/scala/BUILD | 5 - .../test/scala/com/twitter/util/reflect/BUILD | 12 - .../com/twitter/util/reflect/ProxyTest.scala | 230 ------------------ 8 files changed, 8 insertions(+), 390 deletions(-) delete mode 100644 util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala delete mode 100644 util-reflect/src/test/scala/BUILD delete mode 100644 util-reflect/src/test/scala/com/twitter/util/reflect/BUILD delete mode 100644 util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 93f8537b12..c1aea418cf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-reflect: Remove deprecated `c.t.util.reflect.Proxy`. There is no library replacement. + ``PHAB_ID=D630143`` + New Features ~~~~~~~~~~~~ @@ -64,7 +70,7 @@ New Features * util-core: `c.t.util.Duration` now includes `fromJava` and `asJava` conversions to `java.time.Duration` types. ``PHAB_ID=D571885`` - Runtime Behavior Changes +Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ * util-core: `Activity.apply(Event)` will now propagate registry events to the underlying diff --git a/build.sbt b/build.sbt index b3637707f6..399a07ec0f 100644 --- a/build.sbt +++ b/build.sbt @@ -357,13 +357,7 @@ lazy val utilReflect = Project( ).settings( sharedSettings ).settings( - name := "util-reflect", - libraryDependencies ++= Seq( - "asm" % "asm" % "3.3.1", - "asm" % "asm-util" % "3.3.1", - "asm" % "asm-commons" % "3.3.1", - "cglib" % "cglib" % "2.2.2" - ) + name := "util-reflect" ).dependsOn(utilCore) lazy val utilHashing = Project( diff --git a/util-reflect/BUILD b/util-reflect/BUILD index 27d6eb2eb0..ee63e07e1c 100644 --- a/util-reflect/BUILD +++ b/util-reflect/BUILD @@ -1,8 +1,4 @@ target( - tags = [ - # Legacy code that shouldn't be used for new services. - "deprecated", - ], dependencies = [ "util/util-reflect/src/main/scala", ], @@ -11,6 +7,5 @@ target( test_suite( name = "tests", dependencies = [ - "util/util-reflect/src/test/scala", ], ) diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD index 5f40d5e8d0..a98d2a0595 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD @@ -6,10 +6,6 @@ scala_library( repo = artifactory, ), dependencies = [ - "3rdparty/jvm/cglib", "util/util-core:util-core-util", ], - exports = [ - "3rdparty/jvm/cglib", - ], ) diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala deleted file mode 100644 index 50c6b164d0..0000000000 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/Proxy.scala +++ /dev/null @@ -1,126 +0,0 @@ -package com.twitter.util.reflect - -import java.io.Serializable -import java.lang.reflect.Method - -import net.sf.cglib.proxy.{MethodInterceptor => CGMethodInterceptor, _} - -import com.twitter.util.Future - -class NonexistentTargetException extends Exception("MethodCall was invoked without a valid target.") - -object Proxy { - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply[I <: AnyRef: Manifest](f: MethodCall[I] => AnyRef): I = { - new ProxyFactory[I](f).apply() - } - - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply[I <: AnyRef: Manifest](target: I, f: MethodCall[I] => AnyRef): I = { - new ProxyFactory[I](f).apply(target) - } -} - -object ProxyFactory { - private[reflect] object NoOpInterceptor extends MethodInterceptor[AnyRef](None, m => null) - - private[reflect] object IgnoredMethodFilter extends CallbackFilter { - def accept(m: Method): Int = { - m.getName match { - case "hashCode" => 1 - case "equals" => 1 - case "toString" => 1 - case _ => 0 - } - } - } -} - -class AbstractProxyFactory[I <: AnyRef: Manifest] { - import ProxyFactory._ - - final val interface: Class[_] = implicitly[Manifest[I]].runtimeClass - - protected final val proto: Factory = { - val e = new Enhancer - e.setCallbackFilter(IgnoredMethodFilter) - e.setCallbacks(Array(NoOpInterceptor, NoOp.INSTANCE)) - e.setSuperclass(interface) - e.create.asInstanceOf[Factory] - } - - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - protected final def newWithCallback(f: MethodCall[I] => AnyRef): I = { - proto.newInstance(Array(new MethodInterceptor(None, f), NoOp.INSTANCE)).asInstanceOf[I] - } - - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - protected final def newWithCallback[T <: I](target: T, f: MethodCall[I] => AnyRef): I = { - proto.newInstance(Array(new MethodInterceptor(Some(target), f), NoOp.INSTANCE)).asInstanceOf[I] - } -} - -class ProxyFactory[I <: AnyRef: Manifest](f: MethodCall[I] => AnyRef) - extends AbstractProxyFactory[I] { - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply[T <: I](target: T): I = newWithCallback(target, f) - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply(): I = newWithCallback(f) -} - -private[reflect] class MethodInterceptor[I <: AnyRef]( - target: Option[I], - callback: MethodCall[I] => AnyRef) - extends CGMethodInterceptor - with Serializable { - val targetRef: I = target.getOrElse(null).asInstanceOf[I] - - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - final def intercept( - p: AnyRef, - m: Method, - args: Array[AnyRef], - methodProxy: MethodProxy - ): Object = { - callback(new MethodCall(targetRef, m, args, methodProxy)) - } -} - -final class MethodCall[T <: AnyRef] private[reflect] ( - targetRef: T, - val method: Method, - val args: Array[AnyRef], - methodProxy: MethodProxy) - extends (() => AnyRef) { - - lazy val target: Option[T] = if (targetRef ne null) Some(targetRef) else None - - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def clazz: Class[_] = method.getDeclaringClass - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def clazzName: String = clazz.getName - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def className: String = clazzName - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def parameterTypes: Array[Class[_]] = method.getParameterTypes - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def name: String = method.getName - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def returnsUnit: Boolean = { - val rt = method.getReturnType - (rt eq classOf[Unit]) || (rt eq classOf[Null]) || (rt eq java.lang.Void.TYPE) - } - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def returnsFuture: Boolean = classOf[Future[_]] isAssignableFrom method.getReturnType - - private def getTarget = if (targetRef ne null) targetRef else throw new NonexistentTargetException - - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply(): AnyRef = methodProxy.invoke(getTarget, args) - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply(newTarget: T): Object = methodProxy.invoke(newTarget, args) - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply(newArgs: Array[AnyRef]): Object = methodProxy.invoke(getTarget, newArgs) - @deprecated("Legacy code that shouldn't be used for new services", "2018-05-25") - def apply(newTarget: T, newArgs: Array[AnyRef]): Object = methodProxy.invoke(newTarget, newArgs) -} diff --git a/util-reflect/src/test/scala/BUILD b/util-reflect/src/test/scala/BUILD deleted file mode 100644 index f0543a5893..0000000000 --- a/util-reflect/src/test/scala/BUILD +++ /dev/null @@ -1,5 +0,0 @@ -test_suite( - dependencies = [ - "util/util-reflect/src/test/scala/com/twitter/util/reflect", - ], -) diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD deleted file mode 100644 index 2d3efd5a8c..0000000000 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -junit_tests( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], - dependencies = [ - "3rdparty/jvm/junit", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "util/util-core:util-core-util", - "util/util-reflect/src/main/scala/com/twitter/util/reflect", - ], -) diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala deleted file mode 100644 index cb5ce488c5..0000000000 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/ProxyTest.scala +++ /dev/null @@ -1,230 +0,0 @@ -package com.twitter.util.reflect - -import com.twitter.util.{Future, Promise, Stopwatch} -import org.scalatest.wordspec.AnyWordSpec - -object ProxySpec { - trait TestInterface { - def foo: String - def bar(a: Int): Option[Long] - def whoops: Int - def theVoid(): Unit - def theJavaVoid: java.lang.Void - def aFuture: Future[Int] - def aPromise: Promise[Int] - } - - class TestImpl extends TestInterface { - def foo = "foo" - def bar(a: Int) = Some(a.toLong) - def whoops = if (false) 2 else sys.error("whoops") - def theVoid() = () - def theJavaVoid = null - def aFuture = Future.value(2) - def aPromise = new Promise[Int] { - setValue(2) - } - } - - class TestClass { - val slota = "a" - val slotb = 2.0 - } -} - -class ProxyTest extends AnyWordSpec { - - import ProxySpec._ - - "ProxyFactory" should { - "generate a factory for an interface" in { - var called = 0 - - val f1 = new ProxyFactory[TestInterface]({ m => called += 1; m() }) - val obj = new TestImpl - - val proxied = f1(obj) - - assert(proxied.foo == "foo") - assert(proxied.bar(2) == Some(2L)) - - assert(called == 2) - } - - "generate a factory for a class with a default constructor" in { - var called = 0 - - val f2 = new ProxyFactory[TestClass]({ m => called += 1; m() }) - val obj = new TestClass - - val proxied = f2(obj) - - assert(proxied.slota == "a") - assert(proxied.slotb == 2.0) - - assert(called == 2) - } - - "must not throw UndeclaredThrowableException" in { - val pf = new ProxyFactory[TestImpl](_()) - val proxied = pf(new TestImpl) - - intercept[RuntimeException] { - proxied.whoops - } - } - - "MethodCall returnsUnit must be true for unit/void methods" in { - var unitsCalled = 0 - - val pf = new ProxyFactory[TestImpl]({ call => - if (call.returnsUnit) unitsCalled += 1 - call() - }) - - val proxied = pf(new TestImpl) - proxied.foo - assert(unitsCalled == 0) - - proxied.theVoid() - assert(unitsCalled == 1) - - proxied.theJavaVoid - assert(unitsCalled == 2) - } - - "MethodCall returnsFuture must be true for methods that return a future or subclass" in { - var futuresCalled = 0 - - val pf = new ProxyFactory[TestImpl]({ call => - if (call.returnsFuture) futuresCalled += 1 - call() - }) - - val proxied = pf(new TestImpl) - - proxied.foo - assert(futuresCalled == 0) - - proxied.aFuture - assert(futuresCalled == 1) - - proxied.aPromise - assert(futuresCalled == 2) - } - - "MethodCall throws an exception when invoked without a target" in { - val pf = new ProxyFactory[TestImpl](_()) - val targetless = pf() - - intercept[NonexistentTargetException] { - targetless.foo - } - } - - "MethodCall can be invoked with alternate target" in { - val alt = new TestImpl { override def foo = "alt foo" } - val pf = new ProxyFactory[TestImpl](m => m(alt)) - val targetless = pf() - - assert(targetless.foo == "alt foo") - } - - "MethodCall can be invoked with alternate arguments" in { - val pf = new ProxyFactory[TestInterface](m => m(Array(3.asInstanceOf[AnyRef]))) - val proxied = pf(new TestImpl) - - assert(proxied.bar(2) == Some(3L)) - } - - "MethodCall can be invoked with alternate target and arguments" in { - val alt = new TestImpl { override def bar(i: Int) = Some(i.toLong * 10) } - val pf = new ProxyFactory[TestImpl](m => m(alt, Array(3.asInstanceOf[AnyRef]))) - val targetless = pf() - - assert(targetless.bar(2) == Some(30L)) - } - - // Sigh. Benchmarking has no place in unit tests. - - def time(f: => Unit) = { val elapsed = Stopwatch.start(); f; elapsed().inMilliseconds } - def benchmark(f: => Unit) = { - // warm up - for (i <- 1 to 10) f - - // measure - val results = for (i <- 1 to 7) yield time(f) - //println(results) - - // drop the top and bottom then average - results.sortWith((a, b) => a > b).slice(2, 5).sum / 3 - } - - val reflectConstructor = { () => new ReferenceProxyFactory[TestInterface](_()) } - val cglibConstructor = { () => new ProxyFactory[TestInterface](_()) } - /* - "maintains proxy creation speed" in { - val repTimes = 40000 - - val obj = new TestImpl - val factory1 = reflectConstructor() - val factory2 = cglibConstructor() - - val t1 = benchmark { for (i <- 1 to repTimes) factory1(obj) } - val t2 = benchmark { for (i <- 1 to repTimes) factory2(obj) } - - // println("proxy creation") - // println(t1) - // println(t2) - - t2 should beLessThan(200L) - } - */ - "maintains invocation speed" ignore { - val repTimes = 1500000 - - val obj = new TestImpl - val proxy1 = reflectConstructor()(obj) - val proxy2 = cglibConstructor()(obj) - - val t1 = benchmark { for (i <- 1 to repTimes) { proxy1.foo; proxy1.bar(2) } } - val t2 = benchmark { for (i <- 1 to repTimes) { proxy2.foo; proxy2.bar(2) } } - val t3 = benchmark { for (i <- 1 to repTimes) { obj.foo; obj.bar(2) } } - - // println("proxy invocation") - // println(t1) - // println(t2) - // println(t3) - - // faster than normal reflection - assert(t2 < t1) - - // less than 4x raw invocation - assert(t2 < t3 * 4) - } - } - - class ReferenceProxyFactory[I <: AnyRef: Manifest](f: (() => AnyRef) => AnyRef) { - import java.lang.reflect - - protected val interface = implicitly[Manifest[I]].runtimeClass - - private val proxyConstructor = { - reflect.Proxy - .getProxyClass(interface.getClassLoader, interface) - .getConstructor(classOf[reflect.InvocationHandler]) - } - - def apply[T <: I](instance: T) = { - proxyConstructor - .newInstance(new reflect.InvocationHandler { - def invoke(p: AnyRef, method: reflect.Method, args: Array[AnyRef]) = { - try { - f.apply(() => method.invoke(instance, args: _*)) - } catch { case e: reflect.InvocationTargetException => throw e.getCause } - } - }) - .asInstanceOf[I] - } - } -} From b6b00033ceaf2b1c1c73f7b7286a5890a525d170 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Thu, 11 Mar 2021 21:27:03 +0000 Subject: [PATCH 445/796] util: remove util-intellij capture points Problem: We're dropping support for scala 2.11, but util-intellij capture points only work with that scala version. Solution: Remove util-intellij capture points JIRA Issues: CSL-10599 Differential Revision: https://phabricator.twitter.biz/D632095 --- build.sbt | 10 -- doc/src/sphinx/util-capturepoints/index.rst | 107 ------------------ util-intellij/PROJECT | 9 -- util-intellij/README.md | 103 ----------------- util-intellij/src/main/resources/BUILD | 3 - .../TwitterFuturesCapturePoints.xml | 54 --------- .../com/twitter/util/capturepoints/BUILD | 13 --- .../com/twitter/util/capturepoints/Demo.scala | 56 --------- .../TwitterFuturesCapturePointsTest.scala | 62 ---------- 9 files changed, 417 deletions(-) delete mode 100644 doc/src/sphinx/util-capturepoints/index.rst delete mode 100644 util-intellij/PROJECT delete mode 100644 util-intellij/README.md delete mode 100644 util-intellij/src/main/resources/BUILD delete mode 100644 util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml delete mode 100644 util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD delete mode 100644 util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala delete mode 100644 util-intellij/src/test/scala/com/twitter/util/capturepoints/TwitterFuturesCapturePointsTest.scala diff --git a/build.sbt b/build.sbt index 399a07ec0f..16f076038e 100644 --- a/build.sbt +++ b/build.sbt @@ -373,16 +373,6 @@ lazy val utilHashing = Project( ) ).dependsOn(utilCore % "test") -lazy val utilIntellij = Project( - id = "util-intellij", - base = file("util-intellij") -).settings( - baseSettings - ).settings( - name := "util-intellij", - scalaVersion := "2.11.12" - ).dependsOn(utilCore % "test") - lazy val utilJvm = Project( id = "util-jvm", base = file("util-jvm") diff --git a/doc/src/sphinx/util-capturepoints/index.rst b/doc/src/sphinx/util-capturepoints/index.rst deleted file mode 100644 index ca2f658195..0000000000 --- a/doc/src/sphinx/util-capturepoints/index.rst +++ /dev/null @@ -1,107 +0,0 @@ -Twitter Futures Capture Points -============================== - -Stack traces produced from asynchronous execution have undesirable -properties, e.g., they are disorganized and often contain frames from -internal libraries that programmers need not know about. This result is -inevitable since not only do different parts of the same program get -executed on different threads, and possibly different cores, but -execution also jumps between different frames. This means that each -thread used to execute a program may produce a unique stack trace. Thus, -an Asynchronous Stack Trace (AST) gives a fragmented window view into -the execution of a program and thus makes it difficult to understand the -flow of code because the causality between frames is lost. - -In response to this phenomenon, IntelliJ 2017.1 introduced a feature -called Capture Points which is built on top of the IntelliJ Debugger. A -capture point is a place in a computer program where the debugger -collects and saves stack frames to be used later when we reach a -specific point in the code and we want to see how we got there. IntelliJ -IDEA does this by substituting part of the call stack with the captured -frame. - -We have written capture points for Twitter Futures in an XML file. Users can -debug their asynchronous code more easily with these capture points. Note that -as of October 2017, the capture points work with only Scala 2.11.11. - -Setup -^^^^^ - -The capture points are defined in -``util/util-intellij/src/main/resources/TwitterFuturesCapturePoints.xml``. -To import them into IntelliJ, - -1. Open IntelliJ. In the menu bar, click IntelliJ IDEA > Preferences. -2. Navigate to Build, Execution, Deployment > Debugger > Async - Stacktraces. -3. Click the Import icon on the bottom bar. Find the XML file and click - OK. - -Use -^^^ - -TL;DR: set a breakpoint where you wish to see the stack trace, debug -your code, and look at the "Frames" tab in the Debugger. Any -asynchronous calls in the stack trace will appear in logical order. If -you wish to clean up the stack trace, click the funnel icon in the top -right, "Hide Frames from Libraries". - -We will illustrate how to use the capture points to assist with -debugging with a small example. The example is located in -``util/util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala``. - -A brief explanation of this test: the test passes a ``Promise[Int]`` -through three methods and then sets the ``Promise``\ ’s value in a -``futurePool``. The calls are asynchronous, but the logical flow of the -test is as follows: - -1. ``test`` block calls ``someBusinessLogic`` -2. ``someBusinessLogic`` calls ``moreBusinessLogic`` -3. ``moreBusinessLogic`` calls ``lordBusinessLogic`` -4. ``lordBusinessLogic`` waits for the ``Promise``\ ’s value to be set -5. The ``Promise``\ ’s value is set in the test block (this could happen - at any time; it is not necessarily step number 5) -6. ``lordBusinessLogic`` returns -7. ``test``\ ’s ``result`` variable is ``4``, and the test passes - -Suppose we wish to inspect the stack trace from inside -``lordBusinessLogic``. If we set a breakpoint at line 47 and then debug -the test with pants in IntelliJ with the capture points disabled, we see -the following stack trace: - -:: - - apply$mcVI$sp:47, Demo$$anonfun$lordBusinessLogic$1 - apply:1820, Future$$anonfun$onSuccess$1 - apply:205, Promise$Monitored - run:532, Promise$$anon$7 - run:198, LocalScheduler$Activation - submit:157, LocalScheduler$Activation - submit:274, LocalScheduler - submit:109, Scheduler$ - runq:522, Promise - updatelfEmpty:880, Promise - update:859, Promise - setValue:835, Promise - apply$mcV$sp:20, Demo$$anonfun$3$$anonfun$apply$1 - apply:15, Try$ - run:140, ExecutorServiceFuturePool$$anon$4 - -The library calls in the stack trace do not help us understand our code. -If we enable the capture points and again debug the test, we see the -following stack trace: - -:: - - apply$mcVI$sp:47, Demo$$anonfun$lordBusinessLogic$1 - apply:1820, Future$$anonfun$onSuccess$1 - onSuccess:1819, Future - lordBusinessLogic:42, Demo - moreBusinessLogic:38, Demo - someBusinessLogic:31, Demo - apply:17, Demo$$anonfun$3 - -This is much cleaner and it includes only calls to functions we wrote. -It lets us clearly see that the test block called ``someBusinessLogic``, -that ``someBusinessLogic`` called ``moreBusinessLogic``, and that -``moreBusinessLogic`` called ``lordBusinessLogic``. diff --git a/util-intellij/PROJECT b/util-intellij/PROJECT deleted file mode 100644 index 24faddeed5..0000000000 --- a/util-intellij/PROJECT +++ /dev/null @@ -1,9 +0,0 @@ -# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path - -owners: - - banderson - - ccoco - - ibennett - - mnakamura - - roanta - - vkostyukov diff --git a/util-intellij/README.md b/util-intellij/README.md deleted file mode 100644 index 86831a9945..0000000000 --- a/util-intellij/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# util-intellij - -`util-intellij` is a project containing IntelliJ plugins and other utilities. - -## Contents - -### Twitter Futures Capture Points - -Stack traces produced from asynchronous execution have undesirable properties, -e.g., they are disorganized and often contain frames from internal libraries -that programmers need not know about. This result is inevitable since not only -do different parts of the same program get executed on different threads, and -possibly different cores, but execution also jumps between different frames. -This means that each thread used to execute a program may produce a unique stack -trace. Thus, an Asynchronous Stack Trace (AST) gives a fragmented window view -into the execution of a program and thus makes it difficult to understand the -flow of code because the causality between frames is lost. - -In response to this phenomenon, IntelliJ 2017.1 introduced a feature called -Capture Points which is built on top of the IntelliJ Debugger. A capture point -is a place in a computer program where the debugger collects and saves stack -frames to be used later when we reach a specific point in the code and we want -to see how we got there. IntelliJ IDEA does this by substituting part of the -call stack with the captured frame. - -We have written capture points for Twitter Futures in an XML file. Users can -debug their asynchronous code more easily with these capture points. Note that -as of October 2017, the capture points work with only Scala 2.11.11. - -#### Setup - -The capture points are defined in -`util/util-intellij/src/main/resources/TwitterFuturesCapturePoints.xml`. -To import them into IntelliJ, - -1. Open IntelliJ. In the menu bar, click IntelliJ IDEA > Preferences. -2. Navigate to Build, Execution, Deployment > Debugger > Async Stacktraces. -3. Click the Import icon on the bottom bar. Find the XML file and click OK. - -#### Use - -TL;DR: set a breakpoint where you wish to see the stack trace, debug your code, -and look at the "Frames" tab in the Debugger. Any asynchronous calls in the -stack trace will appear in logical order. If you wish to clean up the stack -trace, click the funnel icon in the top right, "Hide Frames from Libraries". - -We will illustrate how to use the capture points to assist with debugging with a -small example. The example is located in -`util/util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala`. - -A brief explanation of this test: the test passes a `Promise[Int]` through three -methods and then sets the `Promise`’s value in a `futurePool`. The calls are -asynchronous, but the logical flow of the test is as follows: - -1. `test` block calls `someBusinessLogic` -2. `someBusinessLogic` calls `moreBusinessLogic` -3. `moreBusinessLogic` calls `lordBusinessLogic` -4. `lordBusinessLogic` waits for the `Promise`’s value to be set -5. The `Promise`’s value is set in the test block (this could happen at any - time; it is not necessarily step number 5) -6. `lordBusinessLogic` returns -7. `test`’s `result` variable is `4`, and the test passes - -Suppose we wish to inspect the stack trace from inside `lordBusinessLogic`. If -we set a breakpoint at line 47 and then debug the test with pants in IntelliJ -with the capture points disabled, we see the following stack trace: - -``` -apply$mcVI$sp:47, Demo$$anonfun$lordBusinessLogic$1 -apply:1820, Future$$anonfun$onSuccess$1 -apply:205, Promise$Monitored -run:532, Promise$$anon$7 -run:198, LocalScheduler$Activation -submit:157, LocalScheduler$Activation -submit:274, LocalScheduler -submit:109, Scheduler$ -runq:522, Promise -updatelfEmpty:880, Promise -update:859, Promise -setValue:835, Promise -apply$mcV$sp:20, Demo$$anonfun$3$$anonfun$apply$1 -apply:15, Try$ -run:140, ExecutorServiceFuturePool$$anon$4 -``` - -The library calls in the stack trace do not help us understand our code. If we -enable the capture points and again debug the test, we see the following stack -trace: - -``` -apply$mcVI$sp:47, Demo$$anonfun$lordBusinessLogic$1 -apply:1820, Future$$anonfun$onSuccess$1 -onSuccess:1819, Future -lordBusinessLogic:42, Demo -moreBusinessLogic:38, Demo -someBusinessLogic:31, Demo -apply:17, Demo$$anonfun$3 -``` - -This is much cleaner and it includes only calls to functions we wrote. It lets -us clearly see that the test block called `someBusinessLogic`, that -`someBusinessLogic` called `moreBusinessLogic`, and that `moreBusinessLogic` -called `lordBusinessLogic`. diff --git a/util-intellij/src/main/resources/BUILD b/util-intellij/src/main/resources/BUILD deleted file mode 100644 index a001371a96..0000000000 --- a/util-intellij/src/main/resources/BUILD +++ /dev/null @@ -1,3 +0,0 @@ -resources( - sources = ["**/*"], -) diff --git a/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml b/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml deleted file mode 100644 index 9fc9fb702c..0000000000 --- a/util-intellij/src/main/resources/util-intellij/TwitterFuturesCapturePoints.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD b/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD deleted file mode 100644 index 4fbb4c3c95..0000000000 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -junit_tests( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - strict_deps = True, - tags = ["non-exclusive"], - dependencies = [ - "3rdparty/jvm/org/scala-lang/modules:scala-xml", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "util/util-core:util-core-util", - "util/util-intellij/src/main/resources", - ], -) diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala b/util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala deleted file mode 100644 index 1480ba50ee..0000000000 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/Demo.scala +++ /dev/null @@ -1,56 +0,0 @@ -package com.twitter.util.capturepoints - -import com.twitter.util.{Await, Future, FuturePool, Promise} -import org.scalatest.FunSuite - -/** - * A trivial unit test used for demonstrating the value of async stacktraces. - * - * Place a breakpoint in the `onSuccess` method inside `lordBusinessLogic` - * to see the difference in the actual stacktrace versus the IDE's synthetic - * one. - */ -class Demo extends FunSuite { - - test("async stack traces are rad") { - val promise = new Promise[Int]() - val future = someBusinessLogic(promise) - - FuturePool.unboundedPool { - promise.setValue(1) - } - - val result: Int = Await.result(future) - assert(4 == result) - } - - def someBusinessLogic(future: Future[Int]): Future[Int] = { - val result: Future[Int] = future.map { i => i + 1 } - moreBusinessLogic(result) - } - - def moreBusinessLogic(future: Future[Int]): Future[Int] = { - val result: Future[Int] = future.map { i => i * 2 } - lordBusinessLogic(result) - } - - def lordBusinessLogic(future: Future[Int]): Future[Int] = { - future.onSuccess { i => - // NOTE: with Intellij's async stacktraces configured a breakpoint - // here *will include* `someBusinessLogic` and `moreBusinessLogic` - // in the debugger's stacktrace, despite them not being in the - // actual stacktrace. - val stackTrace = Thread.currentThread.getStackTrace.mkString("\n") - println(stackTrace) - - assert(!stackTrace.contains("someBusinessLogic")) - assert(!stackTrace.contains("moreBusinessLogic")) - assert(stackTrace.contains("lordBusinessLogic")) - - println("\n (⊙_◎) (⧉ ⦣ ⧉) (｀_´)ゞ 「(°ヘ°)\n") - println(s" How did we get here? Where are `someBusinessLogic` and `moreBusinessLogic`?") - println("") - } - } - -} diff --git a/util-intellij/src/test/scala/com/twitter/util/capturepoints/TwitterFuturesCapturePointsTest.scala b/util-intellij/src/test/scala/com/twitter/util/capturepoints/TwitterFuturesCapturePointsTest.scala deleted file mode 100644 index b9984e7278..0000000000 --- a/util-intellij/src/test/scala/com/twitter/util/capturepoints/TwitterFuturesCapturePointsTest.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.twitter.util.capturepoints - -import java.lang.reflect.Method -import org.scalatest.FunSuite -import scala.xml.XML - -/** - * This test suite is intended to test capture points defined in [TwitterFuturesCapturePoints.xml]. - * The test name corresponds to the capture point being tested and is named in the format - * [Class.method]. Because of the limitations of reflection, we only test for [Capture class - * and method] and [Insert class and method]. This means that the [key-expression] is not - * tested for. As such, these tests will only fail if the class name or method for which a - * capture point depends on has either changed or doesn't exist anymore. - * - * In the case that the [Insert class-name] or [Capture class-name] has been changed or - * doesn't exist, this file will fail to compile. - */ -class TwitterFuturesCapturePointsTest extends FunSuite { - - /** - * extract [methodName] from `Class[_]` - */ - private[this] def findMethod(m: Class[_], methodName: String = "apply"): Array[Method] = { - m.getDeclaredMethods.filter(_.getName == methodName) - } - - /** - * test that [methodName] is a member of `ClassOf[com.twitter.util.className]` - */ - private[this] def testMethod( - methodType: String, - className: String, - methodName: String = "apply" - ): Unit = { - val m = Class.forName(className) - val methodArray = findMethod(m, methodName) - assert( - methodArray.nonEmpty, - s": couldn't find $methodType method ${m.getTypeName}.$methodName." - ) - } - - val capturePointsXML = - XML.load(getClass.getResourceAsStream("/util-intellij/TwitterFuturesCapturePoints.xml")) - - /** - * perform "capture method" and "insert method" tests for each capture point - */ - (capturePointsXML \ "capture-point").foreach { capturePoint => - val attrMap = capturePoint.attributes.asAttrMap - val insertMethodName = attrMap("insert-method-name") - val insertClassName = attrMap("insert-class-name") - val captureMethodName = attrMap("method-name") - val captureClassName = attrMap("class-name") - val testName = s"$captureClassName.$captureMethodName" - - test(testName) { - testMethod("capture", captureClassName, captureMethodName) - testMethod("insert", insertClassName, insertMethodName) - } - } -} From a5e7fe447a0d45c6137f8e5fde0dfc553a8e924b Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Thu, 11 Mar 2021 22:15:47 +0000 Subject: [PATCH 446/796] util: fix github workflows for util-intellij Problem: github workflows are failing Solution: remove util-intellij from tests Differential Revision: https://phabricator.twitter.biz/D633126 --- .github/workflows/continuous-build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/continuous-build.yml b/.github/workflows/continuous-build.yml index 78d6772300..5b7b7645f1 100644 --- a/.github/workflows/continuous-build.yml +++ b/.github/workflows/continuous-build.yml @@ -51,9 +51,6 @@ jobs: fi - name: test run: ${{ format('./sbt ++{0} clean coverage test coverageReport', matrix.scala) }} - - name: test util-intellij with scala 2.11 - if: ${{ startsWith(matrix.scala, '2.11') }} - run: ${{ format('./sbt ++{0} "project util-intellij" coverage test coverageReport', matrix.scala) }} - name: coverage if: ${{ success() }} run: ${{ format('./sbt ++{0} coverageAggregate', matrix.scala) }} From 2c90e2fc9e04eb26ee564c664fd7355380df2575 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 12 Mar 2021 19:46:09 +0000 Subject: [PATCH 447/796] {finagle, finatra, util, twitter-server, scrooge}: Drop scala 2.11 for OSS Problem/Solution: Two scala versions (2.12, 2.13) is sufficient. Let's drop scala 2.11. JIRA Issues: CSL-10599 Differential Revision: https://phabricator.twitter.biz/D631091 --- .github/workflows/continuous-build.yml | 2 +- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-build.yml b/.github/workflows/continuous-build.yml index 5b7b7645f1..24f5785fd8 100644 --- a/.github/workflows/continuous-build.yml +++ b/.github/workflows/continuous-build.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [2.11.12, 2.12.8, 2.13.1] + scala: [2.12.8, 2.13.1] java: ['1.8', '1.11'] runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c1aea418cf..511b6aaa4d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,8 @@ Unreleased Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ +* util: Builds are now only supported for Scala 2.12+ ``PHAB_ID=D631091`` + * util-reflect: Remove deprecated `c.t.util.reflect.Proxy`. There is no library replacement. ``PHAB_ID=D630143`` diff --git a/build.sbt b/build.sbt index 16f076038e..52ef2379af 100644 --- a/build.sbt +++ b/build.sbt @@ -79,7 +79,7 @@ def jdk11GcJavaOptions: Seq[String] = { val defaultProjectSettings = Seq( scalaVersion := "2.12.11", - crossScalaVersions := Seq("2.11.12", "2.12.11", "2.13.1") + crossScalaVersions := Seq("2.12.11", "2.13.1") ) val baseSettings = Seq( From 7e32800ae6345bd02bf62286ce2e9555132990d7 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 15 Mar 2021 18:08:22 +0000 Subject: [PATCH 448/796] util/util-reflect: Add `c.t.util.reflect.Types` utility Problem/Solution We'd like to add some general utilities for dealing with TypeTags. Add `c.t.util.reflect.Types` with a few functions dealing with TypeTags. Differential Revision: https://phabricator.twitter.biz/D631819 --- CHANGELOG.rst | 3 + .../com/twitter/util/reflect/Types.scala | 98 +++++++++++++ .../test/scala/com/twitter/util/reflect/BUILD | 11 ++ .../com/twitter/util/reflect/TypesTest.scala | 132 ++++++++++++++++++ 4 files changed, 244 insertions(+) create mode 100644 util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala create mode 100644 util-reflect/src/test/scala/com/twitter/util/reflect/BUILD create mode 100644 util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 511b6aaa4d..1663e51860 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,9 @@ Breaking API Changes New Features ~~~~~~~~~~~~ +* util-reflect: Add `c.t.util.reflect.Types`, a utility for some limited reflection based + operations. ``PHAB_ID=D631819`` + * util-core: `c.t.io` now supports creating and deconstructing unsigned 128-bit buffers in Buf. ``PHAB_ID=D606905`` diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala new file mode 100644 index 0000000000..8911c9a066 --- /dev/null +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala @@ -0,0 +1,98 @@ +package com.twitter.util.reflect + +import java.lang.reflect.{ParameterizedType, TypeVariable, Type => JavaType} +import scala.reflect.api.TypeCreator +import scala.reflect.runtime.universe._ + +object Types { + + /** + * Convert from the given `Class[T]` to a `TypeTag[T]` in the runtime universe. + * =Usage= + * {{{ + * val clazz: Class[T] = ??? + * val tag: TypeTag[T] = Types.asTypeTag(clazz) + * }}} + * + * @param clazz the class for which to build the resultant [[TypeTag]] + * @return a `TypeTag[T]` representing the given `Class[T]`. + */ + def asTypeTag[T](clazz: Class[_ <: T]): TypeTag[T] = { + val clazzMirror = runtimeMirror(clazz.getClassLoader) + val tpe = clazzMirror.classSymbol(clazz).toType + val typeCreator = new TypeCreator() { + def apply[U <: scala.reflect.api.Universe with scala.Singleton]( + m: scala.reflect.api.Mirror[U] + ): U#Type = { + if (clazzMirror != m) throw new RuntimeException("wrong mirror") + else tpe.asInstanceOf[U#Type] + } + } + TypeTag[T](clazzMirror, typeCreator) + } + + /** + * Return the runtime class from a given [[TypeTag]]. + * =Usage= + * {{{ + * val clazz: Class[T] = Types.runtimeClass[T] + * }}} + * + * or pass in the implict TypeTag explicitly: + * + * {{{ + * val tag: TypeTag[T] = ??? + * val clazz: Class[T] = Types.runtimeClass(tag) + * }}} + * + * + * @note the given [[TypeTag]] must be from the runtime universe otherwise an + * [[IllegalArgumentException]] will be thrown. + * + * @tparam T the [[TypeTag]] and expected class type of the returned class. + * @return the runtime class of the given [[TypeTag]]. + */ + def runtimeClass[T](implicit tag: TypeTag[T]): Class[T] = { + val clazzMirror = runtimeMirror(getClass.getClassLoader) + if (clazzMirror != tag.mirror) { + throw new IllegalArgumentException("TypeTag is not from runtime universe.") + } + tag.mirror.runtimeClass(tag.tpe).asInstanceOf[Class[T]] + } + + /** + * If `thisTypeTag` [[TypeTag]] has the same TypeSymbol as `thatTypeTag` [[TypeTag]]. + * + * @param thatTypeTag [[TypeTag]] to compare + * @param thisTypeTag [[TypeTag]] to compare + * @tparam T type of thisTypeTag + * @return true if the TypeSymbols are equivalent. + * @see [[scala.reflect.api.Symbols]] + */ + def eq[T](thatTypeTag: TypeTag[_])(implicit thisTypeTag: TypeTag[T]): Boolean = + thisTypeTag.tpe.typeSymbol == thatTypeTag.tpe.typeSymbol + + /** + * If the given [[java.lang.reflect.Type]] is parameterized, return an Array of the + * type parameter names. E.g., `Map[T, U]` returns, `Array("T", "U")`. + * + * The use case is when we are trying to match the given type's type parameters to + * a set of bindings that are stored keyed by the type name, e.g. if the given type + * is a `Map[K, V]` we want to be able to look up the binding for the key `K` at runtime + * during reflection operations e.g., if the K type is bound to a String we want to + * be able use that when further processing this type. This type of operation most typically + * happens with Jackson reflection handling of case classes, which is a specialized case + * and thus the utility of this method may not be broadly applicable and therefore is limited + * in visibility. + */ + private[twitter] def parameterizedTypeNames(javaType: JavaType): Array[String] = + javaType match { + case parameterizedType: ParameterizedType => + parameterizedType.getActualTypeArguments.map(_.getTypeName) + case typeVariable: TypeVariable[_] => + Array(typeVariable.getTypeName) + case clazz: Class[_] => + clazz.getTypeParameters.map(_.getName) + case _ => Array.empty + } +} diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD new file mode 100644 index 0000000000..d9a7039540 --- /dev/null +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -0,0 +1,11 @@ +junit_tests( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + dependencies = [ + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "util/util-core:util-core-util", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + ], +) diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala new file mode 100644 index 0000000000..e1a002d8af --- /dev/null +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala @@ -0,0 +1,132 @@ +package com.twitter.util.reflect + +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner +import scala.reflect.runtime.universe._ + +object TypesTest { + trait TestTraitA + trait TestTraitB + + case class Foo[T](data: T) + case class Baz[U, T](bar: U, foo: Foo[T]) + case class Bez[I, J](m: Map[I, J]) + + trait TypedTrait[A, B] +} + +@RunWith(classOf[JUnitRunner]) +class TypesTest extends AnyFunSuite with Matchers { + import TypesTest._ + + test("Types#asTypeTag handles classes") { + typeTagEquals(classOf[String], Types.asTypeTag(classOf[String])) + typeTagEquals(classOf[AnyRef], Types.asTypeTag(classOf[AnyRef])) + typeTagEquals(classOf[AnyVal], Types.asTypeTag(classOf[AnyVal])) + typeTagEquals(classOf[Unit], Types.asTypeTag(classOf[Unit])) + typeTagEquals(classOf[Byte], Types.asTypeTag(classOf[Byte])) + typeTagEquals(classOf[Short], Types.asTypeTag(classOf[Short])) + typeTagEquals(classOf[Char], Types.asTypeTag(classOf[Char])) + typeTagEquals(classOf[Int], Types.asTypeTag(classOf[Int])) + typeTagEquals(classOf[Long], Types.asTypeTag(classOf[Long])) + typeTagEquals(classOf[Float], Types.asTypeTag(classOf[Float])) + typeTagEquals(classOf[Double], Types.asTypeTag(classOf[Double])) + typeTagEquals(classOf[Boolean], Types.asTypeTag(classOf[Boolean])) + typeTagEquals(classOf[java.lang.Object], Types.asTypeTag(classOf[java.lang.Object])) + typeTagEquals(classOf[Any], Types.asTypeTag(classOf[Any])) + typeTagEquals(classOf[Null], Types.asTypeTag(classOf[Null])) + typeTagEquals(classOf[Nothing], Types.asTypeTag(classOf[Nothing])) + + // generics + typeTagEquals(classOf[Baz[_, _]], Types.asTypeTag(classOf[Baz[_, _]])) + typeTagEquals(classOf[Foo[_]], Types.asTypeTag(classOf[Foo[_]])) + typeTagEquals(classOf[Bez[_, _]], Types.asTypeTag(classOf[Bez[_, _]])) + typeTagEquals(classOf[TypedTrait[_, _]], Types.asTypeTag(classOf[TypedTrait[_, _]])) + } + + test("Types#runtimeClass") { + val tag = typeTag[Baz[_, _]] + Types.runtimeClass(tag) should equal(classOf[Baz[_, _]]) + Types.runtimeClass[Baz[_, _]] should equal(classOf[Baz[_, _]]) + + classOf[AnyRef].isAssignableFrom(Types.runtimeClass[AnyRef]) + classOf[AnyVal].isAssignableFrom(Types.runtimeClass[AnyVal]) + classOf[Unit].isAssignableFrom(Types.runtimeClass[Unit]) + classOf[Byte].isAssignableFrom(Types.runtimeClass[Byte]) + classOf[Short].isAssignableFrom(Types.runtimeClass[Short]) + classOf[Char].isAssignableFrom(Types.runtimeClass[Char]) + classOf[Int].isAssignableFrom(Types.runtimeClass[Int]) + classOf[Long].isAssignableFrom(Types.runtimeClass[Long]) + classOf[Float].isAssignableFrom(Types.runtimeClass[Float]) + classOf[Double].isAssignableFrom(Types.runtimeClass[Double]) + classOf[Boolean].isAssignableFrom(Types.runtimeClass[Boolean]) + classOf[java.lang.Object].isAssignableFrom(Types.runtimeClass[java.lang.Object]) + classOf[Null].isAssignableFrom(Types.runtimeClass[scala.runtime.Null$]) + classOf[Nothing].isAssignableFrom(Types.runtimeClass[scala.runtime.Nothing$]) + // `Any` is special and can result in a ClassNotFoundException + //Types.runtimeClass[Any] + + } + + test("Types#eq") { + Types.eq[String](typeTag[String]) should be(true) + Types.eq[AnyRef](TypeTag.AnyRef) should be(true) + Types.eq[AnyVal](TypeTag.AnyVal) should be(true) + Types.eq[Unit](TypeTag.Unit) should be(true) + Types.eq[Byte](TypeTag.Byte) should be(true) + Types.eq[Short](TypeTag.Short) should be(true) + Types.eq[Char](TypeTag.Char) should be(true) + Types.eq[Int](TypeTag.Int) should be(true) + Types.eq[Long](TypeTag.Long) should be(true) + Types.eq[Float](TypeTag.Float) should be(true) + Types.eq[Double](TypeTag.Double) should be(true) + Types.eq[Boolean](TypeTag.Boolean) should be(true) + Types.eq[java.lang.Object](TypeTag.Object) should be(true) + Types.eq[Any](TypeTag.Any) should be(true) + Types.eq[Null](TypeTag.Null) should be(true) + Types.eq[Nothing](TypeTag.Nothing) should be(true) + // generics + Types.eq[Baz[_, _]](typeTag[Baz[_, _]]) should be(true) + Types.eq[Foo[_]](typeTag[Foo[_]]) should be(true) + Types.eq[Bez[_, _]](typeTag[Bez[_, _]]) should be(true) + Types.eq[TypedTrait[_, _]](typeTag[TypedTrait[_, _]]) should be(true) + + Types.eq[Int](TypeTag.Float) should be(false) + Types.eq[String](TypeTag.Int) should be(false) + Types.eq[java.lang.Object](TypeTag.Int) should be(false) + Types.eq[String](TypeTag.Object) should be(false) + } + + test("Types#parameterizedTypeNames") { + val clazz = classOf[Baz[_, _]] + Types.parameterizedTypeNames(clazz) should equal(Array("U", "T")) + + val constructors = clazz.getConstructors + constructors.size should equal(1) + constructors.foreach { cons => + cons.getParameters.length should equal(2) + val first = cons.getParameters.head + val second = cons.getParameters.last + Types.parameterizedTypeNames(first.getParameterizedType) should equal(Array("U")) + Types.parameterizedTypeNames(second.getParameterizedType) should equal(Array("T")) + } + + Types.parameterizedTypeNames(classOf[TypedTrait[_, _]]) should equal(Array("A", "B")) + Types.parameterizedTypeNames(classOf[Map[_, _]]) should equal(Array("K", "V")) + + val clazz2 = classOf[Bez[_, _]] + Types.parameterizedTypeNames(clazz2) should equal(Array("I", "J")) + val constructors2 = clazz2.getConstructors + constructors2.size should equal(1) + constructors2.foreach { cons => + cons.getParameters.length should equal(1) + val first = cons.getParameters.head + Types.parameterizedTypeNames(first.getParameterizedType) should equal(Array("I", "J")) + } + } + + private[this] def typeTagEquals[T](clazz: Class[T], typeTag: TypeTag[T]): Boolean = + clazz.isAssignableFrom(typeTag.mirror.runtimeClass(typeTag.tpe.typeSymbol.asClass)) +} From b2b94e974dcbf478552631633b0cb358aa8cf125 Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Tue, 16 Mar 2021 06:18:44 +0000 Subject: [PATCH 449/796] Bump scala version to 2.12.13 Problem/Solution Bumping scala version to 2.12.13 JIRA Issues: SCALA-25 Differential Revision: https://phabricator.twitter.biz/D632567 --- .github/workflows/continuous-build.yml | 2 +- CHANGELOG.rst | 5 +++++ build.sbt | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/continuous-build.yml b/.github/workflows/continuous-build.yml index 24f5785fd8..70b9f14cad 100644 --- a/.github/workflows/continuous-build.yml +++ b/.github/workflows/continuous-build.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [2.12.8, 2.13.1] + scala: [2.12.13, 2.13.1] java: ['1.8', '1.11'] runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1663e51860..8978f46f3a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Bump scala version to 2.12.13 ``PHAB_ID=D632567`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index 52ef2379af..90a2df21a3 100644 --- a/build.sbt +++ b/build.sbt @@ -78,8 +78,8 @@ def jdk11GcJavaOptions: Seq[String] = { } val defaultProjectSettings = Seq( - scalaVersion := "2.12.11", - crossScalaVersions := Seq("2.12.11", "2.13.1") + scalaVersion := "2.12.13", + crossScalaVersions := Seq("2.12.13", "2.13.1") ) val baseSettings = Seq( From 8b244478506254df53a265ff586123748017b68f Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 16 Mar 2021 19:26:47 +0000 Subject: [PATCH 450/796] Bump scala version to 2.12.13 in workflows Problem/Solution We've updated the supported scala version in the sbt files lets follow up with the CI workflows. Util was already done in the first update. Differential Revision: https://phabricator.twitter.biz/D635494 --- .../{continuous-build.yml => continuous-integration.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{continuous-build.yml => continuous-integration.yml} (100%) diff --git a/.github/workflows/continuous-build.yml b/.github/workflows/continuous-integration.yml similarity index 100% rename from .github/workflows/continuous-build.yml rename to .github/workflows/continuous-integration.yml From c075bb13efb2b051f9963da323c37abb0b6a9ba9 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 17 Mar 2021 01:35:50 +0000 Subject: [PATCH 451/796] Revert to scala 2.12.12 due to scoverage issue with scala 2.12.13 Problem/Solution The scoverage plugin has an issue due to a binary incompatibility introduced in scala 2.12.13 (see: https://github.com/scoverage/sbt-scoverage/issues/319). It has been patched in the plugin in https://github.com/scoverage/scalac-scoverage-plugin/pull/306 which is currently awaiting a release. However after updating our projects to scala 2.12.13, coverage is broken and failing the CI builds. Downgrade to scala 2.12.12 until the scoverage plugin has been published with the fix. Differential Revision: https://phabricator.twitter.biz/D635917 --- .github/workflows/continuous-integration.yml | 2 +- CHANGELOG.rst | 3 +++ build.sbt | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 70b9f14cad..1ad9fb670c 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [2.12.13, 2.13.1] + scala: [2.12.12, 2.13.1] java: ['1.8', '1.11'] runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8978f46f3a..7b7799b8df 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ +* util: Revert to scala version 2.12.12 due to https://github.com/scoverage/sbt-scoverage/issues/319 + ``PHAB_ID=D635917`` + * util: Bump scala version to 2.12.13 ``PHAB_ID=D632567`` Breaking API Changes diff --git a/build.sbt b/build.sbt index 90a2df21a3..93dd3044fa 100644 --- a/build.sbt +++ b/build.sbt @@ -78,8 +78,8 @@ def jdk11GcJavaOptions: Seq[String] = { } val defaultProjectSettings = Seq( - scalaVersion := "2.12.13", - crossScalaVersions := Seq("2.12.13", "2.13.1") + scalaVersion := "2.12.12", + crossScalaVersions := Seq("2.12.12", "2.13.1") ) val baseSettings = Seq( From b3b1e9ca5dec0eb974f728dfd1fafc5d0ee479b5 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 18 Mar 2021 19:53:24 +0000 Subject: [PATCH 452/796] twitter/oss: Unset TRAVIS env var Problem We set the `TRAVIS` env var to convince the various `build.sbt` files to set appropriate configuration in the forked test processes. However, this is confusing our code coverage tool as it is picking up the env var and attempting to verify the project as a TRAVIS CI build, which it is not and thus coverage uploading fails. Solution Unset the `TRAVIS` env var before uploading results such that the auto detection of the CI provider properly picks up the GitHub Action env vars and does the right thing. Differential Revision: https://phabricator.twitter.biz/D636932 --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 1ad9fb670c..7a658004aa 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -56,4 +56,4 @@ jobs: run: ${{ format('./sbt ++{0} coverageAggregate', matrix.scala) }} - name: upload coverage if: ${{ success() }} - run: bash <(curl -s https://codecov.io/bash) + run: unset TRAVIS; bash <(curl -s https://codecov.io/bash) From f1c2ec81959f6d3be128a3592937b2ad957e3a04 Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Thu, 18 Mar 2021 21:54:30 +0000 Subject: [PATCH 453/796] util-stats: label expressions with service name Problem Expressions with the same name clobber each other inside of the Metrics store. Solution Use RoleConfiguredStatsReceiver to store the name of the Client/Server and use this value to populate the ServiceName label, which is used in turn as part of the expressions name in the internal Metrics storage. Result Multiple success_rate queries for different portions of a service can now coexist. Differential Revision: https://phabricator.twitter.biz/D630620 --- .../finagle/stats/InMemoryStatsReceiver.scala | 13 ++++++++-- .../stats/RoleConfiguredStatsReceiver.scala | 9 ++++++- .../finagle/stats/exp/ExpressionSchema.scala | 3 +++ .../finagle/stats/exp/ExprEndToEndTest.scala | 24 ++++++++++++------- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 75437a2ed7..760b314717 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -246,8 +246,17 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { case (k, v) => (k.mkString("/"), toHistogramDetail(v)) } - override protected[finagle] def registerExpression(schema: ExpressionSchema): Unit = - expressions(schema.name) = schema + /** + * Designed to match the behavior of Metrics::registerExpression(). + */ + override protected[finagle] def registerExpression(schema: ExpressionSchema): Unit = { + val expressionId = schema.labels.serviceName match { + case Some(serviceName) => schema.name + "_" + serviceName + case None => schema.name + } + expressions(expressionId) = schema + } + } /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala index 99a7e99870..6c0d3778b2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala @@ -1,5 +1,7 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.exp.ExpressionSchema + /** * A StatsReceiver proxy that configures all counter, stat, and gauge * SourceRoles to the passed in "role". @@ -9,7 +11,8 @@ package com.twitter.finagle.stats */ case class RoleConfiguredStatsReceiver( protected val self: StatsReceiver, - role: SourceRole) + role: SourceRole, + name: Option[String] = None) extends StatsReceiverProxy { override def counter(counterSchema: CounterSchema): Counter = { @@ -23,4 +26,8 @@ case class RoleConfiguredStatsReceiver( override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { self.addGauge(GaugeSchema(gaugeSchema.metricBuilder.withRole(role)))(f) } + + override def registerExpression(expressionSchema: ExpressionSchema): Unit = { + self.registerExpression(expressionSchema.withRole(role = role).withServiceName(name)) + } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index c1d7a4b0a9..15203a55af 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -18,6 +18,9 @@ case class ExpressionSchema private ( private[finagle] def withRole(role: SourceRole): ExpressionSchema = copy(labels = labels.copy(role = role)) + private[finagle] def withServiceName(name: Option[String]): ExpressionSchema = + copy(labels = labels.copy(serviceName = name)) + def register(): Unit = { Expression.getStatsReceivers(expr).toSeq match { case Seq(sr) => sr.registerExpression(this) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala index 5be66171ac..13a15400ff 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala @@ -16,14 +16,20 @@ class ExprEndToEndTest extends FunSuite { test("Demonstrate an end to end example of using metrics") { val sr = new InMemoryStatsReceiver - val successMb = CounterSchema(new MetricBuilder(name = Seq("success"), statsReceiver = sr)) + val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) + + val successMb = + CounterSchema(new MetricBuilder(name = Seq("success"), statsReceiver = clientSR)) val failuresMb = - CounterSchema(new MetricBuilder(name = Seq("failures"), statsReceiver = sr)) - val latencyMb = HistogramSchema(new MetricBuilder(name = Seq("latency"), statsReceiver = sr)) + CounterSchema(new MetricBuilder(name = Seq("failures"), statsReceiver = clientSR)) + val latencyMb = + HistogramSchema(new MetricBuilder(name = Seq("latency"), statsReceiver = clientSR)) + val sum = Expression(successMb).plus(Expression(failuresMb)) - val successCounter = sr.counter(successMb) - val failuresCounter = sr.counter(failuresMb) - val latencyStat = sr.stat(latencyMb) + val successCounter = clientSR.counter(successMb) + val failuresCounter = clientSR.counter(failuresMb) + val latencyStat = clientSR.stat(latencyMb) + val successRate = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) .withBounds(MonotoneThreshold(GreaterThan, 99.5, 99.75)) .withUnit(Percentage) @@ -53,11 +59,13 @@ class ExprEndToEndTest extends FunSuite { runTheQuery(true) runTheQuery(false) - assert(sr.expressions("success_rate") == successRate) - assert(sr.expressions("latency") == latency) + assert(sr.expressions("success_rate_downstream").expr == successRate.expr) + assert(sr.expressions("latency_downstream").expr == latency.expr) assert(sr.counters(Seq("success")) == 1) assert(sr.counters(Seq("failures")) == 1) assert(sr.stats(Seq("latency")) == Seq(50, 50)) + + assert(sr.expressions("success_rate_downstream").labels.role == Client) } } From 9759a70caea1d2fc59a9ce20555e6e2877c66fb9 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 19 Mar 2021 21:10:07 +0000 Subject: [PATCH 454/796] twitter-server: Expression thresholds to JSON Problem/Solution This revision's goal is serializing and deserializing Expression Bounds (thresholds) to/from JSON nicely. The Bounds are self-explained case classes that can leverage Jackson Json mapping, some special cases such as deserializing Operator is supported by adding a customized deserializer. Two additional changes that made for supporting this are: ``` 1. Set ObjectMapper to use snake case naming strategy, all of our customized de/serializers are writing snake case, make the object mapper also snake case to be consistent. The JsonConverter has a few usages outside of the twitter-server, we keep that as a utility helper and switch the twitter-server Admin endpoints to use the configured one. 2. Some Json utilities are redundant. merge them. ``` JIRA Issues: CSL-10619, CSL-10803 Differential Revision: https://phabricator.twitter.biz/D634550 --- build.sbt | 3 ++ .../twitter/finagle/stats/exp/Bounds.scala | 41 ++++++++++++++++++- .../finagle/stats/exp/ExpressionSchema.scala | 8 +++- .../twitter/finagle/stats/exp/Operator.scala | 11 ++++- .../finagle/stats/exp/ExprEndToEndTest.scala | 2 +- 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 93dd3044fa..7cfa3b4187 100644 --- a/build.sbt +++ b/build.sbt @@ -497,6 +497,9 @@ lazy val utilStats = Project( caffeineLib, jsr305Lib, scalacheckLib, + "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, + "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava"), "org.mockito" % "mockito-core" % mockitoVersion % "test", "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Bounds.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Bounds.scala index 396b85b023..49eaef4655 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Bounds.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Bounds.scala @@ -1,10 +1,42 @@ package com.twitter.finagle.stats.exp +import com.fasterxml.jackson.annotation.JsonSubTypes.Type +import com.fasterxml.jackson.annotation.{JsonSubTypes, JsonTypeInfo} + +/** + * Thresholds are specified per-expression. The default is Unbounded. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "kind") +@JsonSubTypes( + Array( + new Type(value = classOf[AcceptableRange], name = "range"), + new Type(value = classOf[MonotoneThresholds], name = "monotone"), + new Type(value = classOf[Unbounded], name = "unbounded") + ) +) sealed trait Bounds +/** + * The state is considered healthy when the expression computation is inside this range. + * Health status and expression computation are not necessary linearly correlated. + */ case class AcceptableRange(lowerBoundInclusive: Double, upperBoundExclusive: Double) extends Bounds -case class MonotoneThreshold( +/** + * Health status and expression computation are linearly correlated, and the healthier + * direction is indicated by [[operator]]. + * + * [lowerBoundInclusive, badThreshold] bad + * [badThreshold, goodThreshold] ok + * [goodThreshold, upperBoundExclusive] good + * + * @note all ranges are lower inclusive and upper exclusive. + * + * @param operator GreaterThan or LessThan + * @param lowerBoundInclusive Optional, undefined meaning unbounded lower bound. + * @param upperBoundExclusive Optional, undefined meaning unbounded upper bound. + */ +case class MonotoneThresholds( operator: Operator, badThreshold: Double, goodThreshold: Double, @@ -12,4 +44,9 @@ case class MonotoneThreshold( upperBoundExclusive: Option[Double] = None) extends Bounds -case object Unbounded extends Bounds +// does not use case object because it cannot provide `Class` for @JsonSubTypes +// see https://github.com/scala/scala/pull/9279 +object Unbounded { + val get = Unbounded() +} +case class Unbounded() extends Bounds diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index 15203a55af..3dadd10132 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -33,5 +33,11 @@ case class ExpressionSchema private ( // expose for testing in twitter-server private[twitter] object ExpressionSchema { def apply(name: String, expr: Expression): ExpressionSchema = - ExpressionSchema(name, ExpressionLabels.empty, expr, Unbounded, "Unspecified", Unspecified) + ExpressionSchema( + name = name, + labels = ExpressionLabels.empty, + expr = expr, + bounds = Unbounded.get, + description = "Unspecified", + unit = Unspecified) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Operator.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Operator.scala index ff23068b09..4671fb1d98 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Operator.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Operator.scala @@ -1,15 +1,22 @@ package com.twitter.finagle.stats.exp +import com.fasterxml.jackson.annotation.JsonValue + +/** + * Represent Metric Expression thresholds operator. + */ sealed trait Operator { def isIncreasing: Boolean + @JsonValue + def toJson: String } case object GreaterThan extends Operator { - override def toString: String = ">" + def toJson: String = ">" val isIncreasing: Boolean = true } case object LessThan extends Operator { - override def toString: String = "<" + def toJson: String = "<" val isIncreasing: Boolean = false } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala index 13a15400ff..34929b0475 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala @@ -31,7 +31,7 @@ class ExprEndToEndTest extends FunSuite { val latencyStat = clientSR.stat(latencyMb) val successRate = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) - .withBounds(MonotoneThreshold(GreaterThan, 99.5, 99.75)) + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) .withUnit(Percentage) .withDescription("The success rate of the slow query") From 95c45f7107fb840a0e0ffa89ef3045f208ba4dd1 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 23 Mar 2021 20:59:59 +0000 Subject: [PATCH 455/796] util-reflect: Add c.t.util.reflect{Annotations, Classes} Problem/Solution Add `c.t.util.reflect.Annotations` a utility for finding annotations on a class and `c.t.util.reflect.Classes` which has a utility for obtaining the `simpleName` of a given class across different JDK versions and while handling mangled names (those with non-supported Java identifier characters). Also add utilities to determine if a given class is a case class in `c.t.util.reflect.Types`. Differential Revision: https://phabricator.twitter.biz/D638655 --- CHANGELOG.rst | 6 + .../twitter/util/reflect/Annotations.scala | 290 +++++++++++++ .../com/twitter/util/reflect/Classes.scala | 26 ++ .../com/twitter/util/reflect/Types.scala | 29 ++ .../com/twitter/util/reflect/Annotation1.java | 11 + .../com/twitter/util/reflect/Annotation2.java | 12 + .../com/twitter/util/reflect/Annotation3.java | 14 + .../com/twitter/util/reflect/Annotation4.java | 11 + .../com/twitter/util/reflect/Annotation5.java | 13 + .../test/java/com/twitter/util/reflect/BUILD | 4 + .../util/reflect/MarkerAnnotation.java | 10 + .../java/com/twitter/util/reflect/Thing.java | 15 + .../com/twitter/util/reflect/ThingImpl.java | 43 ++ .../java/com/twitter/util/reflect/Things.java | 11 + .../java/com/twitter/util/reflect/Widget.java | 15 + .../com/twitter/util/reflect/WidgetImpl.java | 43 ++ .../com/twitter/util/reflect/Widgets.java | 11 + .../util/reflect/AnnotationsTest.scala | 390 ++++++++++++++++++ .../test/scala/com/twitter/util/reflect/BUILD | 1 + .../twitter/util/reflect/ClassesTest.scala | 100 +++++ .../com/twitter/util/reflect/TypesTest.scala | 132 +++++- .../twitter/util/reflect/testclasses.scala | 130 ++++++ 22 files changed, 1304 insertions(+), 13 deletions(-) create mode 100644 util-reflect/src/main/scala/com/twitter/util/reflect/Annotations.scala create mode 100644 util-reflect/src/main/scala/com/twitter/util/reflect/Classes.scala create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/Annotation1.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/Annotation2.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/Annotation3.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/Annotation4.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/Annotation5.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/BUILD create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/MarkerAnnotation.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/Thing.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/ThingImpl.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/Things.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/Widget.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/WidgetImpl.java create mode 100644 util-reflect/src/test/java/com/twitter/util/reflect/Widgets.java create mode 100644 util-reflect/src/test/scala/com/twitter/util/reflect/AnnotationsTest.scala create mode 100644 util-reflect/src/test/scala/com/twitter/util/reflect/ClassesTest.scala create mode 100644 util-reflect/src/test/scala/com/twitter/util/reflect/testclasses.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7b7799b8df..7a075be1f0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,6 +26,12 @@ Breaking API Changes New Features ~~~~~~~~~~~~ +* util-reflect: Add `c.t.util.reflect.Annotations` a utility for finding annotations on a class and + `c.t.util.reflect.Classes` which has a utility for obtaining the `simpleName` of a given class + across JDK versions and while handling mangled names (those with non-supported Java identifier + characters). Also add utilities to determine if a given class is a case class in + `c.t.util.reflect.Types`. ``PHAB_ID=D638655`` + * util-reflect: Add `c.t.util.reflect.Types`, a utility for some limited reflection based operations. ``PHAB_ID=D631819`` diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Annotations.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Annotations.scala new file mode 100644 index 0000000000..ab719907b1 --- /dev/null +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/Annotations.scala @@ -0,0 +1,290 @@ +package com.twitter.util.reflect + +import com.twitter.util.{Return, Throw, Try} +import java.lang.annotation.Annotation +import java.lang.reflect.{Field, Parameter} +import scala.reflect.{ClassTag, classTag} + +/** Utility methods for dealing with [[java.lang.annotation.Annotation]] */ +object Annotations { + + /** + * Filter a list of annotations discriminated on whether the + * annotation is itself annotated with the passed annotation. + * + * @param annotations the list of [[Annotation]] instances to filter. + * @tparam A the type of the annotation by which to filter. + * + * @return the filtered list of matching annotations. + */ + def filterIfAnnotationPresent[A <: Annotation: ClassTag]( + annotations: Array[Annotation] + ): Array[Annotation] = annotations.filter(isAnnotationPresent[A]) + + /** + * Filters a list of annotations by annotation type discriminated by the set of given annotations. + * @param filterSet the Set of [[Annotation]] classes by which to filter. + * @param annotations the list [[Annotation]] instances to filter. + * + * @return the filtered list of matching annotations. + */ + def filterAnnotations( + filterSet: Set[Class[_ <: Annotation]], + annotations: Array[Annotation] + ): Array[Annotation] = + annotations.filter(a => filterSet.contains(a.annotationType)) + + /** + * Find an [[Annotation]] within a given list of annotations of the given target. + * annotation type. + * @param target the class of the [[Annotation]] to find. + * @param annotations the list of [[Annotation]] instances to search. + * + * @return the matching [[Annotation]] instance if found, otherwise None. + */ + // optimized + def findAnnotation( + target: Class[_ <: Annotation], + annotations: Array[Annotation] + ): Option[Annotation] = { + var found: Option[Annotation] = None + var index = 0 + while (index < annotations.length && found.isEmpty) { + val annotation = annotations(index) + if (annotation.annotationType() == target) found = Some(annotation) + index += 1 + } + found + } + + /** + * Find an [[Annotation]] within a given list of annotations annotated by the given type param. + * @param annotations the list of [[Annotation]] instances to search. + * @tparam A the type of the [[Annotation]] to find. + * + * @return the matching [[Annotation]] instance if found, otherwise None. + */ + // optimized + def findAnnotation[A <: Annotation: ClassTag](annotations: Array[Annotation]): Option[A] = { + val size = annotations.length + val annotationType = classTag[A].runtimeClass.asInstanceOf[Class[A]] + var found: Option[A] = None + var index = 0 + while (found.isEmpty && index < size) { + val annotation = annotations(index) + if (annotation.annotationType() == annotationType) found = Some(annotation.asInstanceOf[A]) + index += 1 + } + found + } + + /** + * Attempts to map fields to array of annotations. This is intended to work around the Scala default + * of not carrying annotation information on annotated constructor fields annotated without also + * specifying a @meta annotation (e.g., `@field`). Thus, this method scans the constructor and + * then all declared fields in order to build up the mapping of field name to `Array[java.lang.Annotation]`. + * When the given class has a single constructor, that constructor will be used. Otherwise, the + * given parameter types are used to locate a constructor. + * Steps: + * - walk the constructor and collect annotations on all parameters. + * - for each declared field in the class, collect declared annotations. If a constructor parameter + * is overridden as a declared field, the annotations on the declared field take precedence. + * - for each super interface, walk each declared method, collect declared method annotations IF + * the declared method is the same name of a declared field from step two. That is find only + * super interface methods which are implemented by declared fields in the given class in order + * to locate inherited annotations for the declared field. + * + * @param clazz the `Class` to inspect. This should represent a Scala case class. + * @param parameterTypes an optional list of parameter class types in order to locate the appropriate + * case class constructor when the class has multiple constructors. If a class + * has multiple constructors and parameter types are not specified, an + * [[IllegalArgumentException]] is thrown. Likewise, if a suitable constructor + * cannot be located by the given parameter types an [[IllegalArgumentException]] + * is thrown. + * + * @see [[https://www.scala-lang.org/api/current/scala/annotation/meta/index.html]] + * @return a map of field name to associated annotations. An entry in the map only occurs if the + * associated annotations are non-empty. That is fields without any found annotations are + * not returned. + */ + // optimized + def findAnnotations( + clazz: Class[_], + parameterTypes: Seq[Class[_]] = Seq.empty + ): scala.collection.Map[String, Array[Annotation]] = { + val clazzAnnotations = scala.collection.mutable.HashMap[String, Array[Annotation]]() + val constructorCount = clazz.getConstructors.length + val constructor = if (constructorCount > 1) { + if (parameterTypes.isEmpty) + throw new IllegalArgumentException( + "Multiple constructors detected and no parameter types specified.") + Try(clazz.getConstructor(parameterTypes: _*)) match { + case Return(con) => con + case Throw(t: NoSuchMethodException) => + // translate from the reflection exception to a runtime exception + throw new IllegalArgumentException(t) + case Throw(t) => throw t + } + } else clazz.getConstructors.head + + val fields: Array[Parameter] = constructor.getParameters + val clazzConstructorAnnotations: Array[Array[Annotation]] = constructor.getParameterAnnotations + + var i = 0 + while (i < fields.length) { + val field = fields(i) + val fieldAnnotations = clazzConstructorAnnotations(i) + if (fieldAnnotations.nonEmpty) clazzAnnotations.put(field.getName, fieldAnnotations) + i += 1 + } + + val declaredFields: Array[Field] = clazz.getDeclaredFields + var j = 0 + while (j < declaredFields.length) { + val field = declaredFields(j) + if (field.getDeclaredAnnotations.nonEmpty) { + if (!clazzAnnotations.contains(field.getName)) { + clazzAnnotations.put(field.getName, field.getDeclaredAnnotations) + } else { + val existing = clazzAnnotations(field.getName) + // prefer field annotations over constructor annotation + mergeAnnotationArrays(field.getDeclaredAnnotations, existing) + } + } + + j += 1 + } + + // find inherited annotations for declared fields + findDeclaredAnnotations( + clazz, + declaredFields.map(_.getName).toSet, + clazzAnnotations + ) + } + + /** + * Determines if the given [[Annotation]] has an annotation type of the given type param. + * @param annotation the [[Annotation]] to match. + * @tparam A the type to match against. + * + * @return true if the given [[Annotation]] is of type [[A]], false otherwise. + */ + def annotationEquals[A <: Annotation: ClassTag](annotation: Annotation): Boolean = + annotation.annotationType() == classTag[A].runtimeClass.asInstanceOf[Class[A]] + + /** + * Determines if the given [[Annotation]] is annotated by an [[Annotation]] of the given + * type param. + * @param annotation the [[Annotation]] to match. + * @tparam A the type of the [[Annotation]] to determine if is annotated on the [[Annotation]]. + * + * @return true if the given [[Annotation]] is annotated with an [[Annotation]] of type [[A]], + * false otherwise. + */ + def isAnnotationPresent[A <: Annotation: ClassTag](annotation: Annotation): Boolean = + annotation.annotationType.isAnnotationPresent(classTag[A].runtimeClass.asInstanceOf[Class[A]]) + + /** + * Determines if the given [[A]] is annotated by an [[Annotation]] of the given + * type param [[ToFindAnnotation]]. + * + * @tparam ToFindAnnotation the [[Annotation]] to match. + * @tparam A the type of the [[Annotation]] to determine if is annotated on the [[A]]. + * + * @return true if the given [[Annotation]] is annotated with an [[Annotation]] of type [[ToFindAnnotation]], + * false otherwise. + */ + def isAnnotationPresent[ + ToFindAnnotation <: Annotation: ClassTag, + A <: Annotation: ClassTag + ]: Boolean = { + val annotationToFindClazz: Class[Annotation] = + classTag[ToFindAnnotation].runtimeClass + .asInstanceOf[Class[Annotation]] + + classTag[A].runtimeClass + .asInstanceOf[Class[A]].isAnnotationPresent(annotationToFindClazz) + } + + /** + * Attempts to return the result of invoking `method()` of a given [[Annotation]] which should be + * annotated by an [[Annotation]] of type [[A]]. + * + * If the given [[Annotation]] is not annotated by an [[Annotation]] of type [[A]] or if the + * `Annotation#method` cannot be invoked then [[None]] is returned. + * + * @param annotation the [[Annotation]] to process. + * @param method the method of the [[Annotation]] to invoke to obtain a value. + * @tparam A the [[Annotation]] type to match against before trying to invoke `annotation#method`. + * + * @return the result of invoking `annotation#method()` or None. + */ + def getValueIfAnnotatedWith[A <: Annotation: ClassTag]( + annotation: Annotation, + method: String = "value" + ): Option[String] = { + if (isAnnotationPresent[A](annotation)) { + getValue(annotation, method) + } else None + } + + /** + * Attempts to return the result of invoking `method()` of a given [[Annotation]]. If the + * `Annotation#method` cannot be invoked then [[None]] is returned. + * + * @param annotation the [[Annotation]] to process. + * @param method the method of the [[Annotation]] to invoke to obtain a value. + * + * @return the result of invoking `annotation#method()` or None. + */ + def getValue( + annotation: Annotation, + method: String = "value" + ): Option[String] = { + for { + method <- annotation.getClass.getDeclaredMethods.find(_.getName == method) + value <- Try(method.invoke(annotation).asInstanceOf[String]).toOption + } yield value + } + + // optimized + private[twitter] def findDeclaredAnnotations( + clazz: Class[_], + declaredFields: Set[String], // `.contains` on a Set should be faster than on an Array + acc: scala.collection.mutable.Map[String, Array[Annotation]] + ): scala.collection.Map[String, Array[Annotation]] = { + val methods = clazz.getDeclaredMethods + var i = 0 + while (i < methods.length) { + val method = methods(i) + val methodAnnotations = method.getDeclaredAnnotations + if (methodAnnotations.nonEmpty && declaredFields.contains(method.getName)) { + acc.get(method.getName) match { + case Some(existing) => + acc.put(method.getName, mergeAnnotationArrays(existing, methodAnnotations)) + case _ => + acc.put(method.getName, methodAnnotations) + } + } + i += 1 + } + + val interfaces = clazz.getInterfaces + var j = 0 + while (j < interfaces.length) { + val interface = interfaces(j) + findDeclaredAnnotations(interface, declaredFields, acc) + j += 1 + } + + acc + } + + /** Prefer values in A over B */ + private[this] def mergeAnnotationArrays( + a: Array[Annotation], + b: Array[Annotation] + ): Array[Annotation] = + a ++ b.filterNot(bAnnotation => a.exists(_.annotationType() == bAnnotation.annotationType())) +} diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Classes.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Classes.scala new file mode 100644 index 0000000000..6edbf9bb26 --- /dev/null +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/Classes.scala @@ -0,0 +1,26 @@ +package com.twitter.util.reflect + +import com.twitter.util.Memoize + +object Classes { + private val PRODUCT: Class[Product] = classOf[Product] + private val OPTION: Class[Option[_]] = classOf[Option[_]] + private val LIST: Class[List[_]] = classOf[List[_]] + + /** + * Safely compute the `clazz.getSimpleName` of a given [[Class]] handling malformed + * names with a best-effort to parse the final segment after the last `.` after + * transforming all `$` to `.`. + */ + val simpleName: Class[_] => String = Memoize { clazz: Class[_] => + // class.getSimpleName can fail with an java.lan.InternalError (for malformed class/package names), + // so we attempt to deal with this with a manual translation if necessary + try { + clazz.getSimpleName + } catch { + case _: InternalError => + // replace all $ with . and return the last element + clazz.getName.replace('$', '.').split('.').last + } + } +} diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala index 8911c9a066..dba8f60c5e 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala @@ -5,6 +5,35 @@ import scala.reflect.api.TypeCreator import scala.reflect.runtime.universe._ object Types { + private[this] val PRODUCT: Type = typeOf[Product] + private[this] val OPTION: Type = typeOf[Option[_]] + private[this] val LIST: Type = typeOf[List[_]] + + /** + * Returns `true` if the given class type is considered a case class. + * True if: + * - is assignable from PRODUCT and + * - not assignable from OPTION nor LIST and + * - is not a Tuple and + * - class symbol is case class. + * + * @param clazz runtime representation of class type. + */ + def isCaseClass[T](clazz: Class[T]): Boolean = { + val tpe = asTypeTag(clazz).tpe + val classSymbol = tpe.typeSymbol.asClass + tpe <:< PRODUCT && + !(tpe <:< OPTION || tpe <:< LIST) && + !clazz.getName.startsWith("scala.Tuple") && + classSymbol.isCaseClass + } + + /** + * This is the negation of [[Types.isCaseClass]] + * Determine if a given class type is not a case class. + * Returns `true` if it is NOT considered a case class. + */ + def notCaseClass[T](clazz: Class[T]): Boolean = !isCaseClass(clazz) /** * Convert from the given `Class[T]` to a `TypeTag[T]` in the runtime universe. diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/Annotation1.java b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation1.java new file mode 100644 index 0000000000..19be3a95f6 --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation1.java @@ -0,0 +1,11 @@ +package com.twitter.util.reflect; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.ANNOTATION_TYPE, ElementType.METHOD, ElementType.FIELD, ElementType.PARAMETER}) +@Retention(RetentionPolicy.RUNTIME) +public @interface Annotation1 { +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/Annotation2.java b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation2.java new file mode 100644 index 0000000000..5744fbf3c5 --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation2.java @@ -0,0 +1,12 @@ +package com.twitter.util.reflect; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.ANNOTATION_TYPE, ElementType.METHOD, ElementType.FIELD, ElementType.PARAMETER}) +@Retention(RetentionPolicy.RUNTIME) +@MarkerAnnotation +public @interface Annotation2 { +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/Annotation3.java b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation3.java new file mode 100644 index 0000000000..8ab9103703 --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation3.java @@ -0,0 +1,14 @@ +package com.twitter.util.reflect; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.ANNOTATION_TYPE, ElementType.METHOD, ElementType.FIELD, ElementType.PARAMETER}) +@Retention(RetentionPolicy.RUNTIME) +@MarkerAnnotation +public @interface Annotation3 { + /** A value */ + String value() default "annotation3"; +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/Annotation4.java b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation4.java new file mode 100644 index 0000000000..832c335f56 --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation4.java @@ -0,0 +1,11 @@ +package com.twitter.util.reflect; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.ANNOTATION_TYPE, ElementType.METHOD, ElementType.FIELD, ElementType.PARAMETER}) +@Retention(RetentionPolicy.RUNTIME) +public @interface Annotation4 { +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/Annotation5.java b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation5.java new file mode 100644 index 0000000000..2d378948d8 --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/Annotation5.java @@ -0,0 +1,13 @@ +package com.twitter.util.reflect; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.ANNOTATION_TYPE, ElementType.METHOD, ElementType.FIELD, ElementType.PARAMETER}) +@Retention(RetentionPolicy.RUNTIME) +public @interface Annotation5 { + /** A value */ + String discriminator() default "annotation5"; +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/BUILD b/util-reflect/src/test/java/com/twitter/util/reflect/BUILD new file mode 100644 index 0000000000..fc51b55b44 --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/BUILD @@ -0,0 +1,4 @@ +java_library( + sources = ["*.java"], + compiler_option_sets = ["fatal_warnings"], +) diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/MarkerAnnotation.java b/util-reflect/src/test/java/com/twitter/util/reflect/MarkerAnnotation.java new file mode 100644 index 0000000000..cde33f2bbe --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/MarkerAnnotation.java @@ -0,0 +1,10 @@ +package com.twitter.util.reflect; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface MarkerAnnotation {} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/Thing.java b/util-reflect/src/test/java/com/twitter/util/reflect/Thing.java new file mode 100644 index 0000000000..e9b05d495e --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/Thing.java @@ -0,0 +1,15 @@ +package com.twitter.util.reflect; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@MarkerAnnotation +public @interface Thing { + /** Name of the thing */ + String value() default "thing"; +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/ThingImpl.java b/util-reflect/src/test/java/com/twitter/util/reflect/ThingImpl.java new file mode 100644 index 0000000000..f7d3260068 --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/ThingImpl.java @@ -0,0 +1,43 @@ +package com.twitter.util.reflect; + +import java.io.Serializable; +import java.lang.annotation.Annotation; + +class ThingImpl implements Thing, Serializable { + private final String value; + + public ThingImpl(String value) { + this.value = value; + } + + public String value() { + return this.value; + } + + public int hashCode() { + // This is specified in java.lang.Annotation. + return (127 * "value".hashCode()) ^ value.hashCode(); + } + + /** + * Thing specific equals + */ + public boolean equals(Object o) { + if (!(o instanceof Thing)) { + return false; + } + + Thing other = (Thing) o; + return value.equals(other.value()); + } + + public String toString() { + return "@" + Thing.class.getName() + "(value=" + value + ")"; + } + + public Class annotationType() { + return Thing.class; + } + + private static final long serialVersionUID = 0; +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/Things.java b/util-reflect/src/test/java/com/twitter/util/reflect/Things.java new file mode 100644 index 0000000000..f9ec6f8e7b --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/Things.java @@ -0,0 +1,11 @@ +package com.twitter.util.reflect; + +public final class Things { + private Things() { + } + + /** Creates a {@link Thing} annotation with {@code name} as the value. */ + public static Thing named(String name) { + return new ThingImpl(name); + } +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/Widget.java b/util-reflect/src/test/java/com/twitter/util/reflect/Widget.java new file mode 100644 index 0000000000..d19080e651 --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/Widget.java @@ -0,0 +1,15 @@ +package com.twitter.util.reflect; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +public @interface Widget { + + /** Name of the widget */ + String value() default "widget"; +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/WidgetImpl.java b/util-reflect/src/test/java/com/twitter/util/reflect/WidgetImpl.java new file mode 100644 index 0000000000..284e5f809d --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/WidgetImpl.java @@ -0,0 +1,43 @@ +package com.twitter.util.reflect; + +import java.io.Serializable; +import java.lang.annotation.Annotation; + +class WidgetImpl implements Widget, Serializable { + private final String value; + + public WidgetImpl(String value) { + this.value = value; + } + + public String value() { + return this.value; + } + + public int hashCode() { + // This is specified in java.lang.Annotation. + return (127 * "value".hashCode()) ^ value.hashCode(); + } + + /** + * Widget specific equals + */ + public boolean equals(Object o) { + if (!(o instanceof Widget)) { + return false; + } + + Widget other = (Widget) o; + return value.equals(other.value()); + } + + public String toString() { + return "@" + Widget.class.getName() + "(value=" + value + ")"; + } + + public Class annotationType() { + return Widget.class; + } + + private static final long serialVersionUID = 0; +} diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/Widgets.java b/util-reflect/src/test/java/com/twitter/util/reflect/Widgets.java new file mode 100644 index 0000000000..ca9ae572b6 --- /dev/null +++ b/util-reflect/src/test/java/com/twitter/util/reflect/Widgets.java @@ -0,0 +1,11 @@ +package com.twitter.util.reflect; + +public final class Widgets { + private Widgets() { + } + + /** Creates a {@link Widget} annotation with {@code name} as the value. */ + public static Widget named(String name) { + return new WidgetImpl(name); + } +} diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/AnnotationsTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/AnnotationsTest.scala new file mode 100644 index 0000000000..930b74b5c2 --- /dev/null +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/AnnotationsTest.scala @@ -0,0 +1,390 @@ +package com.twitter.util.reflect + +import com.twitter.util.reflect.testclasses._ +import java.lang.annotation.Annotation +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner +import scala.collection.Map + +@RunWith(classOf[JUnitRunner]) +class AnnotationsTest extends AnyFunSuite with Matchers { + + test("Annotations#filterIfAnnotationPresent") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[CaseClassOneTwo]) + annotationMap.isEmpty should be(false) + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + + val found = Annotations.filterIfAnnotationPresent[MarkerAnnotation](annotations) + found.length should be(1) + found.head.annotationType should equal(classOf[Annotation2]) + } + + test("Annotations#filterAnnotations") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[CaseClassThreeFour]) + annotationMap.isEmpty should be(false) + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + val filterSet: Set[Class[_ <: Annotation]] = Set(classOf[Annotation4]) + val found = Annotations.filterAnnotations(filterSet, annotations) + found.length should be(1) + found.head.annotationType should equal(classOf[Annotation4]) + } + + test("Annotations#findAnnotation") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[CaseClassThreeFour]) + annotationMap.isEmpty should be(false) + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + Annotations.findAnnotation(classOf[Annotation1], annotations) should be(None) // not found + val found = Annotations.findAnnotation(classOf[Annotation3], annotations) + found.isDefined should be(true) + found.get.annotationType() should equal(classOf[Annotation3]) + } + + test("Annotations#findAnnotation by type") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[CaseClassOneTwoThreeFour]) + annotationMap.isEmpty should be(false) + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + Annotations.findAnnotation[MarkerAnnotation](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation1](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation2](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation3](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation4](annotations).isDefined should be(true) + } + + test("Annotations#annotationEquals") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[CaseClassOneTwoThreeFour]) + annotationMap.isEmpty should be(false) + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + val found = Annotations.findAnnotation[Annotation1](annotations) + found.isDefined should be(true) + + val annotation = found.get + Annotations.annotationEquals[Annotation1](annotation) should be(true) + } + + test("Annotations#isAnnotationPresent") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[CaseClassOneTwoThreeFour]) + annotationMap.isEmpty should be(false) + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + val annotation1 = Annotations.findAnnotation[Annotation1](annotations).get + val annotation2 = Annotations.findAnnotation[Annotation2](annotations).get + val annotation3 = Annotations.findAnnotation[Annotation3](annotations).get + val annotation4 = Annotations.findAnnotation[Annotation4](annotations).get + + Annotations.isAnnotationPresent[MarkerAnnotation](annotation1) should be(false) + Annotations.isAnnotationPresent[MarkerAnnotation](annotation2) should be(true) + Annotations.isAnnotationPresent[MarkerAnnotation](annotation3) should be(true) + Annotations.isAnnotationPresent[MarkerAnnotation](annotation4) should be(false) + } + + test("Annotations#findAnnotations") { + var found: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[WithThings]) + found.isEmpty should be(false) + var annotations = found.flatMap { case (_, annotations) => annotations.toList }.toArray + annotations.length should equal(4) + + found = Annotations.findAnnotations(classOf[WithWidgets]) + found.isEmpty should be(false) + annotations = found.flatMap { case (_, annotations) => annotations.toList }.toArray + annotations.length should equal(4) + + found = Annotations.findAnnotations(classOf[CaseClassOneTwo]) + found.isEmpty should be(false) + annotations = found.flatMap { case (_, annotations) => annotations.toList }.toArray + annotations.length should equal(2) + + found = Annotations.findAnnotations(classOf[CaseClassThreeFour]) + found.isEmpty should be(false) + annotations = found.flatMap { case (_, annotations) => annotations.toList }.toArray + annotations.length should equal(2) + + found = Annotations.findAnnotations(classOf[CaseClassOneTwoThreeFour]) + found.isEmpty should be(false) + annotations = found.flatMap { case (_, annotations) => annotations.toList }.toArray + annotations.length should equal(4) + + found = Annotations.findAnnotations(classOf[CaseClassOneTwoWithFields]) + found.isEmpty should be(false) + annotations = found.flatMap { case (_, annotations) => annotations.toList }.toArray + annotations.length should equal(2) + + found = Annotations.findAnnotations(classOf[CaseClassOneTwoWithAnnotatedField]) + found.isEmpty should be(false) + annotations = found.flatMap { case (_, annotations) => annotations.toList }.toArray + annotations.length should equal(3) + + found = Annotations.findAnnotations(classOf[CaseClassThreeFourAncestorOneTwo]) + found.isEmpty should be(false) + annotations = found.flatMap { case (_, annotations) => annotations.toList }.toArray + annotations.length should equal(4) + + found = Annotations.findAnnotations(classOf[CaseClassAncestorOneTwo]) + found.isEmpty should be(false) + annotations = found.flatMap { case (_, annotations) => annotations.toList }.toArray + annotations.length should equal(3) + } + + test("Annotations.findAnnotations error") { + // parameter types are ignored + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations( + classOf[CaseClassOneTwoWithFields], + Seq(classOf[Boolean], classOf[Double])) + annotationMap.isEmpty should be(false) + annotationMap.size should equal(2) + annotationMap("one").length should equal(1) + annotationMap("one").head.annotationType() should equal(classOf[Annotation1]) + annotationMap("two").length should equal(1) + annotationMap("two").head.annotationType() should equal(classOf[Annotation2]) + } + + test("Annotations#getValueIfAnnotatedWith") { + val annotationsMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[WithThings]) + annotationsMap.isEmpty should be(false) + val annotations = annotationsMap.flatMap { case (_, annotations) => annotations.toList }.toArray + val annotation1 = Annotations.findAnnotation[Annotation1](annotations).get + val annotation2 = Annotations.findAnnotation[Annotation2](annotations).get + + // @Annotation1 is not annotated with @MarkerAnnotation + Annotations.getValueIfAnnotatedWith[MarkerAnnotation](annotation1).isDefined should be(false) + // @Annotation2 is annotated with @MarkerAnnotation but does not define a value() method + Annotations.getValueIfAnnotatedWith[MarkerAnnotation](annotation2).isDefined should be(false) + + val things = Annotations.filterAnnotations(Set(classOf[Thing]), annotations) + things.foreach { thing => + Annotations.getValueIfAnnotatedWith[MarkerAnnotation](thing).isDefined should be(true) + } + + val fooThingAnnotation = Things.named("foo") + val result = Annotations.getValueIfAnnotatedWith[MarkerAnnotation](fooThingAnnotation) + result.isDefined should be(true) + result.get should equal("foo") + } + + test("Annotations#getValue") { + val annotationsMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[CaseClassThreeFour]) + annotationsMap.isEmpty should be(false) + val annotations = annotationsMap.flatMap { case (_, annotations) => annotations.toList }.toArray + val annotation3 = Annotations.findAnnotation[Annotation3](annotations).get + val annotation4 = Annotations.findAnnotation[Annotation4](annotations).get + + val annotation3Value = Annotations.getValue(annotation3) + annotation3Value.isDefined should be(true) + annotation3Value.get should equal("annotation3") + + // Annotation4 has no value + Annotations.getValue(annotation4).isDefined should be(false) + + val clazzFiveAnnotationsMap = Annotations.findAnnotations(classOf[CaseClassFive]) + val clazzFiveAnnotations = clazzFiveAnnotationsMap.flatMap { + case (_, annotations) => annotations.toList + }.toArray + val annotation5 = + Annotations.findAnnotation[Annotation5](clazzFiveAnnotations).get + // Annotation5 has a uniquely named method + val annotation5Value = Annotations.getValue(annotation5, "discriminator") + annotation5Value.isDefined should be(true) + annotation5Value.get should equal("annotation5") + + // directly look up a value on an annotation + val fooThingAnnotation = Things.named("foo") + val fooThingAnnotationValue = Annotations.getValue(fooThingAnnotation) + fooThingAnnotationValue.isDefined should be(true) + fooThingAnnotationValue.get should equal("foo") + } + + test("Annotations#secondaryConstructor") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations( + classOf[WithSecondaryConstructor], + Seq(classOf[String], classOf[String])) + + annotationMap.isEmpty should be(false) + annotationMap.size should equal(2) + annotationMap("three").length should equal(1) + annotationMap("three").head.annotationType() should equal(classOf[Annotation3]) + annotationMap("four").length should equal(1) + annotationMap("four").head.annotationType() should equal(classOf[Annotation4]) + annotationMap.get("one") should be(None) // not found + annotationMap.get("two") should be(None) // not found + + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + Annotations.findAnnotation[MarkerAnnotation](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation1](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation2](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation3](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation4](annotations).isDefined should be(true) + } + + test("Annotations#secondaryConstructor 1") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations( + classOf[WithSecondaryConstructor], + Seq(classOf[Int], classOf[Int])) + + annotationMap.isEmpty should be(false) + annotationMap.size should equal(2) + annotationMap("one").length should equal(1) + annotationMap("one").head.annotationType() should equal(classOf[Annotation1]) + annotationMap("two").length should equal(1) + annotationMap("two").head.annotationType() should equal(classOf[Annotation2]) + annotationMap.get("three") should be(None) // not found + annotationMap.get("four") should be(None) // not found + + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + Annotations.findAnnotation[MarkerAnnotation](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation1](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation2](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation3](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation4](annotations) should be(None) // not found + } + + test("Annotations#secondaryConstructor 2") { + val annotationMap: Map[String, Array[Annotation]] = Annotations.findAnnotations( + classOf[StaticSecondaryConstructor], + Seq(classOf[String], classOf[String]) + ) + annotationMap.isEmpty should be(false) + annotationMap.size should equal(2) + annotationMap("one").length should equal(1) + annotationMap("one").head.annotationType() should equal(classOf[Annotation1]) + annotationMap("two").length should equal(1) + annotationMap("two").head.annotationType() should equal(classOf[Annotation2]) + } + + test("Annotations#secondaryConstructor 3") { + val annotationMap: Map[String, Array[Annotation]] = Annotations.findAnnotations( + classOf[StaticSecondaryConstructor], + Seq(classOf[Int], classOf[Int]) + ) + + // secondary int "constructor" is not used/scanned as it is an Object apply method which + // is a factory method and not a constructor. + annotationMap.isEmpty should be(false) + annotationMap.size should equal(2) + annotationMap("one").length should equal(1) + annotationMap("one").head.annotationType() should equal(classOf[Annotation1]) + annotationMap("two").length should equal(1) + annotationMap("two").head.annotationType() should equal(classOf[Annotation2]) + } + + test("Annotations#secondaryConstructor 4") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[StaticSecondaryConstructor]) + + annotationMap.isEmpty should be(false) + annotationMap.size should equal(2) + annotationMap("one").length should equal(1) + annotationMap("one").head.annotationType() should equal(classOf[Annotation1]) + annotationMap("two").length should equal(1) + annotationMap("two").head.annotationType() should equal(classOf[Annotation2]) + + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + Annotations.findAnnotation[MarkerAnnotation](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation1](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation2](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation3](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation4](annotations) should be(None) // not found + } + + test("Annotations#secondaryConstructor 5") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[StaticSecondaryConstructorWithMethodAnnotation]) + annotationMap.isEmpty should be(false) + annotationMap.size should equal(2) + annotationMap("one").length should equal(1) + annotationMap("one").head.annotationType() should equal(classOf[Annotation1]) + annotationMap("two").length should equal(1) + annotationMap("two").head.annotationType() should equal(classOf[Annotation2]) + annotationMap.get("widget1") should be(None) + + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + val annotation1 = Annotations.findAnnotation[Annotation1](annotations).get + val annotation2 = Annotations.findAnnotation[Annotation2](annotations).get + + Annotations.findAnnotation[MarkerAnnotation](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation1](annotations) should be(Some(annotation1)) + Annotations.findAnnotation[Annotation2](annotations) should be(Some(annotation2)) + Annotations.findAnnotation[Annotation3](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation4](annotations) should be(None) // not found + val widgetAnnotationOpt = Annotations.findAnnotation[Widget](annotations) + widgetAnnotationOpt.isDefined should be(false) + } + + test("Annotations#secondaryConstructor 6") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[StaticSecondaryConstructorWithMethodAnnotation]) + + annotationMap.isEmpty should be(false) + annotationMap.size should equal(2) + annotationMap("one").length should equal(1) + annotationMap("one").head.annotationType() should equal(classOf[Annotation1]) + annotationMap("two").length should equal(1) + annotationMap("two").head.annotationType() should equal(classOf[Annotation2]) + annotationMap.get("widget1") should be(None) + + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + Annotations.findAnnotation[MarkerAnnotation](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation1](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation2](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation3](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation4](annotations) should be(None) // not found + val widgetAnnotationOpt = Annotations.findAnnotation[Widget](annotations) + widgetAnnotationOpt.isDefined should be(false) + } + + test("Annotations#no constructor should not fail") { + // we expressly want this to no-op rather than fail outright + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations(classOf[NoConstructor]) + + annotationMap.isEmpty should be(true) + } + + test("Annotations#generic types") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations( + classOf[GenericTestCaseClass[Int]], + Seq(classOf[Object]) + ) // generic types resolve to Object + + annotationMap.isEmpty should be(false) + annotationMap.size should equal(1) + annotationMap("one").length should equal(1) + annotationMap("one").head.annotationType() should equal(classOf[Annotation1]) + + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + Annotations.findAnnotation[MarkerAnnotation](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation1](annotations).isDefined should be(true) + } + + test("Annotations#generic types 1") { + val annotationMap: Map[String, Array[Annotation]] = + Annotations.findAnnotations( + classOf[GenericTestCaseClassWithMultipleArgs[Int]], + Seq(classOf[Object], classOf[Int]) + ) // generic types resolve to Object + + annotationMap.isEmpty should be(false) + annotationMap.size should equal(2) + annotationMap("one").length should equal(1) + annotationMap("one").head.annotationType() should equal(classOf[Annotation1]) + annotationMap("two").length should equal(1) + annotationMap("two").head.annotationType() should equal(classOf[Annotation2]) + + val annotations = annotationMap.flatMap { case (_, annotations) => annotations.toList }.toArray + Annotations.findAnnotation[MarkerAnnotation](annotations) should be(None) // not found + Annotations.findAnnotation[Annotation1](annotations).isDefined should be(true) + Annotations.findAnnotation[Annotation2](annotations).isDefined should be(true) + } +} diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD index d9a7039540..224a0035b0 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -7,5 +7,6 @@ junit_tests( "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", "util/util-reflect/src/main/scala/com/twitter/util/reflect", + "util/util-reflect/src/test/java/com/twitter/util/reflect", ], ) diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/ClassesTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/ClassesTest.scala new file mode 100644 index 0000000000..e141d75154 --- /dev/null +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/ClassesTest.scala @@ -0,0 +1,100 @@ +package com.twitter.util.reflect + +import com.twitter.util.reflect.testclasses.has_underscore.ClassB +import com.twitter.util.reflect.testclasses.number_1.FooNumber +import com.twitter.util.reflect.testclasses.okNaming.Ok +import com.twitter.util.reflect.testclasses.ver2_3.{Ext, Response, This$Breaks$In$ManyWays} +import com.twitter.util.reflect.testclasses.{ClassA, Request} +import com.twitter.util.reflect.DoEverything.{DoEverything$Client, ServicePerEndpoint} +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class ClassesTest extends AnyFunSuite with Matchers { + + test("Classes#simpleName") { + // these cause class.getSimpleName to blow up, ensure we don't + Classes.simpleName(classOf[Ext]) should equal("Ext") + try { + classOf[Ext].getSimpleName should equal("Ext") + } catch { + case _: InternalError => + // do nothing -- fails in JDK8 but not JDK11 + } + Classes.simpleName(classOf[Response]) should equal("Response") + try { + classOf[Response].getSimpleName should equal("Response") + } catch { + case _: InternalError => + // do nothing -- fails in JDK8 but not JDK11 + } + + // show we don't blow up + Classes.simpleName(classOf[Ok]) should equal("Ok") + try { + classOf[Ok].getSimpleName should equal("Ok") + } catch { + case _: InternalError => + // do nothing -- fails in JDK8 but not JDK11 + } + + // ensure we don't blow up + Classes.simpleName(classOf[ClassB]) should equal("ClassB") + try { + classOf[ClassB].getSimpleName should equal("ClassB") + } catch { + case _: InternalError => + // do nothing -- fails in JDK8 but not JDK11 + } + + // this causes class.getSimpleName to blow up, ensure we don't + Classes.simpleName(classOf[FooNumber]) should equal("FooNumber") + try { + classOf[FooNumber].getSimpleName should equal("FooNumber") + } catch { + case _: InternalError => + // do nothing -- fails in JDK8 but not JDK11 + } + + Classes.simpleName(classOf[Fungible[_]]) should equal("Fungible") + Classes.simpleName(classOf[BarService]) should equal("BarService") + Classes.simpleName(classOf[ToBarService]) should equal("ToBarService") + Classes.simpleName(classOf[java.lang.Object]) should equal("Object") + Classes.simpleName(classOf[GeneratedFooService]) should equal("GeneratedFooService") + Classes.simpleName(classOf[com.twitter.util.reflect.DoEverything[_]]) should equal( + "DoEverything") + Classes.simpleName(com.twitter.util.reflect.DoEverything.getClass) should equal("DoEverything$") + Classes.simpleName(classOf[DoEverything.ServicePerEndpoint]) should equal("ServicePerEndpoint") + Classes.simpleName(classOf[ServicePerEndpoint]) should equal("ServicePerEndpoint") + Classes.simpleName(classOf[ClassA]) should equal("ClassA") + Classes.simpleName(classOf[ClassA]) should equal(classOf[ClassA].getSimpleName) + Classes.simpleName(classOf[Request]) should equal("Request") + Classes.simpleName(classOf[Request]) should equal(classOf[Request].getSimpleName) + + Classes.simpleName(classOf[java.lang.String]) should equal("String") + Classes.simpleName(classOf[java.lang.String]) should equal( + classOf[java.lang.String].getSimpleName) + Classes.simpleName(classOf[String]) should equal("String") + Classes.simpleName(classOf[String]) should equal(classOf[String].getSimpleName) + Classes.simpleName(classOf[Int]) should equal("int") + Classes.simpleName(classOf[Int]) should equal(classOf[Int].getSimpleName) + + Classes.simpleName(classOf[DoEverything.DoEverything$Client]) should equal( + "DoEverything$Client") + Classes.simpleName(classOf[DoEverything$Client]) should equal("DoEverything$Client") + Classes.simpleName(classOf[DoEverything$Client]) should equal( + classOf[DoEverything$Client].getSimpleName) + + val manyWays = Classes.simpleName(classOf[This$Breaks$In$ManyWays]) + (manyWays == "ManyWays" /* JDK8 */ || + manyWays == "This$Breaks$In$ManyWays" /* JDK11 */ ) should be(true) + try { + classOf[This$Breaks$In$ManyWays].getSimpleName should equal("This$Breaks$In$ManyWays") + } catch { + case _: InternalError => + // do nothing -- fails in JDK8 but not JDK11 + } + } +} diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala index e1a002d8af..6c5d89c8c6 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala @@ -1,25 +1,19 @@ package com.twitter.util.reflect +import com.twitter.util.{Awaitable, FuturePool} +import com.twitter.util.reflect.testclasses._ +import com.twitter.util.reflect.testclasses.has_underscore.ClassB +import com.twitter.util.reflect.testclasses.number_1.FooNumber +import com.twitter.util.reflect.testclasses.okNaming.Ok +import com.twitter.util.reflect.testclasses.ver2_3.{Ext, Response} import org.junit.runner.RunWith import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatestplus.junit.JUnitRunner import scala.reflect.runtime.universe._ -object TypesTest { - trait TestTraitA - trait TestTraitB - - case class Foo[T](data: T) - case class Baz[U, T](bar: U, foo: Foo[T]) - case class Bez[I, J](m: Map[I, J]) - - trait TypedTrait[A, B] -} - @RunWith(classOf[JUnitRunner]) class TypesTest extends AnyFunSuite with Matchers { - import TypesTest._ test("Types#asTypeTag handles classes") { typeTagEquals(classOf[String], Types.asTypeTag(classOf[String])) @@ -67,7 +61,6 @@ class TypesTest extends AnyFunSuite with Matchers { classOf[Nothing].isAssignableFrom(Types.runtimeClass[scala.runtime.Nothing$]) // `Any` is special and can result in a ClassNotFoundException //Types.runtimeClass[Any] - } test("Types#eq") { @@ -127,6 +120,119 @@ class TypesTest extends AnyFunSuite with Matchers { } } + test("Types#isCaseClass") { + Types.isCaseClass(classOf[Ext]) should be(true) + Types.isCaseClass(classOf[Response]) should be(true) + + Types.isCaseClass(classOf[Ok]) should be(true) + + Types.isCaseClass(classOf[ClassB]) should be(true) + + Types.isCaseClass(classOf[FooNumber]) should be(true) + + Types.isCaseClass(classOf[ClassA]) should be(true) + Types.isCaseClass(classOf[Request]) should be(true) + + Types.isCaseClass(classOf[Awaitable[_]]) should be(false) + Types.isCaseClass(classOf[FuturePool]) should be(false) + + Types.isCaseClass(classOf[AnyRef]) should be(false) + Types.isCaseClass(classOf[AnyVal]) should be(false) + Types.isCaseClass(classOf[Unit]) should be(false) + Types.isCaseClass(classOf[Byte]) should be(false) + Types.isCaseClass(classOf[Short]) should be(false) + Types.isCaseClass(classOf[Char]) should be(false) + Types.isCaseClass(classOf[Int]) should be(false) + Types.isCaseClass(classOf[java.lang.Integer]) should be(false) + Types.isCaseClass(classOf[Float]) should be(false) + Types.isCaseClass(classOf[Double]) should be(false) + Types.isCaseClass(classOf[Boolean]) should be(false) + Types.isCaseClass(classOf[java.lang.Object]) should be(false) + Types.isCaseClass(classOf[Any]) should be(false) + Types.isCaseClass(classOf[Null]) should be(false) + Types.isCaseClass(classOf[Nothing]) should be(false) + Types.isCaseClass(classOf[String]) should be(false) + Types.isCaseClass(classOf[java.lang.String]) should be(false) + + Types.isCaseClass(classOf[Seq[_]]) should be(false) + Types.isCaseClass(classOf[Seq[Ext]]) should be(false) + Types.isCaseClass(classOf[List[_]]) should be(false) + Types.isCaseClass(classOf[List[Ext]]) should be(false) + Types.isCaseClass(classOf[Iterable[_]]) should be(false) + Types.isCaseClass(classOf[Iterable[Ext]]) should be(false) + Types.isCaseClass(classOf[Array[_]]) should be(false) + Types.isCaseClass(classOf[Array[Ext]]) should be(false) + Types.isCaseClass(classOf[Option[_]]) should be(false) + Types.isCaseClass(classOf[Option[ClassB]]) should be(false) + Types.isCaseClass(classOf[Either[_, _]]) should be(false) + Types.isCaseClass(classOf[Either[Request, Response]]) should be(false) + Types.isCaseClass(classOf[Map[_, _]]) should be(false) + Types.isCaseClass(classOf[Map[ClassA, FooNumber]]) should be(false) + Types.isCaseClass(classOf[Tuple1[_]]) should be(false) + Types.isCaseClass(classOf[Tuple2[_, _]]) should be(false) + Types.isCaseClass(classOf[(_, _, _)]) should be(false) + Types.isCaseClass(classOf[(_, _, _, _)]) should be(false) + Types.isCaseClass(classOf[(_, _, _, _, _)]) should be(false) + Types.isCaseClass(classOf[(_, _, _, _, _, _)]) should be(false) + Types.isCaseClass(classOf[Tuple7[_, _, _, _, _, _, _]]) should be(false) + Types.isCaseClass(classOf[Tuple8[_, _, _, _, _, _, _, _]]) should be(false) + Types.isCaseClass(classOf[Tuple9[_, _, _, _, _, _, _, _, _]]) should be(false) + Types.isCaseClass(classOf[Tuple10[_, _, _, _, _, _, _, _, _, _]]) should be(false) + } + + test("Types#notCaseClass") { + Types.notCaseClass(classOf[Ext]) should be(false) + Types.notCaseClass(classOf[Response]) should be(false) + + Types.notCaseClass(classOf[Ok]) should be(false) + + Types.notCaseClass(classOf[ClassB]) should be(false) + + Types.notCaseClass(classOf[FooNumber]) should be(false) + + Types.notCaseClass(classOf[ClassA]) should be(false) + Types.notCaseClass(classOf[Request]) should be(false) + + Types.notCaseClass(classOf[Awaitable[_]]) should be(true) + Types.notCaseClass(classOf[FuturePool]) should be(true) + + Types.notCaseClass(classOf[java.lang.String]) should be(true) + Types.notCaseClass(classOf[String]) should be(true) + Types.notCaseClass(classOf[Int]) should be(true) + + Types.notCaseClass(classOf[String]) should be(true) + Types.notCaseClass(classOf[Boolean]) should be(true) + Types.notCaseClass(classOf[java.lang.Integer]) should be(true) + Types.notCaseClass(classOf[Int]) should be(true) + Types.notCaseClass(classOf[Double]) should be(true) + Types.notCaseClass(classOf[Float]) should be(true) + Types.notCaseClass(classOf[Long]) should be(true) + Types.notCaseClass(classOf[Seq[_]]) should be(true) + Types.notCaseClass(classOf[Seq[Ext]]) should be(true) + Types.notCaseClass(classOf[List[_]]) should be(true) + Types.notCaseClass(classOf[List[Ext]]) should be(true) + Types.notCaseClass(classOf[Iterable[_]]) should be(true) + Types.notCaseClass(classOf[Iterable[Ext]]) should be(true) + Types.notCaseClass(classOf[Array[_]]) should be(true) + Types.notCaseClass(classOf[Array[Ext]]) should be(true) + Types.notCaseClass(classOf[Option[_]]) should be(true) + Types.notCaseClass(classOf[Option[ClassB]]) should be(true) + Types.notCaseClass(classOf[Either[_, _]]) should be(true) + Types.notCaseClass(classOf[Either[Request, Response]]) should be(true) + Types.notCaseClass(classOf[Map[_, _]]) should be(true) + Types.notCaseClass(classOf[Map[ClassA, FooNumber]]) should be(true) + Types.notCaseClass(classOf[Tuple1[_]]) should be(true) + Types.notCaseClass(classOf[Tuple2[_, _]]) should be(true) + Types.notCaseClass(classOf[(_, _, _)]) should be(true) + Types.notCaseClass(classOf[(_, _, _, _)]) should be(true) + Types.notCaseClass(classOf[(_, _, _, _, _)]) should be(true) + Types.notCaseClass(classOf[(_, _, _, _, _, _)]) should be(true) + Types.notCaseClass(classOf[Tuple7[_, _, _, _, _, _, _]]) should be(true) + Types.notCaseClass(classOf[Tuple8[_, _, _, _, _, _, _, _]]) should be(true) + Types.notCaseClass(classOf[Tuple9[_, _, _, _, _, _, _, _, _]]) should be(true) + Types.notCaseClass(classOf[Tuple10[_, _, _, _, _, _, _, _, _, _]]) should be(true) + } + private[this] def typeTagEquals[T](clazz: Class[T], typeTag: TypeTag[T]): Boolean = clazz.isAssignableFrom(typeTag.mirror.runtimeClass(typeTag.tpe.typeSymbol.asClass)) } diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/testclasses.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/testclasses.scala new file mode 100644 index 0000000000..a316e73bf5 --- /dev/null +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/testclasses.scala @@ -0,0 +1,130 @@ +package com.twitter.util.reflect + +import com.twitter.util.Future + +trait Fungible[+T] +trait BarService +trait ToBarService { + def toBarService: BarService +} + +abstract class GeneratedFooService + +trait DoEverything[+MM[_]] extends GeneratedFooService + +object DoEverything extends GeneratedFooService { self => + + trait ServicePerEndpoint extends ToBarService with Fungible[ServicePerEndpoint] + + class DoEverything$Client extends DoEverything[Future] +} + +object testclasses { + trait TestTraitA + trait TestTraitB + + case class Foo[T](data: T) + case class Baz[U, T](bar: U, foo: Foo[T]) + case class Bez[I, J](m: Map[I, J]) + + trait TypedTrait[A, B] + + object ver2_3 { + // this "package" naming will blow up class.getSimpleName + // packages with underscore then a number make Java unhappy + final case class Ext(a: String, b: String) + case class Response(a: String, b: String) + + class This$Breaks$In$ManyWays + } + + object okNaming { + case class Ok(a: String, b: String) + } + + object has_underscore { + case class ClassB(a: String, b: String) + } + + object number_1 { + // this "package" naming will blow up class.getSimpleName + // packages with underscore then number make Java unhappy + final case class FooNumber(a: String) + } + + final case class ClassA(a: String, b: String) + case class Request(a: String) + + class Bar + + case class NoConstructor() + + case class CaseClassOneTwo(@Annotation1 one: String, @Annotation2 two: String) + case class CaseClassOneTwoWithFields(@Annotation1 one: String, @Annotation2 two: String) { + val city: String = "San Francisco" + val state: String = "California" + } + case class CaseClassOneTwoWithAnnotatedField(@Annotation1 one: String, @Annotation2 two: String) { + @Annotation3 val three: String = "three" + } + case class CaseClassThreeFour(@Annotation3 three: String, @Annotation4 four: String) + case class CaseClassFive(@Annotation5 five: String) + case class CaseClassOneTwoThreeFour( + @Annotation1 one: String, + @Annotation2 two: String, + @Annotation3 three: String, + @Annotation4 four: String) + + case class WithThings( + @Annotation1 @Thing("thing1") thing1: String, + @Annotation2 @Thing("thing2") thing2: String) + case class WithWidgets( + @Annotation3 @Widget("widget1") widget1: String, + @Annotation4 @Widget("widget2") widget2: String) + + case class WithSecondaryConstructor( + @Annotation1 one: Int, + @Annotation2 two: Int) { + def this(@Annotation3 three: String, @Annotation4 four: String) { + this(three.toInt, four.toInt) + } + } + + object StaticSecondaryConstructor { + // NOTE: this is a factory method and not a constructor, so annotations will not be picked up + def apply(@Annotation3 three: String, @Annotation4 four: String): StaticSecondaryConstructor = + StaticSecondaryConstructor(three.toInt, four.toInt) + } + case class StaticSecondaryConstructor(@Annotation1 one: Int, @Annotation2 two: Int) + + object StaticSecondaryConstructorWithMethodAnnotation { + def apply(@Annotation3 three: String, @Annotation4 four: String): StaticSecondaryConstructor = + StaticSecondaryConstructor(three.toInt, four.toInt) + } + case class StaticSecondaryConstructorWithMethodAnnotation( + @Annotation1 one: Int, + @Annotation2 two: Int) { + // will not be found as Annotations only scans for declared field annotations, this is a method + @Widget("widget1") def widget1: String = "this is widget 1 method" + } + + case class GenericTestCaseClass[T](@Annotation1 one: T) + case class GenericTestCaseClassWithMultipleArgs[T](@Annotation1 one: T, @Annotation2 two: Int) + + trait AncestorWithAnnotations { + @Annotation1 def one: String + @Annotation2 def two: String + } + + case class CaseClassThreeFourAncestorOneTwo( + one: String, + two: String, + @Annotation3 three: String, + @Annotation4 four: String) + extends AncestorWithAnnotations + + case class CaseClassAncestorOneTwo(@Annotation5 five: String) extends AncestorWithAnnotations { + override val one: String = "one" + override val two: String = "two" + } +} From c2e926770238da3cb409e2c97b9e13ded7840dd3 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 24 Mar 2021 17:44:55 +0000 Subject: [PATCH 456/796] util/doc - Remove link to capturepoints removed with dropping of Scala 2.11 support Problem/Solution util-capturepoints documentation was removed with removal of Scala 2.11 support. Let's remove the link from the toctree as well. Differential Revision: https://phabricator.twitter.biz/D639688 --- doc/src/sphinx/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/src/sphinx/index.rst b/doc/src/sphinx/index.rst index f91666777c..713d567d5f 100644 --- a/doc/src/sphinx/index.rst +++ b/doc/src/sphinx/index.rst @@ -13,7 +13,6 @@ Guides util-stats/index util-cookbook/index - util-capturepoints/index Changelog From 0099049170bc068b8c829a54f900bc72f0486fe0 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Thu, 25 Mar 2021 19:45:41 +0000 Subject: [PATCH 457/796] {finagle, finatra, scrooge, util}: Remove remaining scala 2.11 compatibility code Problem: We've dropped scala 2.11 support from our OSS libraries, but we still have a lot of code that exists specifically for 2.11 compatibility Solution: Remove 2.11 compatibility code where possible JIRA Issues: CSL-10789 Differential Revision: https://phabricator.twitter.biz/D637394 --- .../main/java/com/twitter/util/javainterop/Scala.java | 8 ++------ .../scala/com/twitter/util/mock/MockitoTraitTest.scala | 10 +--------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/util-core/src/main/java/com/twitter/util/javainterop/Scala.java b/util-core/src/main/java/com/twitter/util/javainterop/Scala.java index 625868033a..6990225275 100644 --- a/util-core/src/main/java/com/twitter/util/javainterop/Scala.java +++ b/util-core/src/main/java/com/twitter/util/javainterop/Scala.java @@ -31,9 +31,7 @@ public static scala.collection.immutable.Seq asImmutableSeq( if (jList == null) { return scala.collection.immutable.Seq$.MODULE$.empty(); } else { - //TODO: After dropping support for 2.11.x, replace with - //JavaConverters.asScalaBuffer(jlist).toList(); - return JavaConverters.asScalaBufferConverter(jList).asScala().toList(); + return JavaConverters.asScalaBuffer(jList).toList(); } } @@ -52,9 +50,7 @@ public static scala.collection.immutable.Set asImmutableSet( if (jSet == null) { return scala.collection.immutable.Set$.MODULE$.empty(); } else { - //TODO: After dropping support for 2.11.x, replace with - //JavaConverters.asScalaSet(jSet).toSet - return JavaConverters.asScalaSetConverter(jSet).asScala().toSet(); + return JavaConverters.asScalaSet(jSet).toSet(); } } diff --git a/util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitTest.scala b/util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitTest.scala index a5a3f48361..1a37c2b551 100644 --- a/util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitTest.scala +++ b/util-mock/src/test/scala/com/twitter/util/mock/MockitoTraitTest.scala @@ -95,11 +95,7 @@ object MockitoTraitTest { def traitMethodWithDefaultArgs(defaultArg: Int = 30, anotherDefault: String = "hola"): Int = -1 } - // this is a case class in order to have equals/hashcode properly defined which - // is necessary for the eqTo syntax to work properly in Scala 2.11 - case class ValueClass(v: String) extends AnyVal { - override def toString = s"ValueClass($v)" - } + implicit class ValueClass(val v: String) extends AnyVal case class ValueCaseClassInt(v: Int) extends AnyVal case class ValueCaseClassString(v: String) extends AnyVal case class GenericValue[T](v: Int) extends AnyVal @@ -666,10 +662,6 @@ class MockitoTraitTest extends AnyFunSuite with Matchers with Mockito { Await.result(mockFutureFn(), 2.seconds) should equal(42) } - // Function0 types seem to not play well with Mockito and Future.apply - // in Scala 2.11 so we use Future.value here -- this test is perhaps a bit - // tautological since mockFn() is called immediately in Future.value but it - // did expose a difference between Scala 2.11 and Scala 2.12. test("works with Future whenReady 1") { val mockFn = mock[() => Boolean] mockFn() returns true From e90c5c61b32b7227c763b22db435b96c5f8a816e Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 25 Mar 2021 20:11:53 +0000 Subject: [PATCH 458/796] util-reflect: Rename c.t.util.reflect functions to `equals` Problem The method `Annotations#annotationEquals` is a bit of a mouthful and redundant. Additionally, `Types#eq` is a bit misleading. Solution Let's update from `annotationsEquals` to the shorted but still descriptive `equals` and also update `Types#eq` to `Types#equals`. Result Short and accurate names for both methods. Differential Revision: https://phabricator.twitter.biz/D640386 --- CHANGELOG.rst | 3 ++ .../twitter/util/reflect/Annotations.scala | 2 +- .../com/twitter/util/reflect/Types.scala | 2 +- .../util/reflect/AnnotationsTest.scala | 4 +- .../com/twitter/util/reflect/TypesTest.scala | 52 +++++++++---------- 5 files changed, 33 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7a075be1f0..585cc98b83 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,9 @@ Runtime Behavior Changes Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ +* util: Rename `c.t.util.reflect.Annotations#annotationEquals` to `c.t.util.reflect.Annotations#equals` + and `c.t.util.reflect.Types.eq` to `c.t.util.reflect.Types.equals`. ``PHAB_ID=D640386`` + * util: Builds are now only supported for Scala 2.12+ ``PHAB_ID=D631091`` * util-reflect: Remove deprecated `c.t.util.reflect.Proxy`. There is no library replacement. diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Annotations.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Annotations.scala index ab719907b1..69467c4559 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/Annotations.scala +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/Annotations.scala @@ -170,7 +170,7 @@ object Annotations { * * @return true if the given [[Annotation]] is of type [[A]], false otherwise. */ - def annotationEquals[A <: Annotation: ClassTag](annotation: Annotation): Boolean = + def equals[A <: Annotation: ClassTag](annotation: Annotation): Boolean = annotation.annotationType() == classTag[A].runtimeClass.asInstanceOf[Class[A]] /** diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala index dba8f60c5e..6efffeda93 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala @@ -98,7 +98,7 @@ object Types { * @return true if the TypeSymbols are equivalent. * @see [[scala.reflect.api.Symbols]] */ - def eq[T](thatTypeTag: TypeTag[_])(implicit thisTypeTag: TypeTag[T]): Boolean = + def equals[T](thatTypeTag: TypeTag[_])(implicit thisTypeTag: TypeTag[T]): Boolean = thisTypeTag.tpe.typeSymbol == thatTypeTag.tpe.typeSymbol /** diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/AnnotationsTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/AnnotationsTest.scala index 930b74b5c2..096c0d4f95 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/AnnotationsTest.scala +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/AnnotationsTest.scala @@ -56,7 +56,7 @@ class AnnotationsTest extends AnyFunSuite with Matchers { Annotations.findAnnotation[Annotation4](annotations).isDefined should be(true) } - test("Annotations#annotationEquals") { + test("Annotations#equals") { val annotationMap: Map[String, Array[Annotation]] = Annotations.findAnnotations(classOf[CaseClassOneTwoThreeFour]) annotationMap.isEmpty should be(false) @@ -65,7 +65,7 @@ class AnnotationsTest extends AnyFunSuite with Matchers { found.isDefined should be(true) val annotation = found.get - Annotations.annotationEquals[Annotation1](annotation) should be(true) + Annotations.equals[Annotation1](annotation) should be(true) } test("Annotations#isAnnotationPresent") { diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala b/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala index 6c5d89c8c6..607188f85a 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/TypesTest.scala @@ -63,33 +63,33 @@ class TypesTest extends AnyFunSuite with Matchers { //Types.runtimeClass[Any] } - test("Types#eq") { - Types.eq[String](typeTag[String]) should be(true) - Types.eq[AnyRef](TypeTag.AnyRef) should be(true) - Types.eq[AnyVal](TypeTag.AnyVal) should be(true) - Types.eq[Unit](TypeTag.Unit) should be(true) - Types.eq[Byte](TypeTag.Byte) should be(true) - Types.eq[Short](TypeTag.Short) should be(true) - Types.eq[Char](TypeTag.Char) should be(true) - Types.eq[Int](TypeTag.Int) should be(true) - Types.eq[Long](TypeTag.Long) should be(true) - Types.eq[Float](TypeTag.Float) should be(true) - Types.eq[Double](TypeTag.Double) should be(true) - Types.eq[Boolean](TypeTag.Boolean) should be(true) - Types.eq[java.lang.Object](TypeTag.Object) should be(true) - Types.eq[Any](TypeTag.Any) should be(true) - Types.eq[Null](TypeTag.Null) should be(true) - Types.eq[Nothing](TypeTag.Nothing) should be(true) + test("Types#equals") { + Types.equals[String](typeTag[String]) should be(true) + Types.equals[AnyRef](TypeTag.AnyRef) should be(true) + Types.equals[AnyVal](TypeTag.AnyVal) should be(true) + Types.equals[Unit](TypeTag.Unit) should be(true) + Types.equals[Byte](TypeTag.Byte) should be(true) + Types.equals[Short](TypeTag.Short) should be(true) + Types.equals[Char](TypeTag.Char) should be(true) + Types.equals[Int](TypeTag.Int) should be(true) + Types.equals[Long](TypeTag.Long) should be(true) + Types.equals[Float](TypeTag.Float) should be(true) + Types.equals[Double](TypeTag.Double) should be(true) + Types.equals[Boolean](TypeTag.Boolean) should be(true) + Types.equals[java.lang.Object](TypeTag.Object) should be(true) + Types.equals[Any](TypeTag.Any) should be(true) + Types.equals[Null](TypeTag.Null) should be(true) + Types.equals[Nothing](TypeTag.Nothing) should be(true) // generics - Types.eq[Baz[_, _]](typeTag[Baz[_, _]]) should be(true) - Types.eq[Foo[_]](typeTag[Foo[_]]) should be(true) - Types.eq[Bez[_, _]](typeTag[Bez[_, _]]) should be(true) - Types.eq[TypedTrait[_, _]](typeTag[TypedTrait[_, _]]) should be(true) - - Types.eq[Int](TypeTag.Float) should be(false) - Types.eq[String](TypeTag.Int) should be(false) - Types.eq[java.lang.Object](TypeTag.Int) should be(false) - Types.eq[String](TypeTag.Object) should be(false) + Types.equals[Baz[_, _]](typeTag[Baz[_, _]]) should be(true) + Types.equals[Foo[_]](typeTag[Foo[_]]) should be(true) + Types.equals[Bez[_, _]](typeTag[Bez[_, _]]) should be(true) + Types.equals[TypedTrait[_, _]](typeTag[TypedTrait[_, _]]) should be(true) + + Types.equals[Int](TypeTag.Float) should be(false) + Types.equals[String](TypeTag.Int) should be(false) + Types.equals[java.lang.Object](TypeTag.Int) should be(false) + Types.equals[String](TypeTag.Object) should be(false) } test("Types#parameterizedTypeNames") { From 5876adfc7038dc2e77e4b19a176f6f3faa3fd1ac Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Mon, 29 Mar 2021 19:20:08 +0000 Subject: [PATCH 459/796] {util-core,finagle-core}: Make it possible to directly construct X509Certificates without an on-disk file Problem Right now our entire X509Certificate chain is File-based. This doesn't work well in cases where you don't want to have to use a File. Solution Add a new way to construct an X509Certificate and TrustManager that are based on a buffered String, instead of File. Also rename PemFile to PemByte so that it assumes being fully-buffered instead of not-yet read. JIRA Issues: CSL-10763 Differential Revision: https://phabricator.twitter.biz/D641088 --- CHANGELOG.rst | 12 +++ .../{PemFile.scala => PemBytes.scala} | 40 ++++++---- .../security/Pkcs8EncodedKeySpecFile.scala | 14 ++-- .../X509CertificateDeserializer.scala | 54 +++++++++++++ .../util/security/X509CertificateFile.scala | 41 ++++------ .../twitter/util/security/X509CrlFile.scala | 14 ++-- .../security/X509TrustManagerFactory.scala | 32 +++++--- .../{PemFileTest.scala => PemBytesTest.scala} | 38 +++++----- ...estUtils.scala => PemBytesTestUtils.scala} | 2 +- .../Pkcs8EncodedKeySpecFileTest.scala | 8 +- .../X509CertificateDeserializerTest.scala | 76 +++++++++++++++++++ .../security/X509CertificateFileTest.scala | 29 ++----- .../util/security/X509CrlFileTest.scala | 8 +- 13 files changed, 249 insertions(+), 119 deletions(-) rename util-security/src/main/scala/com/twitter/util/security/{PemFile.scala => PemBytes.scala} (80%) create mode 100644 util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala rename util-security/src/test/scala/com/twitter/util/security/{PemFileTest.scala => PemBytesTest.scala} (60%) rename util-security/src/test/scala/com/twitter/util/security/{PemFileTestUtils.scala => PemBytesTestUtils.scala} (98%) create mode 100644 util-security/src/test/scala/com/twitter/util/security/X509CertificateDeserializerTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 585cc98b83..def2d8991c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,6 +26,12 @@ Breaking API Changes * util-reflect: Remove deprecated `c.t.util.reflect.Proxy`. There is no library replacement. ``PHAB_ID=D630143`` +* util-security: Renamed `com.twitter.util.security.PemFile` to `c.t.u.security.PemBytes`, and + changed its constructor to accept a string and a name. The main change here is that we assume + the PEM-encoded text has been fully buffered. To migrate, please use the helper method on the + companion object, `PemBytes#fromFile`. Note that unlike before with construction, we read from + the file, so it's possible for it to throw. ``PHAB_ID=D641088`` + New Features ~~~~~~~~~~~~ @@ -45,6 +51,12 @@ New Features useful for wrapping an existing `ByteReader` or `ByteWriter` and extending its functionality without modifying the underlying instance. ``PHAB_ID=D622705`` +* util-core: Provided `c.t.u.security.X509CertificateDeserializer` to make it possible to directly + deserialize an `X509Certificate` even if you don't have a file on disk. Also provided + `c.t.u.security.X509TrustManagerFactory#buildTrustManager` to make it possible to directly + construct an `X509TrustManager` with an `X509Certificate` instead of passing in a `File`. + ``PHAB_ID=D641088`` + 21.2.0 ------ diff --git a/util-security/src/main/scala/com/twitter/util/security/PemFile.scala b/util-security/src/main/scala/com/twitter/util/security/PemBytes.scala similarity index 80% rename from util-security/src/main/scala/com/twitter/util/security/PemFile.scala rename to util-security/src/main/scala/com/twitter/util/security/PemBytes.scala index 8dae7cfce8..50910a20fb 100644 --- a/util-security/src/main/scala/com/twitter/util/security/PemFile.scala +++ b/util-security/src/main/scala/com/twitter/util/security/PemBytes.scala @@ -1,8 +1,10 @@ package com.twitter.util.security import com.twitter.util.Try -import com.twitter.util.security.PemFile._ -import java.io.{BufferedReader, File, FileReader, IOException} +import com.twitter.util.security.PemBytes._ +import java.io.{BufferedReader, File, IOException, StringReader} +import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.util.Base64 import scala.collection.mutable.ArrayBuffer @@ -10,8 +12,8 @@ import scala.collection.mutable.ArrayBuffer * Signals that the PEM file failed to parse properly due to a * missing boundary header or footer. */ -class InvalidPemFormatException(file: File, message: String) - extends IOException(s"PemFile (${file.getName}) failed to load: $message") +class InvalidPemFormatException(name: String, message: String) + extends IOException(s"PemBytes ($name) failed to load: $message") /** * An abstract representation of a PEM formatted file. @@ -25,10 +27,9 @@ class InvalidPemFormatException(file: File, message: String) * base64encodedbytes * -----END CERTIFICATE----- */ -class PemFile(file: File) { - - private[this] def openReader(file: File): BufferedReader = - new BufferedReader(new FileReader(file)) +class PemBytes(message: String, name: String) { + private[this] def openReader(): BufferedReader = + new BufferedReader(new StringReader(message)) private[this] def closeReader(bufReader: BufferedReader): Unit = Try(bufReader.close()) @@ -48,7 +49,7 @@ class PemFile(file: File) { private[this] def readEncodedMessage(bufReader: BufferedReader, messageType: String): String = { val contents = readEncodedMessages(bufReader, messageType) if (contents.isEmpty) - throw new InvalidPemFormatException(file, "Empty input") + throw new InvalidPemFormatException(name, "Empty input") else contents.head } @@ -66,7 +67,7 @@ class PemFile(file: File) { var line: String = bufReader.readLine() if (line == null) - throw new InvalidPemFormatException(file, "Empty input") + throw new InvalidPemFormatException(name, "Empty input") while (line != null) { if (state == PemState.Begin || state == PemState.End) { @@ -79,7 +80,7 @@ class PemFile(file: File) { } else { // PemState.Content if (line == header) { // A new matching header, but the previous one had no footer - throw new InvalidPemFormatException(file, "Missing " + footer) + throw new InvalidPemFormatException(name, "Missing " + footer) } if (line == footer) { // A good message, handle it and move to the end state @@ -95,10 +96,10 @@ class PemFile(file: File) { } if (state == PemState.Begin) { // We didn't see any matching header - throw new InvalidPemFormatException(file, "Missing " + header) + throw new InvalidPemFormatException(name, "Missing " + header) } else if (state == PemState.Content) { // We didn't see a corresponding footer for the current message - throw new InvalidPemFormatException(file, "Missing " + footer) + throw new InvalidPemFormatException(name, "Missing " + footer) } contents.toSeq @@ -115,7 +116,7 @@ class PemFile(file: File) { * only the first one will be returned. */ def readMessage(messageType: String): Try[Array[Byte]] = - Try(openReader(file)) + Try(openReader()) .flatMap(reader => Try(readDecodedMessage(reader, messageType)).ensure(closeReader(reader))) /** @@ -128,12 +129,12 @@ class PemFile(file: File) { * are ignored. */ def readMessages(messageType: String): Try[Seq[Array[Byte]]] = - Try(openReader(file)) + Try(openReader()) .flatMap(reader => Try(readDecodedMessages(reader, messageType)).ensure(closeReader(reader))) } -private object PemFile { +object PemBytes { private trait PemState private object PemState { @@ -167,4 +168,11 @@ private object PemFile { decoder.decode(encoded) } + /** + * Reads the given file and constructs a `PemBytes` with the contents. + */ + def fromFile(file: File): Try[PemBytes] = Try { + val buffered = new String(Files.readAllBytes(file.toPath), StandardCharsets.UTF_8) + new PemBytes(buffered, file.getName) + } } diff --git a/util-security/src/main/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFile.scala b/util-security/src/main/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFile.scala index b9fbf33eb4..59946b612b 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFile.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFile.scala @@ -23,13 +23,13 @@ class Pkcs8EncodedKeySpecFile(file: File) { /** * Attempts to read the contents of the private key from the file. */ - def readPkcs8EncodedKeySpec(): Try[PKCS8EncodedKeySpec] = { - val pemFile = new PemFile(file) - pemFile - .readMessage(MessageType) - .map(new PKCS8EncodedKeySpec(_)) - .onFailure(logException) - } + def readPkcs8EncodedKeySpec(): Try[PKCS8EncodedKeySpec] = + PemBytes.fromFile(file).flatMap { pemBytes => + pemBytes + .readMessage(MessageType) + .map(new PKCS8EncodedKeySpec(_)) + .onFailure(logException) + } } private object Pkcs8EncodedKeySpecFile { diff --git a/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala b/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala new file mode 100644 index 0000000000..e208cb74c7 --- /dev/null +++ b/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala @@ -0,0 +1,54 @@ +package com.twitter.util.security + +import com.twitter.util.Try + +import java.io.ByteArrayInputStream +import java.security.cert.{CertificateFactory, X509Certificate} + +/** + * A helper object to deserialize PEM-encoded X.509 Certificates. + * + * @example + * -----BEGIN CERTIFICATE----- + * base64encodedbytes + * -----END CERTIFICATE----- + */ +object X509CertificateDeserializer { + private[this] val MessageType: String = "CERTIFICATE" + private[this] val deserializeX509: Array[Byte] => X509Certificate = { certBytes: Array[Byte] => + val certFactory = CertificateFactory.getInstance("X.509") + val certificate = certFactory + .generateCertificate(new ByteArrayInputStream(certBytes)) + .asInstanceOf[X509Certificate] + certificate.checkValidity() + certificate + } + + /** + * Deserializes an [[InputStream]] that contains a PEM-encoded X.509 + * Certificate. + * + * Closes the InputStream once it has finished reading. + */ + def deserializeCertificate(rawPem: String, name: String): Try[X509Certificate] = { + val pemBytes = new PemBytes(rawPem, name) + val message: Try[Array[Byte]] = pemBytes + .readMessage(MessageType) + + message.map(deserializeX509) + } + + /** + * Deserializes an [[InputStream]] that contains a PEM-encoded X.509 + * Certificate. + * + * Closes the InputStream once it has finished reading. + */ + def deserializeCertificates(rawPem: String, name: String): Try[Seq[X509Certificate]] = { + val pemBytes = new PemBytes(rawPem, name) + val messages: Try[Seq[Array[Byte]]] = pemBytes + .readMessages(MessageType) + + messages.map(_.map(deserializeX509)) + } +} diff --git a/util-security/src/main/scala/com/twitter/util/security/X509CertificateFile.scala b/util-security/src/main/scala/com/twitter/util/security/X509CertificateFile.scala index 683be5439b..26a8b6ff38 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X509CertificateFile.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X509CertificateFile.scala @@ -3,8 +3,10 @@ package com.twitter.util.security import com.twitter.logging.Logger import com.twitter.util.Try import com.twitter.util.security.X509CertificateFile._ -import java.io.{ByteArrayInputStream, File} -import java.security.cert.{CertificateFactory, X509Certificate} +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.security.cert.X509Certificate /** * A representation of an X.509 Certificate PEM-encoded and stored @@ -23,38 +25,23 @@ class X509CertificateFile(file: File) { /** * Attempts to read the contents of the X.509 Certificate from the file. */ - def readX509Certificate(): Try[X509Certificate] = { - val pemFile = new PemFile(file) - pemFile - .readMessage(MessageType) - .map(generateX509Certificate) - .onFailure(logException) - } + def readX509Certificate(): Try[X509Certificate] = Try { + new String(Files.readAllBytes(file.toPath), StandardCharsets.UTF_8) + }.flatMap { buffered => + X509CertificateDeserializer.deserializeCertificate(buffered, file.getName) + }.onFailure(logException) /** * Attempts to read the contents of multiple X.509 Certificates from the file. */ - def readX509Certificates(): Try[Seq[X509Certificate]] = { - val pemFile = new PemFile(file) - pemFile - .readMessages(MessageType) - .map(certBytes => certBytes.map(generateX509Certificate)) - .onFailure(logException) - } + def readX509Certificates(): Try[Seq[X509Certificate]] = Try { + new String(Files.readAllBytes(file.toPath), StandardCharsets.UTF_8) + }.flatMap { buffered => + X509CertificateDeserializer.deserializeCertificates(buffered, file.getName) + }.onFailure(logException) } private object X509CertificateFile { - private val MessageType: String = "CERTIFICATE" - private val log = Logger.get("com.twitter.util.security") - - private def generateX509Certificate(decodedMessage: Array[Byte]): X509Certificate = { - val certFactory = CertificateFactory.getInstance("X.509") - val certificate = certFactory - .generateCertificate(new ByteArrayInputStream(decodedMessage)) - .asInstanceOf[X509Certificate] - certificate.checkValidity() - certificate - } } diff --git a/util-security/src/main/scala/com/twitter/util/security/X509CrlFile.scala b/util-security/src/main/scala/com/twitter/util/security/X509CrlFile.scala index 90e5259c15..dc965c6efe 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X509CrlFile.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X509CrlFile.scala @@ -20,13 +20,13 @@ class X509CrlFile(file: File) { private[this] def logException(ex: Throwable): Unit = log.warning(s"X509Crl (${file.getName}) failed to load: ${ex.getMessage}.") - def readX509Crl(): Try[X509CRL] = { - val pemFile = new PemFile(file) - pemFile - .readMessage(MessageType) - .map(generateX509Crl) - .onFailure(logException) - } + def readX509Crl(): Try[X509CRL] = + PemBytes.fromFile(file).flatMap { pemBytes => + pemBytes + .readMessage(MessageType) + .map(generateX509Crl) + .onFailure(logException) + } } diff --git a/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala b/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala index 4497fb54be..91ae0db1d5 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala @@ -21,6 +21,23 @@ class X509TrustManagerFactory(certsFile: File) { s"failed to create trust manager: ${ex.getMessage()}." ) + /** + * Attempts to read the contents of a file containing a collection of X.509 certificates, + * and combines them into a `javax.net.ssl.TrustManager`. + * The singular value is returned in an Array for ease of use with + * `javax.net.ssl.SSLContext`'s init method. + */ + def getTrustManagers(): Try[Array[TrustManager]] = { + val tryCerts: Try[Seq[X509Certificate]] = + new X509CertificateFile(certsFile).readX509Certificates() + tryCerts.flatMap(buildTrustManager).onFailure(logException) + } + +} + +object X509TrustManagerFactory { + private val log = Logger.get("com.twitter.util.security") + private[this] def setCertificateEntry(ks: KeyStore)(cert: X509Certificate): Unit = { val alias: String = UUID.randomUUID().toString() ks.setCertificateEntry(alias, cert) @@ -46,19 +63,12 @@ class X509TrustManagerFactory(certsFile: File) { } /** - * Attempts to read the contents of a file containing a collection of X.509 certificates, - * and combines them into a `javax.net.ssl.TrustManager`. + * Attempts to combine the passed in X.509 certificates in order to construct a + * `javax.net.ssl.TrustManager`. * The singular value is returned in an Array for ease of use with * `javax.net.ssl.SSLContext`'s init method. */ - def getTrustManagers(): Try[Array[TrustManager]] = { - val tryCerts: Try[Seq[X509Certificate]] = - new X509CertificateFile(certsFile).readX509Certificates() - tryCerts.map(certsToTrustManagers).onFailure(logException) + def buildTrustManager(x509Certificates: Seq[X509Certificate]): Try[Array[TrustManager]] = Try { + certsToTrustManagers(x509Certificates) } - -} - -private object X509TrustManagerFactory { - private val log = Logger.get("com.twitter.util.security") } diff --git a/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/PemBytesTest.scala similarity index 60% rename from util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala rename to util-security/src/test/scala/com/twitter/util/security/PemBytesTest.scala index 8cf1cc794f..b4084c45cf 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PemFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PemBytesTest.scala @@ -5,28 +5,30 @@ import com.twitter.util.Try import java.io.File import org.scalatest.funsuite.AnyFunSuite -class PemFileTest extends AnyFunSuite { +class PemBytesTest extends AnyFunSuite { - private[this] def readPemFileMessage(messageType: String)(tempFile: File): Try[String] = { - val pemFile = new PemFile(tempFile) - pemFile.readMessage(messageType).map(new String(_)) - } + private[this] def readPemBytesMessage(messageType: String)(tempFile: File): Try[String] = + for { + pemBytes <- PemBytes.fromFile(tempFile) + item <- pemBytes.readMessage(messageType) + } yield new String(item) - private[this] def readPemFileMessages(messageType: String)(tempFile: File): Try[Seq[String]] = { - val pemFile = new PemFile(tempFile) - pemFile.readMessages(messageType).map(items => items.map(new String(_))) - } + private[this] def readPemBytesMessages(messageType: String)(tempFile: File): Try[Seq[String]] = + for { + pemBytes <- PemBytes.fromFile(tempFile) + items <- pemBytes.readMessages(messageType) + } yield items.map(new String(_)) - private[this] val readHello = readPemFileMessage("hello") _ - private[this] val readHellos = readPemFileMessages("hello") _ + private[this] val readHello = readPemBytesMessage("hello") _ + private[this] val readHellos = readPemBytesMessages("hello") _ test("no header file") { val tempFile = TempFile.fromResourcePath("/pem/test-no-head.txt") // deleteOnExit is handled by TempFile val tryHello = readHello(tempFile) - PemFileTestUtils.assertException[InvalidPemFormatException, String](tryHello) - PemFileTestUtils.assertExceptionMessageContains("Missing -----BEGIN HELLO-----")(tryHello) + PemBytesTestUtils.assertException[InvalidPemFormatException, String](tryHello) + PemBytesTestUtils.assertExceptionMessageContains("Missing -----BEGIN HELLO-----")(tryHello) } test("no footer file") { @@ -34,17 +36,17 @@ class PemFileTest extends AnyFunSuite { // deleteOnExit is handled by TempFile val tryHello = readHello(tempFile) - PemFileTestUtils.assertException[InvalidPemFormatException, String](tryHello) - PemFileTestUtils.assertExceptionMessageContains("Missing -----END HELLO-----")(tryHello) + PemBytesTestUtils.assertException[InvalidPemFormatException, String](tryHello) + PemBytesTestUtils.assertExceptionMessageContains("Missing -----END HELLO-----")(tryHello) } test("wrong message type") { val tempFile = TempFile.fromResourcePath("/pem/test.txt") // deleteOnExit is handled by TempFile - val tryHello = readPemFileMessage("GOODBYE")(tempFile) - PemFileTestUtils.assertException[InvalidPemFormatException, String](tryHello) - PemFileTestUtils.assertExceptionMessageContains("Missing -----BEGIN GOODBYE-----")(tryHello) + val tryHello = readPemBytesMessage("GOODBYE")(tempFile) + PemBytesTestUtils.assertException[InvalidPemFormatException, String](tryHello) + PemBytesTestUtils.assertExceptionMessageContains("Missing -----BEGIN GOODBYE-----")(tryHello) } test("good file") { diff --git a/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala b/util-security/src/test/scala/com/twitter/util/security/PemBytesTestUtils.scala similarity index 98% rename from util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala rename to util-security/src/test/scala/com/twitter/util/security/PemBytesTestUtils.scala index e1dc7afb3c..e183991517 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PemFileTestUtils.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PemBytesTestUtils.scala @@ -5,7 +5,7 @@ import java.io.{File, IOException} import org.scalatest.Assertions._ import scala.reflect.ClassTag -object PemFileTestUtils { +object PemBytesTestUtils { def assertException[A <: AnyRef, B](tryValue: Try[B])(implicit classTag: ClassTag[A]): Unit = { assert(tryValue.isThrow) diff --git a/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala index edc0241232..0f2feb14de 100644 --- a/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFileTest.scala @@ -15,19 +15,19 @@ class Pkcs8EncodedKeySpecFileTest extends AnyFunSuite { } test("File path doesn't exist") { - PemFileTestUtils.testFileDoesntExist("PKCS8EncodedKeySpec", readKeySpecFromFile) + PemBytesTestUtils.testFileDoesntExist("PKCS8EncodedKeySpec", readKeySpecFromFile) } test("File path isn't a file") { - PemFileTestUtils.testFilePathIsntFile("PKCS8EncodedKeySpec", readKeySpecFromFile) + PemBytesTestUtils.testFilePathIsntFile("PKCS8EncodedKeySpec", readKeySpecFromFile) } test("File path isn't readable") { - PemFileTestUtils.testFilePathIsntReadable("PKCS8EncodedKeySpec", readKeySpecFromFile) + PemBytesTestUtils.testFilePathIsntReadable("PKCS8EncodedKeySpec", readKeySpecFromFile) } test("File isn't a key spec") { - PemFileTestUtils.testEmptyFile[InvalidPemFormatException, PKCS8EncodedKeySpec]( + PemBytesTestUtils.testEmptyFile[InvalidPemFormatException, PKCS8EncodedKeySpec]( "PKCS8EncodedKeySpec", readKeySpecFromFile ) diff --git a/util-security/src/test/scala/com/twitter/util/security/X509CertificateDeserializerTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509CertificateDeserializerTest.scala new file mode 100644 index 0000000000..1d62afc58b --- /dev/null +++ b/util-security/src/test/scala/com/twitter/util/security/X509CertificateDeserializerTest.scala @@ -0,0 +1,76 @@ +package com.twitter.util.security + +import java.security.cert.{CertificateException, X509Certificate} +import org.scalatest.funsuite.AnyFunSuite +import scala.io.Source + +class X509CertificateDeserializerTest extends AnyFunSuite { + private[this] def getResourceFileAsString(path: String): String = { + val resourceStream = getClass.getResourceAsStream(path) + val resourceSource = Source.fromInputStream(resourceStream) + try { + val resource = resourceSource.mkString + resource + } finally { + resourceSource.close() + } + } + + private[this] def assertIsCslCert(cert: X509Certificate): Unit = { + val subjectData: String = cert.getSubjectX500Principal.getName() + assert(subjectData.contains("Core Systems Libraries")) + } + + test("Input is garbage") { + // Lines were manually deleted from a real certificate file + val input = getResourceFileAsString("/certs/test-rsa-garbage.crt") + + val tryCert = X509CertificateDeserializer.deserializeCertificate(input, "test") + + PemBytesTestUtils.assertException[CertificateException, X509Certificate](tryCert) + } + + test("Input is an X509 Certificate") { + val input = getResourceFileAsString("/certs/test-rsa.crt") + + val tryCert = X509CertificateDeserializer.deserializeCertificate(input, "test") + + assert(tryCert.isReturn) + val cert = tryCert.get() + + assert(cert.getSigAlgName == "SHA256withRSA") + } + + test("Input with multiple X509 Certificates") { + val input = getResourceFileAsString("/certs/test-rsa-chain.crt") + + val tryCerts = X509CertificateDeserializer.deserializeCertificates(input, "test") + + assert(tryCerts.isReturn) + val certs = tryCerts.get() + + assert(certs.length == 2) + + val intermediate = certs.head + assertIsCslCert(intermediate) + + val root = certs(1) + assertIsCslCert(root) + } + + test("X509 Certificate File is not valid: expired") { + val input = getResourceFileAsString("/certs/test-rsa-expired.crt") + + intercept[java.security.cert.CertificateExpiredException] { + X509CertificateDeserializer.deserializeCertificate(input, "test").get() + } + } + + test("X509 Certificate File is not valid: not yet ready") { + val input = getResourceFileAsString("/certs/test-rsa-future.crt") + + intercept[java.security.cert.CertificateNotYetValidException] { + X509CertificateDeserializer.deserializeCertificate(input, "test").get() + } + } +} diff --git a/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala index 8f1dea9fcb..d88be71db9 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509CertificateFileTest.scala @@ -14,7 +14,7 @@ class X509CertificateFileTest extends AnyFunSuite { } private[this] def assertCertException(tryCert: Try[X509Certificate]): Unit = - PemFileTestUtils.assertException[CertificateException, X509Certificate](tryCert) + PemBytesTestUtils.assertException[CertificateException, X509Certificate](tryCert) private[this] val readCertFromFile: File => Try[X509Certificate] = (tempFile) => { @@ -23,19 +23,19 @@ class X509CertificateFileTest extends AnyFunSuite { } test("File path doesn't exist") { - PemFileTestUtils.testFileDoesntExist("X509Certificate", readCertFromFile) + PemBytesTestUtils.testFileDoesntExist("X509Certificate", readCertFromFile) } test("File path isn't a file") { - PemFileTestUtils.testFilePathIsntFile("X509Certificate", readCertFromFile) + PemBytesTestUtils.testFilePathIsntFile("X509Certificate", readCertFromFile) } test("File path isn't readable") { - PemFileTestUtils.testFilePathIsntReadable("X509Certificate", readCertFromFile) + PemBytesTestUtils.testFilePathIsntReadable("X509Certificate", readCertFromFile) } test("File isn't a certificate") { - PemFileTestUtils.testEmptyFile[InvalidPemFormatException, X509Certificate]( + PemBytesTestUtils.testEmptyFile[InvalidPemFormatException, X509Certificate]( "X509Certificate", readCertFromFile ) @@ -83,23 +83,4 @@ class X509CertificateFileTest extends AnyFunSuite { val root = certs(1) assertIsCslCert(root) } - - test("X509 Certificate File is not valid: expired") { - val tempFile = TempFile.fromResourcePath("/certs/test-rsa-expired.crt") - // deleteOnExit is handled by TempFile - - intercept[java.security.cert.CertificateExpiredException] { - readCertFromFile(tempFile).get() - } - } - - test("X509 Certificate File is not valid: not yet ready") { - val tempFile = TempFile.fromResourcePath("/certs/test-rsa-future.crt") - // deleteOnExit is handled by TempFile - - intercept[java.security.cert.CertificateNotYetValidException] { - readCertFromFile(tempFile).get() - } - } - } diff --git a/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala b/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala index ba536cecf6..227b2819c8 100644 --- a/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/X509CrlFileTest.scala @@ -9,7 +9,7 @@ import org.scalatest.funsuite.AnyFunSuite class X509CrlFileTest extends AnyFunSuite { private[this] def assertCrlException(tryCrl: Try[X509CRL]): Unit = - PemFileTestUtils.assertException[CRLException, X509CRL](tryCrl) + PemBytesTestUtils.assertException[CRLException, X509CRL](tryCrl) private[this] val readCrlFromFile: File => Try[X509CRL] = (tempFile) => { @@ -18,15 +18,15 @@ class X509CrlFileTest extends AnyFunSuite { } test("File path doesn't exist") { - PemFileTestUtils.testFileDoesntExist("X509Crl", readCrlFromFile) + PemBytesTestUtils.testFileDoesntExist("X509Crl", readCrlFromFile) } test("File path isn't a file") { - PemFileTestUtils.testFilePathIsntFile("X509Crl", readCrlFromFile) + PemBytesTestUtils.testFilePathIsntFile("X509Crl", readCrlFromFile) } test("File isn't a crl") { - PemFileTestUtils.testEmptyFile[InvalidPemFormatException, X509CRL]( + PemBytesTestUtils.testEmptyFile[InvalidPemFormatException, X509CRL]( "X509Crl", readCrlFromFile ) From 57095c3ba28c90dfc3385164bd69767c36f9523b Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 29 Mar 2021 19:37:35 +0000 Subject: [PATCH 460/796] twitter-server: expression in a query language like format Problem The general "expression" field has not yet been formatted but directly uses JSON to reflect the class structure. Solution Make it a one-line string that uses the common arithmetic function names. Result Serialize expression needs more information (latched/unlatched), we cannot fit it in a Jackson customized Serializer. Added another field in the Expression case class. JIRA Issues: CSL-10618 Differential Revision: https://phabricator.twitter.biz/D638561 --- .../finagle/stats/exp/Expression.scala | 90 ++++++++++++++----- .../finagle/stats/exp/ExpressionLabels.scala | 8 ++ .../finagle/stats/exp/ExpressionSchema.scala | 17 +++- ...dTest.scala => ExpressionSchemaTest.scala} | 55 +++++++----- 4 files changed, 125 insertions(+), 45 deletions(-) rename util-stats/src/test/scala/com/twitter/finagle/stats/exp/{ExprEndToEndTest.scala => ExpressionSchemaTest.scala} (50%) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index 985f270e8d..a6e06f19d9 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -1,8 +1,56 @@ package com.twitter.finagle.stats.exp -import com.twitter.finagle.stats.{MetricSchema, StatsReceiver} +import com.twitter.finagle.stats.exp.Expression.HistogramComponent +import com.twitter.finagle.stats.{HistogramSchema, MetricSchema, StatsReceiver} import scala.annotation.varargs +private[twitter] object Expression { + + sealed trait HistogramComponent + case object Min extends HistogramComponent + case object Max extends HistogramComponent + case object Avg extends HistogramComponent + case object Sum extends HistogramComponent + case object Count extends HistogramComponent + + private[exp] def getStatsReceivers(expr: Expression): Set[StatsReceiver] = expr match { + case FunctionExpression(_, exprs) => + exprs.foldLeft(Set.empty[StatsReceiver]) { + case (acc, expr) => + getStatsReceivers(expr) ++ acc + } + case MetricExpression(schema) => Set(schema.metricBuilder.statsReceiver) + case HistogramExpression(schema, _) => Set(schema.metricBuilder.statsReceiver) + case ConstantExpression(_, statsReceiver) => Set(statsReceiver) + } + + /** + * Create a expression with a constant number in double + * @param statsReceiver the statsReceiver to register this expression + */ + def apply(num: Double, statsReceiver: StatsReceiver): Expression = + ConstantExpression(num.toString, statsReceiver) + + /** + * Create a histogram expression + * @param component the histogram component either a [[HistogramComponent]] for Left + * or a percentile in Double for Right. + */ + def apply(schema: HistogramSchema, component: Either[HistogramComponent, Double]) = + HistogramExpression(schema, component) + + /** + * Create an single expression wrapping a counter or gauge. + */ + def apply(schema: MetricSchema): Expression = { + require(!schema.isInstanceOf[HistogramSchema], "provide a component for histogram") + MetricExpression(schema) + } +} + +/** + * Metrics with their arithmetical(or others) calculations + */ private[twitter] sealed trait Expression { def plus(other: Expression): Expression = FunctionExpression("plus", Seq(this, other)) @@ -18,27 +66,29 @@ private[twitter] sealed trait Expression { FunctionExpression(name, this +: rest) } -private[twitter] object Expression { - def getStatsReceivers(expr: Expression): Set[StatsReceiver] = expr match { - case FunctionExpression(_, exprs) => - exprs.foldLeft(Set.empty[StatsReceiver]) { - case (acc, expr) => - getStatsReceivers(expr) ++ acc - } - case MetricExpression(schema) => Set(schema.metricBuilder.statsReceiver) - case ConstantExpression(_, statsReceiver) => Set(statsReceiver) - } - def apply(num: Double, statsReceiver: StatsReceiver): Expression = - ConstantExpression(num.toString, statsReceiver) - def apply(schema: MetricSchema): Expression = MetricExpression(schema) -} - -private[twitter] case class ConstantExpression(repr: String, statsReceiver: StatsReceiver) +/** + * Represents a constant double number + */ +case class ConstantExpression private (repr: String, statsReceiver: StatsReceiver) extends Expression -private[twitter] case class FunctionExpression(fnName: String, exprs: Seq[Expression]) - extends Expression { +/** + * Represents compound metrics and their arithmetical(or others) calculations + */ +case class FunctionExpression private (fnName: String, exprs: Seq[Expression]) extends Expression { require(exprs.size != 0, "Functions must have at least 1 argument") } -private[twitter] case class MetricExpression(schema: MetricSchema) extends Expression +/** + * Represents the leaf metrics + */ +case class MetricExpression private (schema: MetricSchema) extends Expression + +/** + * Represent a histogram expression with specified component, for example the average, or a percentile + * @param component either a [[HistogramComponent]] or a percentile in Double + */ +case class HistogramExpression private ( + schema: HistogramSchema, + component: Either[HistogramComponent, Double]) + extends Expression diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala index f176cc1c30..3cb6779e02 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala @@ -5,6 +5,14 @@ import com.twitter.finagle.stats.{NoRoleSpecified, SourceRole} private[exp] object ExpressionLabels { val empty = ExpressionLabels(None, None, NoRoleSpecified) } + +/** + * ExpressionLabels provide service-related information + * + * @param processPath a universal coordinate for the resource + * @param serviceName a client label or a server label depends on the `role` + * @param role whether the service is playing the part of client or server regarding this metric + */ private[exp] case class ExpressionLabels( processPath: Option[String], serviceName: Option[String], diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index 3dadd10132..f6f26984bb 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -2,13 +2,25 @@ package com.twitter.finagle.stats.exp import com.twitter.finagle.stats.{MetricUnit, SourceRole, StatsReceiver, Unspecified} +/** + * ExpressionSchema is builder class that construct an expression with its metadata. + * + * @param name this is going to be an important query key when fetching expressions + * @param labels service related information, see [[ExpressionLabels]] + * @param expr class representation of the expression, see [[Expression]] + * @param bounds thresholds for this expression + * @param description human-readable description of an expression's significance + * @param unit the unit associated with the metrics value (milliseconds, megabytes, requests, etc) + * @param exprQuery string representation of the expression + */ case class ExpressionSchema private ( name: String, labels: ExpressionLabels, expr: Expression, bounds: Bounds, description: String, - unit: MetricUnit) { + unit: MetricUnit, + exprQuery: String) { def withBounds(bounds: Bounds): ExpressionSchema = copy(bounds = bounds) def withDescription(description: String): ExpressionSchema = copy(description = description) @@ -39,5 +51,6 @@ private[twitter] object ExpressionSchema { expr = expr, bounds = Unbounded.get, description = "Unspecified", - unit = Unspecified) + unit = Unspecified, + exprQuery = "") } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala similarity index 50% rename from util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala rename to util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 34929b0475..fb87910ea9 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExprEndToEndTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -6,39 +6,41 @@ import com.twitter.util.{Stopwatch, Time, TimeControl} import org.scalatest.FunSuite import scala.util.control.NonFatal -class ExprEndToEndTest extends FunSuite { - def mySlowQuery(ctl: TimeControl, succeed: Boolean): Unit = { +class ExpressionSchemaTest extends FunSuite { + + val sr = new InMemoryStatsReceiver + val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) + + val successMb = + CounterSchema(new MetricBuilder(name = Seq("success"), statsReceiver = clientSR)) + val failuresMb = + CounterSchema(new MetricBuilder(name = Seq("failures"), statsReceiver = clientSR)) + val latencyMb = + HistogramSchema(new MetricBuilder(name = Seq("latency"), statsReceiver = clientSR)) + val sum = Expression(successMb).plus(Expression(failuresMb)) + + val successRate = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query") + + val latency = ExpressionSchema("latency", Expression(latencyMb, Right(0.99))) + .withUnit(Milliseconds) + .withDescription("The latency of the slow query") + + def slowQuery(ctl: TimeControl, succeed: Boolean): Unit = { ctl.advance(50.milliseconds) if (!succeed) { throw new Exception("boom!") } } - test("Demonstrate an end to end example of using metrics") { - val sr = new InMemoryStatsReceiver - val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) - - val successMb = - CounterSchema(new MetricBuilder(name = Seq("success"), statsReceiver = clientSR)) - val failuresMb = - CounterSchema(new MetricBuilder(name = Seq("failures"), statsReceiver = clientSR)) - val latencyMb = - HistogramSchema(new MetricBuilder(name = Seq("latency"), statsReceiver = clientSR)) + test("Demonstrate an end to end example of using metric expressions") { - val sum = Expression(successMb).plus(Expression(failuresMb)) val successCounter = clientSR.counter(successMb) val failuresCounter = clientSR.counter(failuresMb) val latencyStat = clientSR.stat(latencyMb) - val successRate = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) - .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) - .withUnit(Percentage) - .withDescription("The success rate of the slow query") - - val latency = ExpressionSchema("latency", Expression(latencyMb)) - .withUnit(Milliseconds) - .withDescription("The latency of the slow query") - successRate.register() latency.register() @@ -46,7 +48,7 @@ class ExprEndToEndTest extends FunSuite { Time.withCurrentTimeFrozen { ctl => val elapsed = Stopwatch.start() try { - mySlowQuery(ctl, succeed) + slowQuery(ctl, succeed) successCounter.incr() } catch { case NonFatal(exn) => @@ -68,4 +70,11 @@ class ExprEndToEndTest extends FunSuite { assert(sr.expressions("success_rate_downstream").labels.role == Client) } + + test("create a histogram expression requires a component") { + val e = intercept[IllegalArgumentException] { + ExpressionSchema("latency", Expression(latencyMb)) + } + assert(e.getMessage.contains("provide a component for histogram")) + } } From 14e6dd1a52b8db1e605a03c7a8909fa1acc6c696 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Mon, 29 Mar 2021 20:58:12 +0000 Subject: [PATCH 461/796] Twitter-oss: Prepare OSS libraries for release 21.3.0 Problem/Solution Let's move the `dodo` CHANGELOG to RST so that it gets updated on each release and prep the release for 21.3.0. JIRA Issues: CSL-10793 Differential Revision: https://phabricator.twitter.biz/D641843 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index def2d8991c..cc04e4db2d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.3.0 +------ + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 114881c792..01509d51f8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.2.0" +val utilCore = "com.twitter" %% "util-core" % "21.3.0" ``` # Units diff --git a/build.sbt b/build.sbt index 7cfa3b4187..b26c4a848a 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.3.0-SNAPSHOT" +val releaseVersion = "21.3.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From b53478732cecf2f02f00cc61e4ff5125d6c7c21e Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Mon, 29 Mar 2021 21:38:21 +0000 Subject: [PATCH 462/796] Twitter-oss: update OSS libraries post-release to 21.4.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10793 Differential Revision: https://phabricator.twitter.biz/D642752 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 01509d51f8..0acc5fb78f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b26c4a848a..d0aa09a90b 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.3.0" +val releaseVersion = "21.4.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 01dc2bc3b80a54af4124bd3ee237fb125155e155 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 31 Mar 2021 21:49:23 +0000 Subject: [PATCH 463/796] {util-stats,twitter-server}: Make it possible to configure counter-ish gauges Problem Some metrics we don't control are essentially counters, tracking the amount of something since the start of an application. However, we don't have a good way of signalling that. Solution Although we considered changing them to be implemented as counters, we were concerned that folks who use latched counters would be surprised that these metrics suddenly started being latched. Although we want to transition in that direction eventually, we chose to do it more cautiously. Instead, we kept the status quo, but added a "counterish_gauge" flag to the metrics metadata, so that if a machine is trying to parse it, they can infer that they may want to take the difference between querying the metrics endpoint. Result The jvm cycles and msec metrics can be more correctly reported! JIRA Issues: CSL-10752 Differential Revision: https://phabricator.twitter.biz/D629892 --- util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala | 8 ++++++-- .../scala/com/twitter/finagle/stats/MetricBuilder.scala | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 02f5151c94..ca23be1707 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -125,8 +125,12 @@ object JvmStats { val gcStats = stats.scope("gc") gcPool.foreach { gc => val name = gc.getName.regexSub("""[^\w]""".r) { m => "_" } - gauges.add(gcStats.addGauge(name, "cycles") { gc.getCollectionCount }) - gauges.add(gcStats.addGauge(name, "msec") { gc.getCollectionTime }) + gauges.add(gcStats.metricBuilder().withCounterishGauge.gauge(name, "cycles") { + gc.getCollectionCount + }) + gauges.add(gcStats.metricBuilder().withCounterishGauge.gauge(name, "msec") { + gc.getCollectionTime + }) } // note, these could be -1 if the collector doesn't have support for it. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index e2a602a753..53bff3ec98 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -51,6 +51,7 @@ object metadataScopeSeparator { * @param relativeName the relative metric name which will be appended to the scope of the StatsReceiver prior to long term storage * @param processPath a universal coordinate for the resource * @param percentiles used to indicate buckets for histograms, to be set by the StatsReceiver + * @param counterLikeGauge used to indicate that a gauge is being used to model something that acts like a counter * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder */ class MetricBuilder( @@ -65,6 +66,7 @@ class MetricBuilder( val processPath: Option[String] = None, // Only persisted and relevant when building histograms. val percentiles: IndexedSeq[Double] = IndexedSeq.empty, + val isCounterishGauge: Boolean = false, val statsReceiver: StatsReceiver) { /** @@ -82,6 +84,7 @@ class MetricBuilder( name: Seq[String] = this.name, relativeName: Seq[String] = this.relativeName, processPath: Option[String] = this.processPath, + isCounterishGauge: Boolean = this.isCounterishGauge, percentiles: IndexedSeq[Double] = this.percentiles ): MetricBuilder = { new MetricBuilder( @@ -95,6 +98,7 @@ class MetricBuilder( relativeName = relativeName, processPath = processPath, percentiles = percentiles, + isCounterishGauge = isCounterishGauge, statsReceiver = this.statsReceiver ) } @@ -127,6 +131,10 @@ class MetricBuilder( def withPercentiles(percentiles: Double*): MetricBuilder = this.copy(percentiles = percentiles.toIndexedSeq) + def withCounterishGauge: MetricBuilder = this.copy(isCounterishGauge = true) + + def withNoCounterishGauge: MetricBuilder = this.copy(isCounterishGauge = false) + /** * Generates a CounterSchema which can be used to create a counter in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. From dab0531a2aa274ea89dbb153438df7eab17cc7de Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 1 Apr 2021 23:06:56 +0000 Subject: [PATCH 464/796] metrics: fix service name setting and correct the constant expression Problem 1.The RoleConfiguredStatsReceiver sets the expression service name either the name is None or Some, LoadedStatsReceiver is sitting at the lower level which reset the service name to None, which makes it possible to be missed when registering in MetricStore. 2.ConstantExpression standalone should not be registered as an expression, hence no need to provide a StatsReceiver. Solution 1. Only set service name when it is not None. 2. Remove StatsReceiver in ConstantExpression. Differential Revision: https://phabricator.twitter.biz/D643761 --- .../finagle/stats/RoleConfiguredStatsReceiver.scala | 6 +++++- .../scala/com/twitter/finagle/stats/exp/Expression.scala | 9 +++------ .../com/twitter/finagle/stats/exp/ExpressionSchema.scala | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala index 6c0d3778b2..6846e440dd 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala @@ -28,6 +28,10 @@ case class RoleConfiguredStatsReceiver( } override def registerExpression(expressionSchema: ExpressionSchema): Unit = { - self.registerExpression(expressionSchema.withRole(role = role).withServiceName(name)) + val configuredExprSchema = name match { + case Some(serviceName) => expressionSchema.withRole(role).withServiceName(serviceName) + case None => expressionSchema.withRole(role) + } + self.registerExpression(configuredExprSchema) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index a6e06f19d9..5427d0c75c 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -21,15 +21,13 @@ private[twitter] object Expression { } case MetricExpression(schema) => Set(schema.metricBuilder.statsReceiver) case HistogramExpression(schema, _) => Set(schema.metricBuilder.statsReceiver) - case ConstantExpression(_, statsReceiver) => Set(statsReceiver) + case ConstantExpression(_) => Set.empty } /** * Create a expression with a constant number in double - * @param statsReceiver the statsReceiver to register this expression */ - def apply(num: Double, statsReceiver: StatsReceiver): Expression = - ConstantExpression(num.toString, statsReceiver) + def apply(num: Double): Expression = ConstantExpression(num.toString) /** * Create a histogram expression @@ -69,8 +67,7 @@ private[twitter] sealed trait Expression { /** * Represents a constant double number */ -case class ConstantExpression private (repr: String, statsReceiver: StatsReceiver) - extends Expression +case class ConstantExpression private (repr: String) extends Expression /** * Represents compound metrics and their arithmetical(or others) calculations diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index f6f26984bb..8c90ad9904 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -30,8 +30,8 @@ case class ExpressionSchema private ( private[finagle] def withRole(role: SourceRole): ExpressionSchema = copy(labels = labels.copy(role = role)) - private[finagle] def withServiceName(name: Option[String]): ExpressionSchema = - copy(labels = labels.copy(serviceName = name)) + private[finagle] def withServiceName(name: String): ExpressionSchema = + copy(labels = labels.copy(serviceName = Some(name))) def register(): Unit = { Expression.getStatsReceivers(expr).toSeq match { From 291af4bb83e50dae54abb6649e48541263401ff6 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 6 Apr 2021 23:42:16 +0000 Subject: [PATCH 465/796] metrics: add yufang and mdannenberg as owners Differential Revision: https://phabricator.twitter.biz/D647575 --- util-stats/PROJECT | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 22a3c9c5f6..0ebef1dd2f 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -4,6 +4,8 @@ owners: - banderson - beisenman - koliver + - mdannenberg - mnakamura - roanta - vkostyukov + - yufang \ No newline at end of file From e1afa7cf5c3b6f4f396251e8a636c2027e2fae77 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 7 Apr 2021 08:31:37 +0000 Subject: [PATCH 466/796] metrics: Add kernel for memoizing the metric builders Problem Metric metadata including fully scoped names is filled through underlying StatsReceiverProxies. This means the same MetricSchema we passed in Expressions doesn't have any metadata. Solution For metrics we used to build expressions, we hash the MetricBuilder and memorizing in the Metric storage using the object reference hashCode as hashMap key. For registered expressions, replace all MetricBuilders with the fully hydrated ones. Result This is a workaround solution to buy us sometime re-design the StatsReceiver to carry MetricBuilder information so we don't need to back-filling. JIRA Issues: CSL-10845 Differential Revision: https://phabricator.twitter.biz/D645590 --- .../finagle/stats/InMemoryStatsReceiver.scala | 35 +++++- .../twitter/finagle/stats/MetricBuilder.scala | 104 +++++++++++++----- .../twitter/finagle/stats/StatsReceiver.scala | 2 +- .../finagle/stats/exp/Expression.scala | 39 ++++++- .../stats/InMemoryStatsReceiverTest.scala | 31 +++++- .../finagle/stats/MetricBuilderTest.scala | 14 ++- .../stats/exp/ExpressionSchemaTest.scala | 12 +- 7 files changed, 198 insertions(+), 39 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 760b314717..b4fab892c2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -1,5 +1,6 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.exp.Expression.replaceExpression import com.twitter.finagle.stats.exp.ExpressionSchema import java.io.PrintStream import java.util.Locale @@ -70,8 +71,19 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { val schemas: mutable.Map[Seq[String], MetricSchema] = new ConcurrentHashMap[Seq[String], MetricSchema]().asScala - val expressions: mutable.Map[String, ExpressionSchema] = - new ConcurrentHashMap[String, ExpressionSchema]().asScala + private val metricBuilders: ConcurrentHashMap[Int, MetricBuilder] = + new ConcurrentHashMap[Int, MetricBuilder]() + + private val preloadExpressions: ConcurrentHashMap[String, ExpressionSchema] = + new ConcurrentHashMap[String, ExpressionSchema]() + + def expressions: mutable.Map[String, ExpressionSchema] = { + preloadExpressions.replaceAll { + case (_, exprSchema) => + exprSchema.copy(expr = replaceExpression(exprSchema.expr, metricBuilders)) + } + preloadExpressions.asScala + } override def counter(name: String*): ReadableCounter = counter(CounterSchema(this.metricBuilder().withName(name: _*))) @@ -89,6 +101,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { if (!counters.contains(schema.metricBuilder.name)) { counters(schema.metricBuilder.name) = 0 schemas(schema.metricBuilder.name) = schema + storeMetricBuilder(schema) } } @@ -119,6 +132,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { if (!stats.contains(schema.metricBuilder.name)) { stats(schema.metricBuilder.name) = Nil schemas(schema.metricBuilder.name) = schema + storeMetricBuilder(schema) } } @@ -143,9 +157,11 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { gauges += schema.metricBuilder.name -> (() => f) schemas += schema.metricBuilder.name -> schema verbosity += schema.metricBuilder.name -> schema.metricBuilder.verbosity + storeMetricBuilder(schema) def remove(): Unit = { gauges -= schema.metricBuilder.name + removeMetricBuilder(schema) schemas -= schema.metricBuilder.name } @@ -159,6 +175,19 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } } + private[this] def storeMetricBuilder(schema: MetricSchema): Unit = { + schema.metricBuilder.kernel match { + case Some(kernel) => metricBuilders.put(kernel, schema.metricBuilder) + case None => + } + } + + private[this] def removeMetricBuilder(schema: MetricSchema): Unit = { + schema.metricBuilder.kernel match { + case Some(kernel) => metricBuilders.remove(kernel) + case None => + } + } override def toString: String = "InMemoryStatsReceiver" /** @@ -254,7 +283,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { case Some(serviceName) => schema.name + "_" + serviceName case None => schema.name } - expressions(expressionId) = schema + preloadExpressions.put(expressionId, schema) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 53bff3ec98..f058e9581e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -36,37 +36,76 @@ object metadataScopeSeparator { } } +object MetricBuilder { + + /** + * Construct a MethodBuilder. + * + * @param keyIndicator indicates whether this metric is crucial to this service (ie, an SLO metric) + * @param description human-readable description of a metric's significance + * @param units the unit associated with the metrics value (milliseconds, megabytes, requests, etc) + * @param role whether the service is playing the part of client or server regarding this metric + * @param verbosity see StatsReceiver for details + * @param sourceClass the name of the class which generated this metric (ie, com.twitter.finagle.StatsFilter) + * @param name the full metric name + * @param relativeName the relative metric name which will be appended to the scope of the StatsReceiver prior to long term storage + * @param processPath a universal coordinate for the resource + * @param percentiles used to indicate buckets for histograms, to be set by the StatsReceiver + * @param isCounterishGauge used to indicate that a gauge is being used to model something that acts like a counter + * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder + */ + def apply( + keyIndicator: Boolean = false, + description: String = "No description provided", + units: MetricUnit = Unspecified, + role: SourceRole = NoRoleSpecified, + verbosity: Verbosity = Verbosity.Default, + sourceClass: Option[String] = None, + name: Seq[String] = Seq.empty, + relativeName: Seq[String] = Seq.empty, + processPath: Option[String] = None, + percentiles: IndexedSeq[Double] = IndexedSeq.empty, + isCounterishGauge: Boolean = false, + statsReceiver: StatsReceiver + ): MetricBuilder = { + new MetricBuilder( + keyIndicator, + description, + units, + role, + verbosity, + sourceClass, + name, + relativeName, + processPath, + percentiles, + isCounterishGauge, + kernel = None, + statsReceiver + ) + } +} + /** * A builder class used to configure settings and metadata for metrics prior to instantiating them. * Calling any of the three build methods (counter, gauge, or histogram) will cause the metric to be * instantiated in the underlying StatsReceiver. - * - * @param keyIndicator indicates whether this metric is crucial to this service (ie, an SLO metric) - * @param description human-readable description of a metric's significance - * @param units the unit associated with the metrics value (milliseconds, megabytes, requests, etc) - * @param role whether the service is playing the part of client or server regarding this metric - * @param verbosity see StatsReceiver for details - * @param sourceClass the name of the class which generated this metric (ie, com.twitter.finagle.StatsFilter) - * @param name the full metric name - * @param relativeName the relative metric name which will be appended to the scope of the StatsReceiver prior to long term storage - * @param processPath a universal coordinate for the resource - * @param percentiles used to indicate buckets for histograms, to be set by the StatsReceiver - * @param counterLikeGauge used to indicate that a gauge is being used to model something that acts like a counter - * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder */ -class MetricBuilder( - val keyIndicator: Boolean = false, - val description: String = "No description provided", - val units: MetricUnit = Unspecified, - val role: SourceRole = NoRoleSpecified, - val verbosity: Verbosity = Verbosity.Default, - val sourceClass: Option[String] = None, - val name: Seq[String] = Seq.empty, - val relativeName: Seq[String] = Seq.empty, - val processPath: Option[String] = None, +class MetricBuilder private ( + val keyIndicator: Boolean, + val description: String, + val units: MetricUnit, + val role: SourceRole, + val verbosity: Verbosity, + val sourceClass: Option[String], + val name: Seq[String], + val relativeName: Seq[String], + val processPath: Option[String], // Only persisted and relevant when building histograms. - val percentiles: IndexedSeq[Double] = IndexedSeq.empty, - val isCounterishGauge: Boolean = false, + val percentiles: IndexedSeq[Double], + val isCounterishGauge: Boolean, + // see withKernel + val kernel: Option[Int], val statsReceiver: StatsReceiver) { /** @@ -85,7 +124,8 @@ class MetricBuilder( relativeName: Seq[String] = this.relativeName, processPath: Option[String] = this.processPath, isCounterishGauge: Boolean = this.isCounterishGauge, - percentiles: IndexedSeq[Double] = this.percentiles + percentiles: IndexedSeq[Double] = this.percentiles, + kernel: Option[Int] = this.kernel ): MetricBuilder = { new MetricBuilder( keyIndicator = keyIndicator, @@ -99,10 +139,20 @@ class MetricBuilder( processPath = processPath, percentiles = percentiles, isCounterishGauge = isCounterishGauge, - statsReceiver = this.statsReceiver + statsReceiver = this.statsReceiver, + kernel = kernel ) } + // This is a work-around for memoizing this MetricBuilder, it is the object reference + // hashCode of `this` MetricBuilder, only explicitly configured for metrics used for expressions. + // Metric use it as the hash key to find the fully hydrated MetricBuilder. + // Copying the MetricBuilder to configure other metadata must keep the kernel untouched. + private[finagle] def withKernel: MetricBuilder = { + require(this.kernel.isEmpty, "This MetricBuilder is already hashed") + this.copy(kernel = Some(System.identityHashCode(this))) + } + def withKeyIndicator(isKeyIndicator: Boolean = true): MetricBuilder = this.copy(keyIndicator = isKeyIndicator) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 92986e7aa0..6bc1b0c771 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -108,7 +108,7 @@ trait StatsReceiver { /** * Get a [[MetricBuilder metricBuilder]] for this StatsReceiver. */ - def metricBuilder(): MetricBuilder = new MetricBuilder(statsReceiver = this) + def metricBuilder(): MetricBuilder = MetricBuilder(statsReceiver = this) /** * Get a [[Counter counter]] with the given `name`. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index 5427d0c75c..bb66fa6e09 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -1,7 +1,15 @@ package com.twitter.finagle.stats.exp import com.twitter.finagle.stats.exp.Expression.HistogramComponent -import com.twitter.finagle.stats.{HistogramSchema, MetricSchema, StatsReceiver} +import com.twitter.finagle.stats.{ + CounterSchema, + GaugeSchema, + HistogramSchema, + MetricBuilder, + MetricSchema, + StatsReceiver +} +import java.util.concurrent.ConcurrentHashMap import scala.annotation.varargs private[twitter] object Expression { @@ -44,6 +52,35 @@ private[twitter] object Expression { require(!schema.isInstanceOf[HistogramSchema], "provide a component for histogram") MetricExpression(schema) } + + // utility methods shared by Metrics.scala and InMemoryStatsReceiver + private[stats] def replaceExpression( + expression: Expression, + metricBuilders: ConcurrentHashMap[Int, MetricBuilder] + ): Expression = { + expression match { + case f @ FunctionExpression(_, exprs) => + f.copy(exprs = exprs.map(replaceExpression(_, metricBuilders))) + case m @ MetricExpression(schema) if schema.metricBuilder.kernel.isDefined => + val builder = metricBuilders.get(schema.metricBuilder.kernel.get) + if (builder != null) m.copy(schema = reformSchema(schema, builder)) + else m + case h @ HistogramExpression(schema, _) if schema.metricBuilder.kernel.isDefined => + val builder = metricBuilders.get(schema.metricBuilder.kernel.get) + if (builder != null) + h.copy(schema = reformSchema(schema, builder).asInstanceOf[HistogramSchema]) + else h + case otherExpression => otherExpression + } + } + + private[this] def reformSchema(schema: MetricSchema, builder: MetricBuilder): MetricSchema = { + schema match { + case CounterSchema(_) => CounterSchema(builder) + case HistogramSchema(_) => HistogramSchema(builder) + case GaugeSchema(_) => GaugeSchema(builder) + } + } } /** diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index b143c6bdcb..01a6894cdc 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -124,7 +124,7 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra val stats = new InMemoryStatsReceiver() ExpressionSchema( "a", - Expression(CounterSchema(new MetricBuilder(name = Seq("counter"), statsReceiver = stats)))) + Expression(CounterSchema(MetricBuilder(name = Seq("counter"), statsReceiver = stats)))) stats.expressions.contains("a") } @@ -364,4 +364,33 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra ps.close() } } + + test("expression are reloaded with fully scoped names") { + val sr = new InMemoryStatsReceiver + + val aSchema = CounterSchema(MetricBuilder(name = Seq("a"), statsReceiver = sr).withKernel) + val bSchema = HistogramSchema(MetricBuilder(name = Seq("b"), statsReceiver = sr).withKernel) + val cSchema = GaugeSchema(MetricBuilder(name = Seq("c"), statsReceiver = sr).withKernel) + + val expression = ExpressionSchema( + "test_expression", + Expression(aSchema).plus(Expression(bSchema, Left(Expression.Min)).plus(Expression(cSchema)))) + .register() + + val aCounter = sr.scope("test").counter(aSchema) + val bHisto = sr.scope("test").stat(bSchema) + val cGauge = sr.scope(("test")).addGauge(cSchema) { 1 } + + // what we expected as hydrated metric builders + val aaSchema = CounterSchema(MetricBuilder(name = Seq("test", "a"), statsReceiver = sr)) + val bbSchema = HistogramSchema(MetricBuilder(name = Seq("test", "b"), statsReceiver = sr)) + val ccSchema = GaugeSchema(MetricBuilder(name = Seq("test", "c"), statsReceiver = sr)) + + val expected_expression = ExpressionSchema( + "test_expression", + Expression(aaSchema).plus( + Expression(bbSchema, Left(Expression.Min)).plus(Expression(ccSchema)))) + + assert(sr.expressions.get("test_expression").get.expr == expected_expression.expr) + } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala index ead3a74cdd..4474095f4c 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala @@ -5,7 +5,7 @@ import org.scalatest.funsuite.AnyFunSuite class MetricBuilderTest extends AnyFunSuite { test("metadataScopeSeparator affects stringification") { - val mb = new MetricBuilder(name = Seq("foo", "bar"), statsReceiver = new InMemoryStatsReceiver) + val mb = MetricBuilder(name = Seq("foo", "bar"), statsReceiver = new InMemoryStatsReceiver) assert(mb.toString.contains("foo/bar")) metadataScopeSeparator.setSeparator("-") assert(mb.toString.contains("foo-bar")) @@ -13,4 +13,16 @@ class MetricBuilderTest extends AnyFunSuite { assert(mb.toString.contains("foo_aTerriblyLongStringForSomeReason_bar")) } + test("kernels are object reference hashCode") { + val sr = new InMemoryStatsReceiver + val mb1 = MetricBuilder(name = Seq("a"), statsReceiver = sr).withKernel + val mb2 = MetricBuilder(name = Seq("a"), statsReceiver = sr).withKernel + assert(mb2.kernel != mb1.kernel) + } + + test("kernels remain the same after copy") { + val mb1 = MetricBuilder(name = Seq("a"), statsReceiver = new InMemoryStatsReceiver).withKernel + val mb2 = mb1.withDescription("I am a description").withName("a", "b") + assert(mb1.kernel == mb2.kernel) + } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index fb87910ea9..be04d3d311 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -12,11 +12,11 @@ class ExpressionSchemaTest extends FunSuite { val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) val successMb = - CounterSchema(new MetricBuilder(name = Seq("success"), statsReceiver = clientSR)) + CounterSchema(MetricBuilder(name = Seq("success"), statsReceiver = clientSR).withKernel) val failuresMb = - CounterSchema(new MetricBuilder(name = Seq("failures"), statsReceiver = clientSR)) + CounterSchema(MetricBuilder(name = Seq("failures"), statsReceiver = clientSR).withKernel) val latencyMb = - HistogramSchema(new MetricBuilder(name = Seq("latency"), statsReceiver = clientSR)) + HistogramSchema(MetricBuilder(name = Seq("latency"), statsReceiver = clientSR).withKernel) val sum = Expression(successMb).plus(Expression(failuresMb)) val successRate = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) @@ -61,8 +61,10 @@ class ExpressionSchemaTest extends FunSuite { runTheQuery(true) runTheQuery(false) - assert(sr.expressions("success_rate_downstream").expr == successRate.expr) - assert(sr.expressions("latency_downstream").expr == latency.expr) + assert(sr.expressions("success_rate_downstream").name == successRate.name) + assert(sr.expressions("success_rate_downstream").expr != successRate.expr) + assert(sr.expressions("latency_downstream").name == latency.name) + assert(sr.expressions("latency_downstream").expr != latency.expr) assert(sr.counters(Seq("success")) == 1) assert(sr.counters(Seq("failures")) == 1) From a29e97185f2930a456065414fbd471e2c35fc58c Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Wed, 7 Apr 2021 21:06:37 +0000 Subject: [PATCH 467/796] util-core: Remove c.t.io.{BufReaders, Readers, Writers} Problem/Solution: These files are deprecated and only necessary for scala 2.11 compatibility. Lets remove them. JIRA Issues: CSL-10817 Differential Revision: https://phabricator.twitter.biz/D645254 --- .../main/java/com/twitter/io/BufReaders.java | 25 ----- .../src/main/java/com/twitter/io/Readers.java | 97 ------------------- .../src/main/java/com/twitter/io/Writers.java | 31 ------ .../main/scala/com/twitter/io/Reader.scala | 10 ++ .../com/twitter/io/ReaderCompilationTest.java | 49 ---------- .../com/twitter/io/WriterCompilationTest.java | 16 --- 6 files changed, 10 insertions(+), 218 deletions(-) delete mode 100644 util-core/src/main/java/com/twitter/io/BufReaders.java delete mode 100644 util-core/src/main/java/com/twitter/io/Readers.java delete mode 100644 util-core/src/main/java/com/twitter/io/Writers.java delete mode 100644 util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java delete mode 100644 util-core/src/test/java/com/twitter/io/WriterCompilationTest.java diff --git a/util-core/src/main/java/com/twitter/io/BufReaders.java b/util-core/src/main/java/com/twitter/io/BufReaders.java deleted file mode 100644 index 04f9b6c8fe..0000000000 --- a/util-core/src/main/java/com/twitter/io/BufReaders.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.twitter.io; - -import com.twitter.util.Future; - -/** - * Better Java APIs of BufReader for Scala 2.11. - * This will be removed when drop Scala 2.11 support. - * - * @see com.twitter.io.BufReader - * @deprecated This will no longer be necessary when 2.11 support is dropped. 2019-10-04 - */ -@Deprecated() -public final class BufReaders { - - private BufReaders() { - throw new IllegalStateException(); - } - - /** - * See {@code com.twitter.io.BufReader.readAll}. - */ - public static Future readAll(Reader r) { - return BufReader$.MODULE$.readAll(r); - } -} diff --git a/util-core/src/main/java/com/twitter/io/Readers.java b/util-core/src/main/java/com/twitter/io/Readers.java deleted file mode 100644 index 4e684ce5e3..0000000000 --- a/util-core/src/main/java/com/twitter/io/Readers.java +++ /dev/null @@ -1,97 +0,0 @@ -/* Copyright 2015 Twitter, Inc. */ -package com.twitter.io; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.util.List; -import java.util.stream.Stream; - -import com.twitter.concurrent.AsyncStream; -import com.twitter.util.Future; - -import scala.collection.JavaConverters; -import scala.runtime.BoxedUnit; - -/** - * Better Java APIs of Reader for Scala 2.11. - * This will be removed when drop Scala 2.11 support. - * - * @see com.twitter.io.Reader - * @deprecated This will no longer be necessary when 2.11 support is dropped. 2019-10-04 - */ -@Deprecated -public final class Readers { - - private Readers() { throw new IllegalStateException(); } - - public static Reader newEmptyReader() { - return Reader$.MODULE$.empty(); - } - - /** - * See {@code com.twitter.io.Reader.fromBuf}. - */ - public static Reader newBufReader(Buf buf, int chunkSize) { - return Reader$.MODULE$.fromBuf(buf, chunkSize); - } - - /** - * See {@code com.twitter.io.Reader.concat}. - */ - public static Reader concat(AsyncStream> readers) { - return Reader$.MODULE$.concat(readers); - } - - /** - * See [[com.twitter.io.Reader.concat(scala.collection.Seq)]] - */ - public static Reader concat(List> list) { - return Reader$.MODULE$.concat(JavaConverters.asScalaIteratorConverter(list.iterator()) - .asScala().toSeq()); - } - - /** - * See {@code com.twitter.io.Pipe.copy}. - */ - public static Future copy(Reader r, Writer w) { - return Pipe$.MODULE$.copy(r, w); - } - - /** - * See {@code com.twitter.io.Reader.fromFile()}. - */ - public static Reader newFileReader(File f, int chunkSize) throws FileNotFoundException { - return Reader$.MODULE$.fromFile(f, chunkSize); - } - - /** - * See {@code com.twitter.io.Reader.fromStream()}. - */ - public static Reader newInputStreamReader(InputStream in, int chunkSize) { - return Reader$.MODULE$.fromStream(in, chunkSize); - } - - /** - * See [[com.twitter.io.Reader.fromSeq(scala.collection.Seq)]] - */ - public static Reader fromSeq(List list) { - return Reader$.MODULE$.fromSeq(JavaConverters.asScalaIteratorConverter(list.iterator()) - .asScala().toSeq()); - } - - /** - * See [[com.twitter.io.Reader.fromSeq(scala.collection.Seq)]] - */ - public static Reader fromSeq(Stream stream) { - return Reader$.MODULE$.fromSeq(JavaConverters.asScalaIteratorConverter(stream.iterator()) - .asScala().toStream()); - } - - /** - * See [[com.twitter.io.Reader.toAsyncStream]] - */ - public static AsyncStream toAsyncStream(Reader reader) { - return Reader$.MODULE$.toAsyncStream(reader); - } -} diff --git a/util-core/src/main/java/com/twitter/io/Writers.java b/util-core/src/main/java/com/twitter/io/Writers.java deleted file mode 100644 index 0a1895240d..0000000000 --- a/util-core/src/main/java/com/twitter/io/Writers.java +++ /dev/null @@ -1,31 +0,0 @@ -/* Copyright 2015 Twitter, Inc. */ -package com.twitter.io; - -import java.io.OutputStream; - -/** - * Java APIs for Writer. - * - * @see com.twitter.io.Writer - * @deprecated This will no longer be necessary when 2.11 support is dropped. 2019-10-04 - */ -@Deprecated -public final class Writers { - - private Writers() { throw new IllegalStateException(); } - - /** - * See {@code com.twitter.io.Writer.fromOutputStream} - */ - public static Writer newOutputStreamWriter(OutputStream out) { - return Writer$.MODULE$.fromOutputStream(out); - } - - /** - * See {@code com.twitter.io.Writer.fromOutputStream} - */ - public static Writer newOutputStreamWriter(OutputStream out, int bufsize) { - return Writer$.MODULE$.fromOutputStream(out, bufsize); - } - -} diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 4b308ad20b..7c264a2e91 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -4,6 +4,8 @@ import com.twitter.concurrent.AsyncStream import com.twitter.util.Promise.Detachable import com.twitter.util._ import java.io.{File, FileInputStream, InputStream} +import java.util.{Collection => JCollection} +import scala.collection.JavaConverters /** * A reader exposes a pull-based API to model a potentially infinite stream of arbitrary elements. @@ -284,6 +286,14 @@ object Reader { */ def fromSeq[A](seq: Seq[A]): Reader[A] = fromIterator(seq.iterator) + /** + * Java-Friendly version of fromSeq + * Create a new [[Reader]] from a given Java `List` + */ + def fromCollection[A](list: JCollection[A]): Reader[A] = fromSeq( + JavaConverters.asScalaIteratorConverter(list.iterator).asScala.toSeq + ) + /** * Allow [[com.twitter.concurrent.AsyncStream]] to be consumed as a [[Reader]] */ diff --git a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java b/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java deleted file mode 100644 index 61c6d54180..0000000000 --- a/util-core/src/test/java/com/twitter/io/ReaderCompilationTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* Copyright 2015 Twitter, Inc. */ -package com.twitter.io; - -import com.twitter.concurrent.AsyncStream; -import java.io.ByteArrayInputStream; -import java.io.File; - -import org.junit.Test; - -import static org.junit.Assert.assertNotNull; - -public class ReaderCompilationTest { - - @Test - public void testNewBufReader() { - Readers.newBufReader(Bufs.EMPTY, 1024); - } - - @Test - public void testReadAll() { - BufReaders.readAll(Readers.newEmptyReader()); - } - - @Test - public void testConcat() { - Readers.concat(AsyncStream.>empty()); - } - - - @Test - public void testPipe() { - Pipe w = new Pipe<>(); - w.close(); - } - - @Test - public void testNewFileReader() { - try { - Readers.newFileReader(new File("a") ,1024); - } catch (Exception x) { - // ok - } - } - - @Test - public void testNewInputStreamReader() throws Exception { - Readers.newInputStreamReader(new ByteArrayInputStream(new byte[0]), 1024); - } -} diff --git a/util-core/src/test/java/com/twitter/io/WriterCompilationTest.java b/util-core/src/test/java/com/twitter/io/WriterCompilationTest.java deleted file mode 100644 index d30f24441e..0000000000 --- a/util-core/src/test/java/com/twitter/io/WriterCompilationTest.java +++ /dev/null @@ -1,16 +0,0 @@ -/* Copyright 2015 Twitter, Inc. */ -package com.twitter.io; - -import java.io.ByteArrayOutputStream; - -import org.junit.Test; - -public class WriterCompilationTest { - - @Test - public void testNewOutputStreamWriter() { - Writer w = Writers.newOutputStreamWriter(new ByteArrayOutputStream()); - w.close(); - } - -} From 0cb8dc864963e1db8dd9d7b5ebfcbceb5faa3fd4 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 8 Apr 2021 21:16:54 +0000 Subject: [PATCH 468/796] oss: change github link for CSL projects Problem/Solution Change github links to `blob/release` from `blob/master` JIRA Issues: CSL-10831 Differential Revision: https://phabricator.twitter.biz/D648883 --- CONTRIBUTING.md | 4 ++-- doc/src/sphinx/util-stats/user_guide.rst | 6 +++--- util-core/src/main/scala/com/twitter/util/Timer.scala | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21dba985b1..136c2d41a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -178,7 +178,7 @@ metadata will be preserved. Please let us know if you have any questions about this process! [0]: https://github.com/twitter/util/pull/109 -[1]: https://github.com/twitter/util/blob/master/util-core/src/test/java/com/twitter/util/VarCompilationTest.java +[1]: https://github.com/twitter/util/blob/release/util-core/src/test/java/com/twitter/util/VarCompilationTest.java [changes]: https://github.com/twitter/util/blob/develop/CHANGELOG.rst [es]: https://twitter.github.io/effectivescala/ [funsuite]: https://www.scalatest.org/getting_started_with_fun_suite @@ -189,4 +189,4 @@ Please let us know if you have any questions about this process! [scalatest]: https://www.scalatest.org/ [sd]: https://docs.scala-lang.org/style/scaladoc.html [ssg]: https://docs.scala-lang.org/style/scaladoc.html -[travis-ci]: https://travis-ci.org/twitter/util \ No newline at end of file +[travis-ci]: https://travis-ci.org/twitter/util diff --git a/doc/src/sphinx/util-stats/user_guide.rst b/doc/src/sphinx/util-stats/user_guide.rst index 68ec17dbbb..48cfcef612 100644 --- a/doc/src/sphinx/util-stats/user_guide.rst +++ b/doc/src/sphinx/util-stats/user_guide.rst @@ -194,14 +194,14 @@ using. Via TwitterServer/finagle-stats — the `HTTP admin interface`_ responds with json at ``/admin/metrics.json`` and there is a web UI for watching them in real-time at ``/admin/metrics``. -.. _LoadService: https://github.com/twitter/finagle/blob/master/finagle-core/src/main/scala/com/twitter/finagle/util/LoadService.scala +.. _LoadService: https://github.com/twitter/finagle/blob/release/finagle-core/src/main/scala/com/twitter/finagle/util/LoadService.scala .. _finagle-stats: https://github.com/twitter/finagle/tree/master/finagle-stats -.. _BroadcastStatsReceiver: https://github.com/twitter/util/blob/master/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala +.. _BroadcastStatsReceiver: https://github.com/twitter/util/blob/release/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala .. _NullStatsReceiver: https://github.com/twitter/util/blob/develop/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala .. _Futures: https://twitter.github.io/finagle/guide/Futures.html .. _Stat.time: https://github.com/twitter/util/blob/develop/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala .. _Stat.timeFuture: https://github.com/twitter/util/blob/develop/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala .. _java.lang.ref.WeakReference: https://docs.oracle.com/javase/8/docs/api/java/lang/ref/WeakReference.html -.. _InMemoryStatsReceiver: https://github.com/twitter/util/blob/master/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +.. _InMemoryStatsReceiver: https://github.com/twitter/util/blob/release/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala .. _HTTP admin interface: https://twitter.github.io/twitter-server/Features.html#http-admin-interface .. _DenylistStatsReceiver: https://github.com/twitter/util/blob/develop/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala diff --git a/util-core/src/main/scala/com/twitter/util/Timer.scala b/util-core/src/main/scala/com/twitter/util/Timer.scala index a5890c53ea..6c2f0ecc57 100644 --- a/util-core/src/main/scala/com/twitter/util/Timer.scala +++ b/util-core/src/main/scala/com/twitter/util/Timer.scala @@ -195,7 +195,7 @@ class ReferenceCountingTimer(factory: () => Timer) extends ProxyTimer with Refer * If your code has a reasonably high throughput of task scheduling * and can trade off some precision of when tasks run, * [[https://twitter.github.io/finagle/ Finagle]] has a higher throughput - * [[https://github.com/twitter/finagle/blob/master/finagle-core/src/main/scala/com/twitter/finagle/util/DefaultTimer.scala + * [[https://github.com/twitter/finagle/blob/release/finagle-core/src/main/scala/com/twitter/finagle/util/DefaultTimer.scala * hashed-wheel implementation]]. * * @note Due to the implementation using a single `Thread`, be wary of From 1ba6457412c84526906a4a3539c55133296ebfca Mon Sep 17 00:00:00 2001 From: Tushar Singh Date: Fri, 16 Apr 2021 01:27:35 +0000 Subject: [PATCH 469/796] util-security: Make KeyManager, TrustManager Factory helper methods public [Problem] Many common KeyManager, TrustManager Factory conversion helper methods inside com.twitter.util.security are private and cannot be used externally by other packages. [Solution] This revision exposes the visibility of those helper functions to be used publicly. [Result] KeyManager, TrustManager Factory helper methods are exposed for external use. JIRA Issues: RTC-69 Differential Revision: https://phabricator.twitter.biz/D644502 --- CHANGELOG.rst | 4 ++ .../com/twitter/util/security/PemBytes.scala | 17 +++++- .../security/Pkcs8EncodedKeySpecFile.scala | 4 +- .../security/Pkcs8KeyManagerFactory.scala | 58 +++++++++++-------- .../util/security/PrivateKeyFile.scala | 46 +++++++-------- .../security/X509TrustManagerFactory.scala | 20 ++++--- .../twitter/util/security/PemBytesTest.scala | 53 +++++++++++++++++ 7 files changed, 141 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cc04e4db2d..0f6e130362 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,6 +35,10 @@ Breaking API Changes companion object, `PemBytes#fromFile`. Note that unlike before with construction, we read from the file, so it's possible for it to throw. ``PHAB_ID=D641088`` +* util-security: Make KeyManager, TrustManager Factory helper methods public. The main change here + is that we expose util methods that are used to generate TrustManagerFactory, KeyManagerFactory + inside `c.t.u.security` for public usage. ``PHAB_ID=D644502`` + New Features ~~~~~~~~~~~~ diff --git a/util-security/src/main/scala/com/twitter/util/security/PemBytes.scala b/util-security/src/main/scala/com/twitter/util/security/PemBytes.scala index 50910a20fb..e5bb7e2f82 100644 --- a/util-security/src/main/scala/com/twitter/util/security/PemBytes.scala +++ b/util-security/src/main/scala/com/twitter/util/security/PemBytes.scala @@ -171,8 +171,19 @@ object PemBytes { /** * Reads the given file and constructs a `PemBytes` with the contents. */ - def fromFile(file: File): Try[PemBytes] = Try { - val buffered = new String(Files.readAllBytes(file.toPath), StandardCharsets.UTF_8) - new PemBytes(buffered, file.getName) + def fromFile(inputFile: File): Try[PemBytes] = { + Try { + Files.readAllBytes(inputFile.toPath) + }.flatMap { file => + fromBytes(file, inputFile.getName) + } + } + + /** + * Reads the given byte array and constructs a `PemBytes` with the contents. + */ + def fromBytes(bytes: Array[Byte], name: String): Try[PemBytes] = Try { + val buffered = new String(bytes, StandardCharsets.UTF_8) + new PemBytes(buffered, name) } } diff --git a/util-security/src/main/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFile.scala b/util-security/src/main/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFile.scala index 59946b612b..2e42f5223c 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFile.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Pkcs8EncodedKeySpecFile.scala @@ -32,8 +32,8 @@ class Pkcs8EncodedKeySpecFile(file: File) { } } -private object Pkcs8EncodedKeySpecFile { - private val MessageType: String = "PRIVATE KEY" +object Pkcs8EncodedKeySpecFile { + val MessageType: String = "PRIVATE KEY" private val log = Logger.get("com.twitter.util.security") } diff --git a/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala b/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala index f7c2741acc..44d4d9f6e2 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Pkcs8KeyManagerFactory.scala @@ -22,28 +22,6 @@ class Pkcs8KeyManagerFactory(certsFile: File, keyFile: File) { s"failed to create key manager: ${ex.getMessage()}." ) - private[this] def keySpecToPrivateKey(keySpec: PKCS8EncodedKeySpec): PrivateKey = { - val kf: KeyFactory = KeyFactory.getInstance("RSA") - kf.generatePrivate(keySpec) - } - - private[this] def createKeyStore( - certs: Seq[X509Certificate], - privateKey: PrivateKey - ): KeyStore = { - val alias: String = UUID.randomUUID().toString() - val ks: KeyStore = KeyStore.getInstance("JKS") - ks.load(null) - ks.setKeyEntry(alias, privateKey, "".toCharArray(), certs.toArray) - ks - } - - private[this] def keyStoreToKeyManagers(keyStore: KeyStore): Array[KeyManager] = { - val kmf = KeyManagerFactory.getInstance("SunX509") - kmf.init(keyStore, "".toCharArray()) - kmf.getKeyManagers() - } - /** * Attempts to read the contents of both the X.509 Certificates file and the PKCS#8 * Private Key file and combine the contents into a `javax.net.ssl.KeyManager` @@ -59,12 +37,12 @@ class Pkcs8KeyManagerFactory(certsFile: File, keyFile: File) { val tryCertsKey: Try[(Seq[X509Certificate], PrivateKey)] = join(tryCerts, tryPrivateKey) val tryKeyStore: Try[KeyStore] = tryCertsKey.map((createKeyStore _).tupled) - tryKeyStore.map(keyStoreToKeyManagers).onFailure(logException) + tryKeyStore.map(keyStoreToKeyManagerFactory(_).getKeyManagers()).onFailure(logException) } } -private object Pkcs8KeyManagerFactory { +object Pkcs8KeyManagerFactory { private val log = Logger.get("com.twitter.util.security") private def join[A, B](tryA: Try[A], tryB: Try[B]): Try[(A, B)] = { @@ -74,4 +52,36 @@ private object Pkcs8KeyManagerFactory { case (_, Throw(_)) => tryB.asInstanceOf[Try[(A, B)]] } } + + /** + * Convert a `PKCS8EncodedKeySpec` into a `java.security.PrivateKey` + */ + def keySpecToPrivateKey(keySpec: PKCS8EncodedKeySpec): PrivateKey = { + val kf: KeyFactory = KeyFactory.getInstance("RSA") + kf.generatePrivate(keySpec) + } + + /** + * Use the loaded `java.security.KeyStore` to construct a + * `javax.net.ssl.KeyManagerFactory`. + */ + def keyStoreToKeyManagerFactory(keyStore: KeyStore): KeyManagerFactory = { + val kmf = KeyManagerFactory.getInstance("SunX509") + kmf.init(keyStore, "".toCharArray()) + kmf + } + + /** + * Load passed in X.509 certificates into a `java.security.KeyStore` + */ + def createKeyStore( + certs: Seq[X509Certificate], + privateKey: PrivateKey + ): KeyStore = { + val alias: String = UUID.randomUUID().toString() + val ks: KeyStore = KeyStore.getInstance("JKS") + ks.load(null) + ks.setKeyEntry(alias, privateKey, "".toCharArray(), certs.toArray) + ks + } } diff --git a/util-security/src/main/scala/com/twitter/util/security/PrivateKeyFile.scala b/util-security/src/main/scala/com/twitter/util/security/PrivateKeyFile.scala index 5fb18c86c2..b588983955 100644 --- a/util-security/src/main/scala/com/twitter/util/security/PrivateKeyFile.scala +++ b/util-security/src/main/scala/com/twitter/util/security/PrivateKeyFile.scala @@ -21,6 +21,28 @@ class PrivateKeyFile(file: File) { private[this] def logException(ex: Throwable): Unit = log.warning(s"PrivateKeyFile (${file.getName()}) failed to load: ${ex.getMessage()}.") + /** + * Attemps to read the contents of an unencrypted PrivateKey file. + * + * @note Throws `InvalidPemFormatException` if the file is not a PEM format file. + * + * @note Throws `InvalidKeySpecException` if the file cannot be loaded as an + * RSA, DSA, or EC PrivateKey. + * + * @note This method will attempt to load the key first as an RSA key, then + * as a DSA key, and then as an EC (Elliptic Curve) key. + */ + def readPrivateKey(): Try[PrivateKey] = + new Pkcs8EncodedKeySpecFile(file) + .readPkcs8EncodedKeySpec() + .flatMap(keySpecToPrivateKey) + .onFailure(logException) + +} + +object PrivateKeyFile { + private val log = Logger.get("com.twitter.util.security") + /** * Attempts to convert a `PKCS8EncodedKeySpec` to a `PrivateKey`. * @@ -28,7 +50,7 @@ class PrivateKeyFile(file: File) { * keeps identical behavior to netty * https://github.com/netty/netty/blob/netty-4.1.11.Final/handler/src/main/java/io/netty/handler/ssl/SslContext.java#L1006 */ - private[this] def keySpecToPrivateKey(keySpec: PKCS8EncodedKeySpec): Try[PrivateKey] = + def keySpecToPrivateKey(keySpec: PKCS8EncodedKeySpec): Try[PrivateKey] = Try { KeyFactory.getInstance("RSA").generatePrivate(keySpec) }.handle { @@ -43,26 +65,4 @@ class PrivateKeyFile(file: File) { case ex: InvalidKeySpecException => Throw(new InvalidKeySpecException("None of RSA, DSA, EC worked", ex)) } - - /** - * Attemps to read the contents of an unencrypted PrivateKey file. - * - * @note Throws `InvalidPemFormatException` if the file is not a PEM format file. - * - * @note Throws `InvalidKeySpecException` if the file cannot be loaded as an - * RSA, DSA, or EC PrivateKey. - * - * @note This method will attempt to load the key first as an RSA key, then - * as a DSA key, and then as an EC (Elliptic Curve) key. - */ - def readPrivateKey(): Try[PrivateKey] = - new Pkcs8EncodedKeySpecFile(file) - .readPkcs8EncodedKeySpec() - .flatMap(keySpecToPrivateKey) - .onFailure(logException) - -} - -private object PrivateKeyFile { - private val log = Logger.get("com.twitter.util.security") } diff --git a/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala b/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala index 91ae0db1d5..0e13c2f824 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X509TrustManagerFactory.scala @@ -43,25 +43,27 @@ object X509TrustManagerFactory { ks.setCertificateEntry(alias, cert) } - private[this] def certsToKeyStore(certs: Seq[X509Certificate]): KeyStore = { + /** + * Load passed in X.509 certificates into a `java.security.KeyStore` + */ + def certsToKeyStore(certs: Seq[X509Certificate]): KeyStore = { val ks: KeyStore = KeyStore.getInstance("JKS") ks.load(null) certs.foreach(setCertificateEntry(ks)) ks } - private[this] def keyStoreToTmf(ks: KeyStore): TrustManagerFactory = { + /** + * Combine the passed in X.509 certificates in order to construct a + * `javax.net.ssl.TrustManagerFactory`. + */ + def certsToTrustManagerFactory(certs: Seq[X509Certificate]): TrustManagerFactory = { + val ks: KeyStore = certsToKeyStore(certs) val tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()) tmf.init(ks) tmf } - private[this] def certsToTrustManagers(certs: Seq[X509Certificate]): Array[TrustManager] = { - val ks: KeyStore = certsToKeyStore(certs) - val tmf: TrustManagerFactory = keyStoreToTmf(ks) - tmf.getTrustManagers() - } - /** * Attempts to combine the passed in X.509 certificates in order to construct a * `javax.net.ssl.TrustManager`. @@ -69,6 +71,6 @@ object X509TrustManagerFactory { * `javax.net.ssl.SSLContext`'s init method. */ def buildTrustManager(x509Certificates: Seq[X509Certificate]): Try[Array[TrustManager]] = Try { - certsToTrustManagers(x509Certificates) + certsToTrustManagerFactory(x509Certificates).getTrustManagers } } diff --git a/util-security/src/test/scala/com/twitter/util/security/PemBytesTest.scala b/util-security/src/test/scala/com/twitter/util/security/PemBytesTest.scala index b4084c45cf..4f1c94849b 100644 --- a/util-security/src/test/scala/com/twitter/util/security/PemBytesTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/PemBytesTest.scala @@ -3,10 +3,31 @@ package com.twitter.util.security import com.twitter.io.TempFile import com.twitter.util.Try import java.io.File +import java.nio.file.Files import org.scalatest.funsuite.AnyFunSuite class PemBytesTest extends AnyFunSuite { + private[this] def readPemBytesMessageFromBytes( + messageType: String + )( + tempBytes: Array[Byte] + ): Try[String] = + for { + pemBytes <- PemBytes.fromBytes(tempBytes, "pem") + item <- pemBytes.readMessage(messageType) + } yield new String(item) + + private[this] def readPemBytesMessagesFromBytes( + messageType: String + )( + tempBytes: Array[Byte] + ): Try[Seq[String]] = + for { + pemBytes <- PemBytes.fromBytes(tempBytes, "pem") + items <- pemBytes.readMessages(messageType) + } yield items.map(new String(_)) + private[this] def readPemBytesMessage(messageType: String)(tempFile: File): Try[String] = for { pemBytes <- PemBytes.fromFile(tempFile) @@ -21,6 +42,8 @@ class PemBytesTest extends AnyFunSuite { private[this] val readHello = readPemBytesMessage("hello") _ private[this] val readHellos = readPemBytesMessages("hello") _ + private[this] val readHellofromBytes = readPemBytesMessageFromBytes("hello") _ + private[this] val readHellosfromBytes = readPemBytesMessagesFromBytes("hello") _ test("no header file") { val tempFile = TempFile.fromResourcePath("/pem/test-no-head.txt") @@ -59,6 +82,16 @@ class PemBytesTest extends AnyFunSuite { assert(hello == "hello") } + test("good bytes data") { + val tempBytes = Files.readAllBytes(TempFile.fromResourcePath("/pem/test.txt").toPath) + // deleteOnExit is handled by TempFile + + val tryHello = readHellofromBytes(tempBytes) + assert(tryHello.isReturn) + val hello = tryHello.get() + assert(hello == "hello") + } + test("good file with text before and after") { val tempFile = TempFile.fromResourcePath("/pem/test-before-after.txt") // deleteOnExit is handled by TempFile @@ -88,4 +121,24 @@ class PemBytesTest extends AnyFunSuite { val messages = tryHello.get() assert(messages == Seq("hello", "goodbye")) } + + test("read single message from good encoded data with multiple messages as bytes") { + val tempBytes = Files.readAllBytes(TempFile.fromResourcePath("/pem/test-multiple.txt").toPath) + // deleteOnExit is handled by TempFile + + val tryHello = readHellofromBytes(tempBytes) + assert(tryHello.isReturn) + val hello = tryHello.get() + assert(hello == "hello") + } + + test("read multiple message from good encoded data with multiple messages") { + val tempBytes = Files.readAllBytes(TempFile.fromResourcePath("/pem/test-multiple.txt").toPath) + // deleteOnExit is handled by TempFile + + val tryHello = readHellosfromBytes(tempBytes) + assert(tryHello.isReturn) + val messages = tryHello.get() + assert(messages == Seq("hello", "goodbye")) + } } From 9968ad4022f7d55fdb8c4df28e1059d2c5c0f634 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Thu, 22 Apr 2021 16:02:37 +0000 Subject: [PATCH 470/796] util-stats: Add metadata support to metrics instances Problem We want to construct metrics expressions, but it's challenging without having access to the fully constructed `MetricBuilder` instance. We don't have access to the fully constructed instance because of the way StatsReceiver composition works. We only have partial context when we actually invoke the MetricBuilder, and the rest of the context is assembled when the metric is constructed. Solution We should provide access to the fully constructed `MetricBuilder` instance after the metric has been constructed. We also introduce a `Metadata` abstraction to represent when the metric actually represents multiple metrics behind the scenes. JIRA Issues: CSL-10887 Differential Revision: https://phabricator.twitter.biz/D653751 --- CHANGELOG.rst | 20 ++++++++++++ .../stats/CumulativeGaugeBenchmark.scala | 4 +-- .../stats/BroadcastStatsReceiver.scala | 12 +++++++ .../com/twitter/finagle/stats/Counter.scala | 1 + .../finagle/stats/CumulativeGauge.scala | 8 ++--- .../com/twitter/finagle/stats/Gauge.scala | 1 + .../finagle/stats/InMemoryStatsReceiver.scala | 6 ++++ .../finagle/stats/LazyStatsReceiver.scala | 2 ++ .../twitter/finagle/stats/MetricBuilder.scala | 31 ++++++++++++------- .../finagle/stats/NullStatsReceiver.scala | 21 ++++++++++--- .../finagle/stats/RollupStatsReceiver.scala | 3 ++ .../com/twitter/finagle/stats/Stat.scala | 1 + .../finagle/stats/CumulativeGaugeTest.scala | 12 +++---- .../finagle/stats/StatsReceiverTest.scala | 2 ++ 14 files changed, 96 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0f6e130362..35f40d30a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,26 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-stats: Added a methods `c.t.f.stats.Counter#metadata: Metadata`, + `c.t.f.stats.Stat#metadata: Metadata`, and `c.t.f.stats.Gauge#metadata: + Metadata` to make it easier to introspect the constructed metric. In + particular, this will enable constructing `Expression`s based on the full name + of the metric. If you don't have access to a concrete `Metadata` instance + (like `MetricBuilder`) for constructing a Counter, Stat, or Gauge, you can + instead supply `NoMetadata`. ``PHAB_ID=D653751`` + +New Features +~~~~~~~~~~~~ + +* util-stats: Added a `com.twitter.finagle.stats.Metadata` abstraction, that can + be either many `com.twitter.finagle.stats.Metadata`, a `MetricBuilder`, or a + `NoMetadata`, which is the null `Metadata`. This enabled constructing + metadata for counters that represent multiple counters under the hood. + ``PHAB_ID=D653751`` + 21.3.0 ------ diff --git a/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala index 3b525092d4..462532228a 100644 --- a/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala @@ -24,7 +24,7 @@ class CumulativeGaugeBenchmark extends StdBenchAnnotations { def setup(): Unit = { cg = new CGauge() gauges.clear() - 0.until(num).foreach { _ => gauges.add(cg.addGauge(1)) } + 0.until(num).foreach { _ => gauges.add(cg.addGauge(1, NoMetadata)) } } @Benchmark @@ -38,7 +38,7 @@ class CumulativeGaugeBenchmark extends StdBenchAnnotations { @OutputTimeUnit(TimeUnit.MICROSECONDS) @Fork(5) def addGauge(): Gauge = { - cg.addGauge(1) + cg.addGauge(1, NoMetadata) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala index 29c4561bf8..ca7a504eff 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala @@ -36,6 +36,7 @@ object BroadcastStatsReceiver { firstGauge.remove() secondGauge.remove() } + def metadata: Metadata = MultiMetadata(Seq(firstGauge.metadata, secondGauge.metadata)) } override protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = { @@ -61,6 +62,7 @@ object BroadcastStatsReceiver { def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { val gauges = srs.map { _.addGauge(schema)(f) } def remove(): Unit = gauges.foreach { _.remove() } + def metadata: Metadata = MultiMetadata(gauges.map(_.metadata)) } override protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = @@ -90,6 +92,7 @@ object BroadcastCounter { private object NullCounter extends Counter { def incr(delta: Long): Unit = () + def metadata: Metadata = NoMetadata } private[stats] class Two(a: Counter, b: Counter) extends Counter { @@ -97,6 +100,7 @@ object BroadcastCounter { a.incr(delta) b.incr(delta) } + def metadata: Metadata = MultiMetadata(Seq(a.metadata, b.metadata)) } private class Three(a: Counter, b: Counter, c: Counter) extends Counter { @@ -105,6 +109,7 @@ object BroadcastCounter { b.incr(delta) c.incr(delta) } + def metadata: Metadata = MultiMetadata(Seq(a.metadata, b.metadata, c.metadata)) } private class Four(a: Counter, b: Counter, c: Counter, d: Counter) extends Counter { @@ -114,10 +119,12 @@ object BroadcastCounter { c.incr(delta) d.incr(delta) } + def metadata: Metadata = MultiMetadata(Seq(a.metadata, b.metadata, c.metadata, d.metadata)) } private class N(counters: Seq[Counter]) extends Counter { def incr(delta: Long): Unit = { counters.foreach(_.incr(delta)) } + def metadata: Metadata = MultiMetadata(counters.map(_.metadata)) } } @@ -138,6 +145,7 @@ object BroadcastStat { private object NullStat extends Stat { def add(value: Float): Unit = () + def metadata: Metadata = NoMetadata } private[stats] class Two(a: Stat, b: Stat) extends Stat { @@ -145,6 +153,7 @@ object BroadcastStat { a.add(value) b.add(value) } + def metadata: Metadata = MultiMetadata(Seq(a.metadata, b.metadata)) } private class Three(a: Stat, b: Stat, c: Stat) extends Stat { @@ -153,6 +162,7 @@ object BroadcastStat { b.add(value) c.add(value) } + def metadata: Metadata = MultiMetadata(Seq(a.metadata, b.metadata, c.metadata)) } private class Four(a: Stat, b: Stat, c: Stat, d: Stat) extends Stat { @@ -162,9 +172,11 @@ object BroadcastStat { c.add(value) d.add(value) } + def metadata: Metadata = MultiMetadata(Seq(a.metadata, b.metadata, c.metadata, d.metadata)) } private class N(stats: Seq[Stat]) extends Stat { def add(value: Float): Unit = { stats.foreach(_.add(value)) } + def metadata: Metadata = MultiMetadata(stats.map(_.metadata)) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/Counter.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/Counter.scala index 1a8286f54f..c3b9034912 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/Counter.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/Counter.scala @@ -7,4 +7,5 @@ package com.twitter.finagle.stats trait Counter { def incr(delta: Long): Unit final def incr(): Unit = { incr(1) } + def metadata: Metadata } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index d60c8458c2..bacafab967 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -22,7 +22,7 @@ private[finagle] abstract class CumulativeGauge(executor: Executor) { self => // uses the default `Executor` for the cache. def this() = this(ForkJoinPool.commonPool()) - private[this] class UnderlyingGauge(val f: () => Float) extends Gauge { + private[this] class UnderlyingGauge(val f: () => Float, val metadata: Metadata) extends Gauge { def remove(): Unit = self.remove(this) } @@ -66,8 +66,8 @@ private[finagle] abstract class CumulativeGauge(executor: Executor) { self => /** * Returns a gauge unless it is dead, in which case it returns null. */ - def addGauge(f: => Float): Gauge = { - val underlyingGauge = new UnderlyingGauge(() => f) + def addGauge(f: => Float, metadata: Metadata): Gauge = { + val underlyingGauge = new UnderlyingGauge(() => f, metadata) refs.put(underlyingGauge, JBoolean.TRUE) if (register()) underlyingGauge else null @@ -172,7 +172,7 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => while (gauge == null) { val cumulativeGauge = gauges.computeIfAbsent(schema.metricBuilder.name, getWhenNotPresent(schema)) - gauge = cumulativeGauge.addGauge(f) + gauge = cumulativeGauge.addGauge(f, schema.metricBuilder) } gauge } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/Gauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/Gauge.scala index a6f8ea43ad..ed3126254d 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/Gauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/Gauge.scala @@ -6,4 +6,5 @@ package com.twitter.finagle.stats */ trait Gauge { def remove(): Unit + def metadata: Metadata } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index b4fab892c2..a144c8ff0d 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -112,6 +112,8 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { def apply(): Long = counters.getOrElse(schema.metricBuilder.name, 0) + def metadata: Metadata = schema.metricBuilder + override def toString: String = s"Counter(${schema.metricBuilder.name.mkString("/")}=${apply()})" } @@ -142,6 +144,8 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } def apply(): Seq[Float] = stats.getOrElse(schema.metricBuilder.name, Seq.empty) + def metadata: Metadata = schema.metricBuilder + override def toString: String = { val vals = apply() s"Stat(${schema.metricBuilder.name.mkString("/")}=${statValuesToStr(vals)})" @@ -165,6 +169,8 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { schemas -= schema.metricBuilder.name } + def metadata: Metadata = schema.metricBuilder + override def toString: String = { // avoid holding a reference to `f` val current = gauges.get(schema.metricBuilder.name) match { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala index 5f7e76c82c..8765bc27ec 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala @@ -22,10 +22,12 @@ final class LazyStatsReceiver(val self: StatsReceiver) extends StatsReceiverProx override def counter(schema: CounterSchema): Counter = new Counter { private[this] lazy val underlying = self.counter(schema) def incr(delta: Long): Unit = underlying.incr(delta) + def metadata: Metadata = schema.metricBuilder } override def stat(schema: HistogramSchema): Stat = new Stat { private[this] lazy val underlying = self.stat(schema) def add(value: Float): Unit = underlying.add(value) + def metadata: Metadata = schema.metricBuilder } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index f058e9581e..b00761f0c1 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -86,6 +86,12 @@ object MetricBuilder { } } +sealed trait Metadata + +case object NoMetadata extends Metadata + +case class MultiMetadata(schemas: Seq[Metadata]) extends Metadata + /** * A builder class used to configure settings and metadata for metrics prior to instantiating them. * Calling any of the three build methods (counter, gauge, or histogram) will cause the metric to be @@ -106,7 +112,8 @@ class MetricBuilder private ( val isCounterishGauge: Boolean, // see withKernel val kernel: Option[Int], - val statsReceiver: StatsReceiver) { + val statsReceiver: StatsReceiver) + extends Metadata { /** * This copy method omits statsReceiver and percentiles as arguments because they should be @@ -190,29 +197,29 @@ class MetricBuilder private ( * Used to test that builder class correctly propagates configured metadata. * @return a CounterSchema describing a counter. */ - private[MetricBuilder] def counterSchema: CounterSchema = CounterSchema(this) + private[MetricBuilder] final def counterSchema: CounterSchema = CounterSchema(this) /** * Generates a GaugeSchema which can be used to create a gauge in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. * @return a GaugeSchema describing a gauge. */ - private[MetricBuilder] def gaugeSchema: GaugeSchema = GaugeSchema(this) + private[MetricBuilder] final def gaugeSchema: GaugeSchema = GaugeSchema(this) /** * Generates a HistogramSchema which can be used to create a histogram in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. * @return a HistogramSchema describing a histogram. */ - private[MetricBuilder] def histogramSchema: HistogramSchema = HistogramSchema(this) + private[MetricBuilder] final def histogramSchema: HistogramSchema = HistogramSchema(this) /** * Produce a counter as described by the builder inside the underlying StatsReceiver. * @return the counter created. */ @varargs - def counter(name: String*): Counter = { - val schema = this.copy(name = name).counterSchema + final def counter(name: String*): Counter = { + val schema = withName(name: _*).counterSchema this.statsReceiver.counter(schema) } @@ -220,8 +227,8 @@ class MetricBuilder private ( * Produce a gauge as described by the builder inside the underlying StatsReceiver. * @return the gauge created. */ - def gauge(name: String*)(f: => Float): Gauge = { - val schema = this.copy(name = name).gaugeSchema + final def gauge(name: String*)(f: => Float): Gauge = { + val schema = withName(name: _*).gaugeSchema this.statsReceiver.addGauge(schema)(f) } @@ -231,8 +238,8 @@ class MetricBuilder private ( * @return the gauge created. */ @varargs - def gauge(f: Supplier[Float], name: String*): Gauge = { - val schema = this.copy(name = name).gaugeSchema + final def gauge(f: Supplier[Float], name: String*): Gauge = { + val schema = withName(name: _*).gaugeSchema this.statsReceiver.addGauge(schema)(f.get()) } @@ -241,8 +248,8 @@ class MetricBuilder private ( * @return the histogram created. */ @varargs - def histogram(name: String*): Stat = { - val schema = this.copy(name = name).histogramSchema + final def histogram(name: String*): Stat = { + val schema = withName(name: _*).histogramSchema this.statsReceiver.stat(schema) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala index a6dae83ae9..0f8ba9b610 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala @@ -2,6 +2,21 @@ package com.twitter.finagle.stats object NullStatsReceiver extends NullStatsReceiver { def get(): NullStatsReceiver.type = this + + val NullCounter: Counter = new Counter { + def incr(delta: Long): Unit = () + def metadata: Metadata = NoMetadata + } + + val NullStat: Stat = new Stat { + def add(value: Float): Unit = () + def metadata: Metadata = NoMetadata + } + + val NullGauge: Gauge = new Gauge { + def remove(): Unit = () + def metadata: Metadata = NoMetadata + } } /** @@ -10,11 +25,9 @@ object NullStatsReceiver extends NullStatsReceiver { * required. */ class NullStatsReceiver extends StatsReceiver { - def repr: NullStatsReceiver = this + import NullStatsReceiver._ - private[this] val NullCounter = new Counter { def incr(delta: Long): Unit = () } - private[this] val NullStat = new Stat { def add(value: Float): Unit = () } - private[this] val NullGauge = new Gauge { def remove(): Unit = () } + def repr: NullStatsReceiver = this def counter(schema: CounterSchema) = NullCounter def stat(schema: HistogramSchema) = NullStat diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala index c630a99cea..3e30dea072 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala @@ -28,6 +28,7 @@ class RollupStatsReceiver(protected val self: StatsReceiver) extends StatsReceiv self.counter(CounterSchema(schema.metricBuilder.withName(n: _*)))) ) def incr(delta: Long): Unit = allCounters.incr(delta) + def metadata: Metadata = allCounters.metadata } override def stat(schema: HistogramSchema) = new Stat { private[this] val allStats = BroadcastStat( @@ -35,6 +36,7 @@ class RollupStatsReceiver(protected val self: StatsReceiver) extends StatsReceiv self.stat(HistogramSchema(schema.metricBuilder.withName(n: _*)))) ) def add(value: Float): Unit = allStats.add(value) + def metadata: Metadata = allStats.metadata } override def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { @@ -42,5 +44,6 @@ class RollupStatsReceiver(protected val self: StatsReceiver) extends StatsReceiv tails(schema.metricBuilder.name).map(n => self.addGauge(GaugeSchema(schema.metricBuilder.withName(n: _*)))(f)) def remove(): Unit = underlying.foreach(_.remove()) + def metadata: Metadata = MultiMetadata(underlying.map(_.metadata)) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala index a8e3ef97c8..15a712115c 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala @@ -19,6 +19,7 @@ import scala.util.control.NonFatal */ trait Stat { def add(value: Float): Unit + def metadata: Metadata } /** diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala index e5d8a3f258..5037c85620 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CumulativeGaugeTest.scala @@ -28,13 +28,13 @@ class CumulativeGaugeTest extends AnyFunSuite with Eventually with IntegrationPa val gauge = new TestGauge() assert(0 == gauge.numRegisters.get) - gauge.addGauge { 0.0f } + gauge.addGauge({ 0.0f }, NoMetadata) assert(1 == gauge.numRegisters.get) } test("a CumulativeGauge with size = 1 should deregister when all gauges are removed") { val gauge = new TestGauge() - val added = gauge.addGauge { 1.0f } + val added = gauge.addGauge({ 1.0f }, NoMetadata) assert(0 == gauge.numDeregisters.get) added.remove() @@ -46,7 +46,7 @@ class CumulativeGaugeTest extends AnyFunSuite with Eventually with IntegrationPa ) { val gauge = new TestGauge() assert(0 == gauge.numDeregisters.get) - val added = gauge.addGauge { 1.0f } + val added = gauge.addGauge({ 1.0f }, NoMetadata) System.gc() gauge.cleanRefs() @@ -58,7 +58,7 @@ class CumulativeGaugeTest extends AnyFunSuite with Eventually with IntegrationPa "a CumulativeGauge with size = 1 should deregister after a System.gc when no references are held onto, after enough gets" ) { val gauge = new TestGauge() - var added = gauge.addGauge { 1.0f } + var added = gauge.addGauge({ 1.0f }, NoMetadata) assert(0 == gauge.numDeregisters.get) added = null @@ -74,14 +74,14 @@ class CumulativeGaugeTest extends AnyFunSuite with Eventually with IntegrationPa test("a CumulativeGauge should sum values across all registered gauges") { val gauge = new TestGauge() - val underlying = 0.until(100).map { _ => gauge.addGauge { 10.0f } } + val underlying = 0.until(100).map { _ => gauge.addGauge({ 10.0f }, NoMetadata) } assert(gauge.getValue == (10.0f * 100)) } test("a CumulativeGauge should discount gauges once removed") { val gauge = new TestGauge() - val underlying = Array.fill(100) { gauge.addGauge { 10.0f } } + val underlying = Array.fill(100) { gauge.addGauge({ 10.0f }, NoMetadata) } assert(gauge.getValue == (10.0f * 100)) underlying(0).remove() assert(gauge.getValue == (10.0f * 99)) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 946ba3492f..a1e7f7ccea 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -28,6 +28,7 @@ class StatsReceiverTest extends AnyFunSuite { class MemCounter extends Counter { var c: Long = 0 def incr(delta: Long): Unit = { c += delta } + def metadata: Metadata = NoMetadata } val c1 = new MemCounter val c2 = new MemCounter @@ -42,6 +43,7 @@ class StatsReceiverTest extends AnyFunSuite { class MemStat extends Stat { var values: scala.collection.Seq[Float] = ArrayBuffer.empty[Float] def add(f: Float): Unit = { values = values :+ f } + def metadata: Metadata = NoMetadata } val s1 = new MemStat val s2 = new MemStat From 9319a68358146eae433ff9105c5734831652054a Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 23 Apr 2021 17:38:59 +0000 Subject: [PATCH 471/796] util-reflect: Memoize Types#isCaseClass computation util-reflect: Memoize isCaseClass computation Problem Computing if a class is a Scala case class using the Scala TypesApi is expensive. Solution Memoize the computation to not have to repeat it for a given class. Result Much faster results for repeated calls to `isCaseClass` for a given class. Differential Revision: https://phabricator.twitter.biz/D657748 --- CHANGELOG.rst | 5 ++ build.sbt | 2 +- util-benchmark/src/main/scala/BUILD | 1 + .../com/twitter/util/TypesBenchmark.scala | 56 +++++++++++++++++++ .../com/twitter/util/reflect/Types.scala | 5 +- 5 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 util-benchmark/src/main/scala/com/twitter/util/TypesBenchmark.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 35f40d30a8..a0a2ca0829 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-reflect: Memoize `c.t.util.reflect.Types#isCaseClass` computation. ``PHAB_ID=D657748`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index d0aa09a90b..afd513c4cc 100644 --- a/build.sbt +++ b/build.sbt @@ -248,7 +248,7 @@ lazy val utilBenchmark = Project( JmhPlugin ).settings( name := "util-benchmark" - ).dependsOn(utilCore, utilHashing, utilJvm, utilStats) + ).dependsOn(utilCore, utilHashing, utilJvm, utilReflect, utilStats) lazy val utilCache = Project( id = "util-cache", diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index 5f995d2941..a7945b1056 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -7,6 +7,7 @@ scala_library( "3rdparty/jvm/org/scala-lang:scala-library", "util/util-core:scala", "util/util-hashing/src/main/scala", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", "util/util-stats/src/main/scala", ], ) diff --git a/util-benchmark/src/main/scala/com/twitter/util/TypesBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/TypesBenchmark.scala new file mode 100644 index 0000000000..515cba38f3 --- /dev/null +++ b/util-benchmark/src/main/scala/com/twitter/util/TypesBenchmark.scala @@ -0,0 +1,56 @@ +package com.twitter.util + +import com.twitter.util.reflect.Types +import org.openjdk.jmh.annotations._ +import scala.util.control.NonFatal + +private object TypesBenchmark { + class NotCaseClass { + val a: String = "A" + val b: String = "B" + } + + case class IsCaseClass(a: String, b: String) +} + +// ./sbt 'project util-benchmark' 'jmh:run TypesBenchmark' +@State(Scope.Benchmark) +class TypesBenchmark extends StdBenchAnnotations { + import TypesBenchmark._ + + @Benchmark + def notCaseClassWithClass(): Unit = { + try { + Types.notCaseClass(classOf[NotCaseClass]) + } catch { + case NonFatal(_) => // avoid throwing exceptions so the benchmark can finish + } + } + + @Benchmark + def notCaseClassWithCaseClass(): Unit = { + try { + Types.notCaseClass(classOf[IsCaseClass]) + } catch { + case NonFatal(_) => // avoid throwing exceptions so the benchmark can finish + } + } + + @Benchmark + def caseClassWithClass(): Unit = { + try { + Types.isCaseClass(classOf[NotCaseClass]) + } catch { + case NonFatal(_) => // avoid throwing exceptions so the benchmark can finish + } + } + + @Benchmark + def caseClassWithCaseClass(): Unit = { + try { + Types.isCaseClass(classOf[IsCaseClass]) + } catch { + case NonFatal(_) => // avoid throwing exceptions so the benchmark can finish + } + } +} diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala b/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala index 6efffeda93..0f0b1662b7 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/Types.scala @@ -1,5 +1,6 @@ package com.twitter.util.reflect +import com.twitter.util.Memoize import java.lang.reflect.{ParameterizedType, TypeVariable, Type => JavaType} import scala.reflect.api.TypeCreator import scala.reflect.runtime.universe._ @@ -16,10 +17,8 @@ object Types { * - not assignable from OPTION nor LIST and * - is not a Tuple and * - class symbol is case class. - * - * @param clazz runtime representation of class type. */ - def isCaseClass[T](clazz: Class[T]): Boolean = { + val isCaseClass: Class[_] => Boolean = Memoize { clazz: Class[_] => val tpe = asTypeTag(clazz).tpe val classSymbol = tpe.typeSymbol.asClass tpe <:< PRODUCT && From e54328db8bfc2f6d9619dbab1dcea3cf7951a7e8 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 26 Apr 2021 21:05:08 +0000 Subject: [PATCH 472/796] Twitter-oss: Prepare OSS libraries for release 21.4 Problem We want to release the next versions of our Twitter OSS libraries 21.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-10889 Differential Revision: https://phabricator.twitter.biz/D658986 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a0a2ca0829..04727680fe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.4.0 +------ + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 0acc5fb78f..5f2780ab7e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.3.0" +val utilCore = "com.twitter" %% "util-core" % "21.4.0" ``` # Units diff --git a/build.sbt b/build.sbt index afd513c4cc..3f8f0dda40 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.4.0-SNAPSHOT" +val releaseVersion = "21.4.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 6e121f43380ed435982822a6e8897cec4f9e2221 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 26 Apr 2021 21:52:22 +0000 Subject: [PATCH 473/796] Twitter-oss: update OSS libraries post-release to 21.5-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.5-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-10889 Differential Revision: https://phabricator.twitter.biz/D659035 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5f2780ab7e..8d16368213 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 3f8f0dda40..51e17d9428 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.4.0" +val releaseVersion = "21.5.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 5849d91c49d036300af9ebaf40511f27b18ecb94 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 27 Apr 2021 01:29:45 +0000 Subject: [PATCH 474/796] util/util-validator: The new validation Problem We'd like to have a generic utility for doing case class validations that follows the Java bean specification (https://beanvalidation.org/). Solution Introduce `util/util-validator` which is Scala library wrapping the Hibernate Validator Java library but with extended functionality to help things work in Scala. Differential Revision: https://phabricator.twitter.biz/D638603 --- CHANGELOG.rst | 5 + build.sbt | 23 +- doc/src/sphinx/index.rst | 1 + doc/src/sphinx/util-validator/index.rst | 1799 +++++++++++++++++ util-benchmark/src/main/scala/BUILD | 2 + .../twitter/util/ValidationBenchmark.scala | 79 + util-benchmark/src/test/scala/BUILD | 17 + .../util/ValidationBenchmarkTest.scala | 21 + .../java/com/twitter/util/validation/BUILD | 16 + .../util/validation/MethodValidation.java | 47 + .../twitter/util/validation/constraints/BUILD | 16 + .../validation/constraints/CountryCode.java | 52 + .../util/validation/constraints/OneOf.java | 57 + .../util/validation/constraints/UUID.java | 52 + util-validator/src/main/resources/BUILD | 6 + .../main/resources/META-INF/validation.xml | 8 + .../resources/ValidationMessages.properties | 3 + .../util/validation/constraint-mappings.xml | 32 + .../scala/com/twitter/util/validation/BUILD | 49 + .../util/validation/DescriptorFactory.scala | 429 ++++ .../util/validation/ScalaValidator.scala | 1470 ++++++++++++++ .../com/twitter/util/validation/cfg/BUILD | 17 + .../validation/cfg/ConstraintMapping.scala | 41 + .../validation/constraintvalidation/BUILD | 16 + .../TwitterConstraintValidatorContext.scala | 221 ++ .../twitter/util/validation/conversions/BUILD | 19 + .../conversions/ConstraintViolationOps.scala | 22 + .../util/validation/conversions/PathOps.scala | 20 + .../com/twitter/util/validation/engine/BUILD | 19 + .../engine/ConstraintViolationHelper.scala | 41 + .../engine/MethodValidationResult.scala | 88 + .../twitter/util/validation/executable/BUILD | 18 + .../executable/ScalaExecutableValidator.scala | 178 ++ .../internal/AnnotationFactory.scala | 38 + .../twitter/util/validation/internal/BUILD | 19 + .../util/validation/internal/Types.scala | 71 + .../internal/ValidationContext.scala | 12 + .../internal/constraintvalidation/BUILD | 15 + .../ConstraintValidatorContextFactory.scala | 26 + .../util/validation/internal/engine/BUILD | 17 + .../engine/ConstraintViolationFactory.scala | 167 ++ .../internal/metadata/descriptor/BUILD | 16 + .../ConstraintDescriptorFactory.scala | 52 + ...MethodValidationConstraintDescriptor.scala | 41 + .../util/validation/internal/validators/BUILD | 18 + ...SO3166CountryCodeConstraintValidator.scala | 63 + .../validators/OneOfConstraintValidator.scala | 57 + .../validators/UUIDConstraintValidator.scala | 17 + .../internal/validators/notempty/BUILD | 16 + .../NotEmptyValidatorForIterable.scala | 23 + .../internal/validators/package.scala | 11 + .../validation/internal/validators/size/BUILD | 16 + .../size/SizeValidatorForIterable.scala | 38 + .../twitter/util/validation/metadata/BUILD | 19 + .../metadata/CaseClassDescriptor.scala | 13 + .../metadata/ConstructorDescriptor.scala | 10 + .../util/validation/metadata/Descriptor.scala | 4 + .../metadata/ExecutableDescriptor.scala | 10 + .../metadata/MethodDescriptor.scala | 10 + .../metadata/PropertyDescriptor.scala | 11 + .../com/twitter/util/validation/package.scala | 10 + .../hibernate/validator/internal/engine/BUILD | 18 + .../engine/ValidatorFactoryInspector.scala | 41 + .../internal/properties/javabean/BUILD | 19 + .../javabean/ConstructorCallable.scala | 20 + .../javabean/ExecutableCallable.scala | 186 ++ .../properties/javabean/MethodCallable.scala | 13 + .../java/com/twitter/util/validation/BUILD | 14 + .../com/twitter/util/validation/CarMake.java | 8 + .../twitter/util/validation/DoesNothing.java | 16 + .../constraints/InvalidConstraint.java | 29 + .../constraints/ValidPassengerCount.java | 46 + .../ValidPassengerCountReturnValue.java | 46 + .../com/twitter/util/validation/tests/BUILD | 24 + .../twitter/util/validation/tests/Car.java | 52 + .../twitter/util/validation/tests/Person.java | 16 + .../tests/ScalaValidatorJavaTest.java | 155 ++ .../validation/tests/TestConstraintGroup.java | 4 + ...alidPassengerCountConstraintValidator.java | 25 + .../util/validation/AssertViolationTest.scala | 59 + .../scala/com/twitter/util/validation/BUILD | 34 + .../validation/OptionUnwrappingTest.scala | 170 ++ .../util/validation/ScalaValidatorTest.scala | 1507 ++++++++++++++ .../validation/WrongMessageInterpolator.scala | 17 + .../twitter/util/validation/caseclasses.scala | 523 +++++ .../constraints/ConsistentDateParameters.java | 30 + .../constraints/StateConstraint.java | 32 + .../constraints/StateConstraintPayload.scala | 8 + .../validation/conversions/PathOpsTest.scala | 58 + .../ScalaExecutableValidatorTest.scala | 820 ++++++++ .../ConsistentDateParametersValidator.scala | 21 + .../validators/ConstraintValidatorTest.scala | 29 + ...66CountryCodeConstraintValidatorTest.scala | 112 + .../InvalidConstraintValidator.scala | 12 + .../NotEmptyPathConstraintValidator.scala | 25 + .../OneOfConstraintValidatorTest.scala | 80 + .../validators/PackageObjectTest.scala | 69 + .../validators/StateConstraintValidator.scala | 23 + .../UUIDConstraintValidatorTest.scala | 38 + ...lidPassengerCountConstraintValidator.scala | 21 + ...rCountReturnValueConstraintValidator.scala | 22 + 101 files changed, 10145 insertions(+), 1 deletion(-) create mode 100644 doc/src/sphinx/util-validator/index.rst create mode 100644 util-benchmark/src/main/scala/com/twitter/util/ValidationBenchmark.scala create mode 100644 util-benchmark/src/test/scala/BUILD create mode 100644 util-benchmark/src/test/scala/com/twitter/util/ValidationBenchmarkTest.scala create mode 100644 util-validator/src/main/java/com/twitter/util/validation/BUILD create mode 100644 util-validator/src/main/java/com/twitter/util/validation/MethodValidation.java create mode 100644 util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD create mode 100644 util-validator/src/main/java/com/twitter/util/validation/constraints/CountryCode.java create mode 100644 util-validator/src/main/java/com/twitter/util/validation/constraints/OneOf.java create mode 100644 util-validator/src/main/java/com/twitter/util/validation/constraints/UUID.java create mode 100644 util-validator/src/main/resources/BUILD create mode 100644 util-validator/src/main/resources/META-INF/validation.xml create mode 100644 util-validator/src/main/resources/ValidationMessages.properties create mode 100644 util-validator/src/main/resources/com/twitter/util/validation/constraint-mappings.xml create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/DescriptorFactory.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/cfg/ConstraintMapping.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/TwitterConstraintValidatorContext.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/conversions/ConstraintViolationOps.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/conversions/PathOps.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/engine/ConstraintViolationHelper.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/engine/MethodValidationResult.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/executable/ScalaExecutableValidator.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/AnnotationFactory.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/Types.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/ValidationContext.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/ConstraintValidatorContextFactory.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/engine/ConstraintViolationFactory.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/ConstraintDescriptorFactory.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/MethodValidationConstraintDescriptor.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidator.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidator.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/validators/UUIDConstraintValidator.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/NotEmptyValidatorForIterable.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/validators/package.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/SizeValidatorForIterable.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/metadata/CaseClassDescriptor.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/metadata/ConstructorDescriptor.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/metadata/Descriptor.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/metadata/ExecutableDescriptor.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/metadata/MethodDescriptor.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/metadata/PropertyDescriptor.scala create mode 100644 util-validator/src/main/scala/com/twitter/util/validation/package.scala create mode 100644 util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD create mode 100644 util-validator/src/main/scala/org/hibernate/validator/internal/engine/ValidatorFactoryInspector.scala create mode 100644 util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD create mode 100644 util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/ConstructorCallable.scala create mode 100644 util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/ExecutableCallable.scala create mode 100644 util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/MethodCallable.scala create mode 100644 util-validator/src/test/java/com/twitter/util/validation/BUILD create mode 100644 util-validator/src/test/java/com/twitter/util/validation/CarMake.java create mode 100644 util-validator/src/test/java/com/twitter/util/validation/DoesNothing.java create mode 100644 util-validator/src/test/java/com/twitter/util/validation/constraints/InvalidConstraint.java create mode 100644 util-validator/src/test/java/com/twitter/util/validation/constraints/ValidPassengerCount.java create mode 100644 util-validator/src/test/java/com/twitter/util/validation/constraints/ValidPassengerCountReturnValue.java create mode 100644 util-validator/src/test/java/com/twitter/util/validation/tests/BUILD create mode 100644 util-validator/src/test/java/com/twitter/util/validation/tests/Car.java create mode 100644 util-validator/src/test/java/com/twitter/util/validation/tests/Person.java create mode 100644 util-validator/src/test/java/com/twitter/util/validation/tests/ScalaValidatorJavaTest.java create mode 100644 util-validator/src/test/java/com/twitter/util/validation/tests/TestConstraintGroup.java create mode 100644 util-validator/src/test/java/com/twitter/util/validation/tests/ValidPassengerCountConstraintValidator.java create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/AssertViolationTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/BUILD create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/OptionUnwrappingTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/ScalaValidatorTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/WrongMessageInterpolator.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/caseclasses.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/constraints/ConsistentDateParameters.java create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/constraints/StateConstraint.java create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/constraints/StateConstraintPayload.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/conversions/PathOpsTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/executable/ScalaExecutableValidatorTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ConsistentDateParametersValidator.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ConstraintValidatorTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidatorTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/InvalidConstraintValidator.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/NotEmptyPathConstraintValidator.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidatorTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/PackageObjectTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/StateConstraintValidator.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/UUIDConstraintValidatorTest.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ValidPassengerCountConstraintValidator.scala create mode 100644 util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ValidPassengerCountReturnValueConstraintValidator.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 04727680fe..6e2dd6303f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* util-validator: Introduce new library for case class validations (akin to Java bean validation) + which follows the Jakarta Bean Validation specification (https://beanvalidation.org/) by wrapping + the Hibernate Validator library and thus supports `jakarta.validation.Constraint` annotations and + validators for annotating and validating fields of Scala case classes. ``PHAB_ID=D638603`` + 21.4.0 ------ diff --git a/build.sbt b/build.sbt index 51e17d9428..0d3f198ac1 100644 --- a/build.sbt +++ b/build.sbt @@ -213,6 +213,7 @@ lazy val util = Project( utilTest, utilThrift, utilTunable, + utilValidator, utilZk, utilZkTest ) @@ -248,7 +249,7 @@ lazy val utilBenchmark = Project( JmhPlugin ).settings( name := "util-benchmark" - ).dependsOn(utilCore, utilHashing, utilJvm, utilReflect, utilStats) + ).dependsOn(utilCore, utilHashing, utilJvm, utilReflect, utilStats, utilValidator) lazy val utilCache = Project( id = "util-cache", @@ -557,6 +558,26 @@ lazy val utilTunable = Project( ) ).dependsOn(utilApp, utilCore) +lazy val utilValidator = Project( + id = "util-validator", + base = file("util-validator") +).settings( + sharedSettings + ).settings( + name := "util-validator", + libraryDependencies ++= Seq( + caffeineLib, + scalacheckLib, + "jakarta.validation" % "jakarta.validation-api" % "3.0.0", + "org.hibernate.validator" % "hibernate-validator" % "7.0.1.Final", + "org.glassfish" % "jakarta.el" % "4.0.0", + "org.json4s" %% "json4s-core" % "3.6.7", + "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion % "test", + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", + "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" + ) + ).dependsOn(utilCore, utilReflect, utilSlf4jApi) + lazy val utilZk = Project( id = "util-zk", base = file("util-zk") diff --git a/doc/src/sphinx/index.rst b/doc/src/sphinx/index.rst index 713d567d5f..6d24186b36 100644 --- a/doc/src/sphinx/index.rst +++ b/doc/src/sphinx/index.rst @@ -13,6 +13,7 @@ Guides util-stats/index util-cookbook/index + util-validator/index Changelog diff --git a/doc/src/sphinx/util-validator/index.rst b/doc/src/sphinx/util-validator/index.rst new file mode 100644 index 0000000000..e58ddbeb2c --- /dev/null +++ b/doc/src/sphinx/util-validator/index.rst @@ -0,0 +1,1799 @@ +.. _util-validator-index: + +util-validator Guide +==================== + +Preface +------- + +Validating data is a common task that occurs throughout all application layers, from the +presentation to the persistence layer. Often the same validation logic is implemented in each +layer which is time consuming and error-prone. To avoid duplication of these validations, +developers often bundle validation logic directly into the domain model, cluttering domain classes +with validation code which is really metadata about the class itself. + +The `Jakarta Bean Validation 2.0 `__ - defines a metadata model and API +for entity and method validation. The default metadata source are Java annotations (with the ability +to override and extend the meta-data through the use of XML if desired). The API is not tied to a +specific application tier nor programming model and it is specifically not tied to either the web or +persistence tier, and is available for both server-side application programming or rich client +applications. + +The `ScalaValidator` is a Scala wrapper around the Java `Hibernate Validator `__ +which itself is the reference implementation of `Jakarta Bean Validation specification `__. + +Getting Started +--------------- + +The `ScalaValidator` depends only on the `Hibernate Validator `__, +the `Glassfish reference implementation `__ of the +`Jakarta Expression Language `__ for message interpolation, +and `json4s-core `__ for Scala reflection. + +We highly recommend reading through the Hibernate Validator `documentation `__ +as our guide is largely based on this information. + +In order to use the `util-validator` add a dependency on the project, e.g. in `sbt `__: + +.. code:: scala + + "com.twitter" %% "util-validator" % utilValidatorVersion + +or with `Maven `__: + +.. code:: xml + + + com.twitter + util-validator_2.12 + ${util.validator.version} + + +where `utilValidatorVersion` and `util.validator.version` are the expected library version. + +Applying constraints +~~~~~~~~~~~~~~~~~~~~ + +Let's start with a basic example to see how to apply constraints. Let's assume you have a +case class defined as such: + +.. code:: scala + + package com.example + + import jakarta.validation.constraints.{Min, NotEmpty, Size} + + case class Car( + @NotEmpty manufacturer: String, + @NotEmpty @Size(min = 2, max = 14) licensePlate: String, + @Min(2) seatCount: Int) + +The `@NotEmpty`, `@Size` and `@Min` annotations are used to declare the constraints which should be +applied to the fields of a `Car` instance: + +* `manufacturer` must never be an empty String (and thus also not `null`) +* `licensePlate` must never be an empty String (thus not also not `null`) and must be between 2 and 14 characters +* `seatCount` must be equal to at least 2 + +Validating constraints +~~~~~~~~~~~~~~~~~~~~~~ + +`ScalaValidator#validate` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +To perform a validation of these constraints, you use an instance of the `ScalaValidator`. Let’s +take a look at a test for validating `Car` instances: + +.. code:: scala + :emphasize-lines: 15,25,35,45 + + package com.example + + import com.twitter.util.validation.ScalaValidator + import jakarta.validation.ConstraintViolation + import org.scalatest.funsuite.AnyFunSuite + import org.scalatest.matchers.should.Matchers + + class CarTest extends AnyFunSuite with Matchers { + + private[this] val validator: ScalaValidator = ScalaValidator() + + test("manufacturer is empty") { + val car: Car = Car("", "DD-AB-123", 4) + + val violations: Set[ConstraintViolation[Car]] = validator.validate(car) + violations.size should equal(1) + val violation = violations.head + violation.getPropertyPath.toString should equal("manufacturer") + violation.getMessage should be("must not be empty") + } + + test("licensePlate is too short") { + val car: Car = Car("Greenwich", "D", 4) + + val violations: Set[ConstraintViolation[Car]] = validator.validate(car) + violations.size should equal(1) + val violation = violations.head + violation.getPropertyPath.toString should equal("licensePlate") + violation.getMessage should be("size must be between 2 and 14") + } + + test("seatCount is too small") { + val car: Car = Car("Greenwich", "DD-AB-123", 1) + + val violations: Set[ConstraintViolation[Car]] = validator.validate(car) + violations.size should equal(1) + val violation = violations.head + violation.getPropertyPath.toString should equal("seatCount") + violation.getMessage should be("must be greater than or equal to 2") + } + + test("car is valid") { + val car: Car = Car("Greenwich", "DD-AB-123", 2) + + val violations: Set[ConstraintViolation[Car]] = validator.validate(car) + violations.isEmpty should be(true) + } + } + +In the test constructor we instantiate an instance of a `ScalaValidator`. A `ScalaValidator` instance +is thread-safe and may be reused multiple times. It thus can safely be stored in a member variable +field and be used in the test cases to validate the different `Car` instances. + +The `ScalaValidator#validate` method returns a set of `ConstraintViolation` instances, which you can +iterate over in order to see which validation errors occurred. The first three tests show some +expected constraint violations: + +* The `@NotEmpty` constraint on `manufacturer` is violated in `test("manufacturer is empty")` +* The `@Size` constraint on `licensePlate` is violated in `test("licensePlate is too short")` +* The `@Min` constraint on `seatCount` is violated in `test("seatCount is too small")` + +If the object validates successfully, `ScalaValidator#validate` returns an empty set as you can see +in `test("car is valid")`. + +Note that this method is recursive and will cascade validations as explained in the section on +`Object graphs <#object-graphs>`__. + +`ScalaValidator#verify` +^^^^^^^^^^^^^^^^^^^^^^^ + +The `ScalaValidator` also has an API to throw constraint violations as a type of `ValidationException` +rather than returning a `Set[ConstraintViolation[_]`. + +Instead of calling `ScalaValidator#validate`, use `ScalaValidator#verify`: + +.. code:: scala + :emphasize-lines: 16,28,40,52 + + package com.example + + import com.twitter.util.validation.ScalaValidator + import jakarta.validation.{ConstraintViolation, ConstraintViolationException} + import org.scalatest.funsuite.AnyFunSuite + import org.scalatest.matchers.should.Matchers + + class CarTest extends AnyFunSuite with Matchers { + + private[this] val validator: ScalaValidator = ScalaValidator() + + test("manufacturer is empty") { + val car: Car = Car("", "DD-AB-123", 4) + + val e = intercept[ConstraintViolationException] { + validator.verify(car) + } + e.getConstraintViolations.size should equal(1) + val violation = e.getConstraintViolations.iterator.next + violation.getPropertyPath.toString should equal("manufacturer") + violation.getMessage should be("must not be empty") + } + + test("licensePlate is too short") { + val car: Car = Car("Greenwich", "D", 4) + + val e = intercept[ConstraintViolationException] { + validator.verify(car) + } + e.getConstraintViolations.size should equal(1) + val violation = e.getConstraintViolations.iterator.next + violation.getPropertyPath.toString should equal("licensePlate") + violation.getMessage should be("size must be between 2 and 14") + } + + test("seatCount is too small") { + val car: Car = Car("Greenwich", "DD-AB-123", 1) + + val e = intercept[ViolationException] { + validator.verify(car) + } + e.isInstanceOf[ConstraintViolationException] should be(true) + e.asInstanceOf[ConstraintViolationException].getConstraintViolations.size should equal(1) + val violation = e.asInstanceOf[ConstraintViolationException].getConstraintViolations.iterator.next + violation.getPropertyPath.toString should equal("seatCount") + violation.getMessage should be("must be greater than or equal to 2") + } + + test("car is valid") { + val car: Car = Car("Greenwich", "DD-AB-123", 2) + + validator.verify(car) + } + } + +Like `ScalaValidator#validate`, this method is recursive and will cascade validations as explained +in the section on `Object graphs <#object-graphs>`__. + +Declaring and validating case class constraints +----------------------------------------------- + +In this section we show how to declare (see `“Declaring case class constraints” <#declaring-case-class-constraints>`__) +and validate (see `“Validating bean constraints” <#validating-case-class-constraints>`__) case class +constraints. + +You will likely want to review Hibernate's `“Built-in constraints” `__ +documentation which provides an overview of all built-in constraints coming with the Hibernate +Validator and thus supported here. + +If you are interested in applying constraints to method parameters and return values, refer to +`Declaring and validating method constraints <#declaring-and-validating-method-constraints>`__. + +Declaring case class constraints +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Constraints in Jakarta Bean Validation are expressed via Java annotations. In this section we show +how to enhance an object model with these annotations. There are three types of bean constraints: + +* field constraints +* property constraints +* container element constraints +* class constraints + +When defining a class field in Scala the compiler creates up to four accessors for it: a getter, +a setter, and if the field is annotated with `@BeanProperty`, a bean getter and a bean setter +[`reference `__]. + +Thus, if you had the following class definition: + +.. code:: scala + + class C(@myAnnot var c: Int) + +There are **six** entities which can carry the `@myAnnot` annotation: the constructor param, the +generated field and the four accessors. **By default, annotations on constructor parameters end up +only on the constructor parameter and not on any other entity**. Annotations on fields, by default, +only end up on the field. + +If you wanted Scala to copy the annotation to the generated field, you would need to apply the +`scala.annotation.meta.field` meta annotation to the annotation. + +For example, using `class C` again from above: + +.. code:: scala + + class C(@(myAnnot @field) var c: Int) + +Thus to use the Hibernate Validator Java library directly from Scala, you would need to always +annotate constraint annotations with a `scala.annotation.meta` annotation along with translating +between Scala and Java collection types. + +The benefit of `util-validator` is that the library will work with simply annotated case class +constructor parameters (i.e., without needing to add a `scala.annotation.meta` annotation) with an +API that uses and expresses Scala collection types. + +Constructor param constraints +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Constraints can be expressed by annotating the constructor param of a case class. E.g., + +.. code:: scala + + package com.example + + import jakarta.validation.constraints.{Min, NotEmpty, Size} + + case class Car( + @NotEmpty manufacturer: String, + @NotEmpty @Size(min = 2, max = 14) licensePlate: String, + @Min(2) seatCount: Int) + +.. important:: + + When using constructor parameter constraints, field access strategy is used to access the value + to be validated. This means the validation logic directly accesses the instance variable and + will never invoke the generated property accessor method. + +Constraints can be applied to fields of any access type (public, private etc.), including inherited +members. Neither constraints on companion object fields, nor secondary companion object `#apply` +method constraints are supported. + +Secondary constructors +++++++++++++++++++++++ + +The validation library is not used for case class construction but because of how annotations are +carried by default in Scala for classes, the constructor of a case class plays an important role +in case class validation. By default, the library only tracks constraint annotations for constructor +parameters which are also declared fields within the class. Thus, if you have a primary constructor +of `Int` types with a secondary constructor of constraint-annotated `String` types (which does some +translation into the required `Int` types), the secondary constructor constraints have no bearing on +validation. + +Container element constraints +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note that due to a long-standing issue [`SCALA-9883 `__] +which affects how to access Java annotations from the Scala compiler, that **container element** +**constraints are not supported**. + +For examples of container element constraints, see the Hibernate `documentation `__. + +Thus, take note that constraint annotations will apply only to the container and not the type(s) of +the container with the one caveat being a field of type `Option[_]` which is always unwrapped by +default and thus the annotation always pertains to the contained type. + +With `Option[_]` +++++++++++++++++ + +When applying a constraint on the type argument of `Option[_]`, the `ScalaValidator` will +*automatically unwrap* the `Option[_]` type and validate the internal value. For example: + +.. code-block:: bash + :emphasize-lines: 21,22 + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> import jakarta.validation.constraints._ + import jakarta.validation.constraints._ + + scala> case class Car(@Min(1000) towingCapacity: Option[Int] = None) + defined class Car + + scala> val car = Car(Some(100)) + car: Car = Car(Some(100)) + + scala> import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.ScalaValidator + + scala> val validator = ScalaValidator() + Mar 30, 2021 11:58:47 AM org.hibernate.validator.internal.util.Version + INFO: HV000001: Hibernate Validator 7.0.1.Final + validator: com.twitter.util.validation.ScalaValidator = com.twitter.util.validation.ScalaValidator@574f9e36 + + scala> val violations = validator.validate(car) + violations: Set[jakarta.validation.ConstraintViolation[Car]] = Set(ConstraintViolationImpl{interpolatedMessage='must be greater than or equal to 1000', propertyPath=towingCapacity, rootBeanClass=class Car, messageTemplate='{jakarta.validation.constraints.Min.message}'}) + + scala> val violation = violations.head + violation: jakarta.validation.ConstraintViolation[Car] = ConstraintViolationImpl{interpolatedMessage='must be greater than or equal to 1000', propertyPath=towingCapacity, rootBeanClass=class Car, messageTemplate='{jakarta.validation.constraints.Min.message}'} + + scala> violation.getMessage + res0: String = must be greater than or equal to 1000 + + scala> violation.getPropertyPath.toString + res1: String = towingCapacity + + scala> + +.. note:: + + The property path only contains the name of the property, in this way, `Options` are treated + as a "transparent" container. + +Class-level constraints +^^^^^^^^^^^^^^^^^^^^^^^ + +A constraint can also be placed on the class level. In this case not a single property is subject +of the validation but the complete object. Class-level constraints are useful if the validation +depends on a correlation between several properties of an object. + +For instance, if we had a `Car` case class defined with two field `seatCount` and `passengers` and +we want to ensure that the list of `passengers` does not have more entries than available seats. +For that purpose the `@ValidPassengerCount` constraint is added on the class level. The validator of +that constraint has access to the complete `Car` object, allowing to compare the numbers of seats +and passengers. + +.. code:: scala + :emphasize-lines: 5 + + package com.example + + import jakarta.validation.constraints.Min + + @ValidPassengerCount + case class Car( + @Min(2) seatCount: Int, + passengers: Seq[Person]) + +See the corresponding section in `Creating custom constraints - Class-level constraints <#>`__ for +documentation on how to implement such a custom constraint. + +Constraint inheritance +^^^^^^^^^^^^^^^^^^^^^^ + +When a class implements a trait or extends an abstract class, all constraint annotations declared +on the super-type apply in the same manner as the constraints specified in the class itself. +For example, if we had a trait, `Car` with a `RentalCar` implementation: + +.. code:: scala + + trait Car { + @NotEmpty def manufacturer: String + } + + case class RentalCar(manufacturer: String, @NotEmpty rentalStation: String) extends Car + +Here the case class `RentalCar` implements the `Car` trait and adds the property `rentalStation`. If +an instance of `RentalCar` is validated, not only the `@NotEmpty` constraint on `rentalStation` is +evaluated, but also the constraint on `manufacturer` from the `Car` trait. + +.. code-block:: bash + :emphasize-lines: 26,27,41,42,56,57,70,71 + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> import jakarta.validation.constraints._ + import jakarta.validation.constraints._ + + scala> trait Car { + | @NotEmpty def manufacturer: String + | } + defined trait Car + + scala> case class RentalCar(manufacturer: String, @NotEmpty rentalStation: String) extends Car + defined class RentalCar + + scala> import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.ScalaValidator + + scala> val validator = ScalaValidator() + Mar 30, 2021 12:21:41 PM org.hibernate.validator.internal.util.Version + INFO: HV000001: Hibernate Validator 7.0.1.Final + validator: com.twitter.util.validation.ScalaValidator = com.twitter.util.validation.ScalaValidator@64e89bb2 + + scala> val rental = RentalCar("", "Hertz") + rental: RentalCar = RentalCar(,Hertz) + + scala> val violations = validator.validate(rental) + violations: Set[jakarta.validation.ConstraintViolation[Car]] = Set(ConstraintViolationImpl{interpolatedMessage='must not be empty', propertyPath=manufacturer, rootBeanClass=class RentalCar, messageTemplate='{jakarta.validation.constraints.NotEmpty.message}'}) + + scala> violations.size + res0: Int = 1 + + scala> violations.head.getPropertyPath.toString + res: String = manufacturer + + scala> violations.head.getMessage + res: String = must not be empty + + scala> val rental = RentalCar("Renault", "") + rental: RentalCar = RentalCar(Renault,) + + scala> val violations = validator.validate(rental) + violations: Set[jakarta.validation.ConstraintViolation[Car]] = Set(ConstraintViolationImpl{interpolatedMessage='must not be empty', propertyPath=rentalStation, rootBeanClass=class RentalCar, messageTemplate='{jakarta.validation.constraints.NotEmpty.message}'}) + + scala> violations.size + res: Int = 1 + + scala> violations.head.getPropertyPath.toString + res: String = rentalStation + + scala> violations.head.getMessage + res: String = must not be empty + + scala> val rental = RentalCar("", "") + rental: RentalCar = RentalCar(,) + + scala> val violations = validator.validate(rental) + violations: Set[jakarta.validation.ConstraintViolation[Car]] = Set(ConstraintViolationImpl{interpolatedMessage='must not be empty', propertyPath=manufacturer, rootBeanClass=class RentalCar, messageTemplate='{jakarta.validation.constraints.NotEmpty.message}'}, ConstraintViolationImpl{interpolatedMessage='must not be empty', propertyPath=rentalStation, rootBeanClass=class RentalCar, messageTemplate='{jakarta.validation.constraints.NotEmpty.message}'}) + + scala> violations.size + res: Int = 2 + + scala> violations.map(v => s"${v.getPropertyPath}: ${v.getMessage}").mkString("\n") + res: String = + manufacturer: must not be empty + rentalStation: must not be empty + + scala> val rental = RentalCar("Renault", "Hertz") + rental: RentalCar = RentalCar(Renault,Hertz) + + scala> val violations = validator.validate(rental) + violations: Set[jakarta.validation.ConstraintViolation[Car]] = Set() + + scala> + +The same would be true, if `Car` was not a trait but an abstract class implemented by `RentalCar`. + +**Constraint annotations are aggregated if members are overridden**. So if the `RentalCar` case class +also specifies any constraints on the `manufacturer` field, it would be evaluated in addition to the +`@NotEmpty` constraint from the `Car` trait: + +.. code:: bash + :emphasize-lines: 7,8,22,23 + + scala> case class RentalCar(@Size(min = 2, max = 14) manufacturer: String, @NotEmpty rentalStation: String) extends Car + defined class RentalCar + + scala> val rental = RentalCar("A", "Hertz") + rental: RentalCar = RentalCar(A,Hertz) + + scala> val violations = validator.validate(rental) + violations: Set[jakarta.validation.ConstraintViolation[Car]] = Set(ConstraintViolationImpl{interpolatedMessage='size must be between 2 and 14', propertyPath=manufacturer, rootBeanClass=class RentalCar, messageTemplate='{jakarta.validation.constraints.Size.message}'}) + + scala> violations.size + res: Int = 1 + + scala> violations.head.getPropertyPath.toString + res: String = manufacturer + + scala> violations.head.getMessage + res: String = size must be between 2 and 14 + + scala> val rental = RentalCar("", "Hertz") + rental: RentalCar = RentalCar(,Hertz) + + scala> val violations = validator.validate(rental) + violations: Set[jakarta.validation.ConstraintViolation[Car]] = Set(ConstraintViolationImpl{interpolatedMessage='size must be between 2 and 14', propertyPath=manufacturer, rootBeanClass=class RentalCar, messageTemplate='{jakarta.validation.constraints.Size.message}'}, ConstraintViolationImpl{interpolatedMessage='must not be empty', propertyPath=manufacturer, rootBeanClass=class RentalCar, messageTemplate='{jakarta.validation.constraints.NotEmpty.message}'}) + + scala> violations.size + res: Int = 2 + + scala> violations.map(v => s"${v.getPropertyPath}: ${v.getMessage}").mkString("\n") + res: String = + manufacturer: size must be between 2 and 14 + manufacturer: must not be empty + + scala> + +Object graphs +^^^^^^^^^^^^^ + +The Jakarta Bean Validation API does not only allow to validate single case class instances but also +complete object graphs via cascaded validation. To do so, just annotate a field or property +representing a reference to another case class with `@Valid`. + +.. code:: scala + + case class Person(@NotEmpty name: String) + + case class Car(@NotEmpty manufacturer: String, @Valid driver: Person) + +If an instance of `Car` is validated, the referenced `Person` case class will also be validated, as +the `driver` field is annotated with `@Valid`. Therefore the validation of a `Car` will fail if the +`name` field of the referenced Person instance is empty. + +The validation of object graphs is recursive, i.e. if a reference marked for cascaded validation +points to an object which itself has properties annotated with `@Valid`, these references will be +followed up by the validation logic as well. The validation logic will ensure that no infinite +loops occur during cascaded validation, for example if two objects hold references to each other. + +Note that `null` or `None` values are ignored during cascaded validation. + +`Iterable[_]` ++++++++++++++ + +Unlike constraints, object graph validation works for some container elements (those which are +an `Iterable[_]`). That means the field of the container can be annotated with `@Valid`, which will +cause each contained element to be validated when the parent object is validated. + +.. code:: scala + + case class Person(@NotEmpty name: String) + + case class Car(@NotEmpty manufacturer: String, @Valid drivers: Seq[Person]) + +If an instance of `Car` is validated, the referenced sequence of `Person` case classes will also +be validated, as the `drivers` field is annotated with `@Valid`. Therefore the validation of a `Car` +will fail if the `name` field of the referenced Person instance is empty. + +.. code-block:: bash + :emphasize-lines: 27,28,39,40 + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> import jakarta.validation.constraints._ + import jakarta.validation.constraints._ + + scala> case class Person(@NotEmpty name: String) + defined class Person + + scala> import jakarta.validation.Valid + import jakarta.validation.Valid + + scala> case class Car(@NotEmpty manufacturer: String, @Valid drivers: Seq[Person]) + defined class Car + + scala> val car = Car("Renault", Seq(Person(""))) + car: Car = Car(Renault,List(Person())) + + scala> import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.ScalaValidator + + scala> val validator = ScalaValidator() + Mar 30, 2021 3:59:53 PM org.hibernate.validator.internal.util.Version + INFO: HV000001: Hibernate Validator 7.0.1.Final + validator: com.twitter.util.validation.ScalaValidator = com.twitter.util.validation.ScalaValidator@6dcc7696 + + scala> val violations = validator.validate(car) + violations: Set[jakarta.validation.ConstraintViolation[Car]] = Set(ConstraintViolationImpl{interpolatedMessage='must not be empty', propertyPath=drivers[0].name, rootBeanClass=class Car, messageTemplate='{jakarta.validation.constraints.NotEmpty.message}'}) + + scala> violations.head.getPropertyPath.toString + res: String = drivers[0].name + + scala> violations.head.getMessage + res: String = must not be empty + + scala> val car = Car("Renault", Seq(Person("Lupin"), Person(""))) + car: Car = Car(Renault,List(Person(Lupin), Person())) + + scala> val violations = validator.validate(car) + violations: Set[jakarta.validation.ConstraintViolation[Car]] = Set(ConstraintViolationImpl{interpolatedMessage='must not be empty', propertyPath=drivers[1].name, rootBeanClass=class Car, messageTemplate='{jakarta.validation.constraints.NotEmpty.message}'}) + + scala> violations.head.getPropertyPath.toString + res: String = drivers[1].name + + scala> violations.head.getMessage + res: String = must not be empty + + scala> + +Validating case class constraints +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The `ScalaValidator` does not directly implement the Jakarta Bean Validation interface but provides +the same methods (and a few more) with Scala collection types. Note, however, since the +`ScalaValidator` is a wrapper that the underlying Jakarta Bean Validation is available for direct +access for Java users or to use functionality not exposed in the `ScalaValidator`. Also note that +this is a Java library and bypassing the `ScalaValidator` methods will thus not support all Scala +types. + +This section shows how to obtain a configured `ScalaValidator` instance and then will walk through +the different methods of the `ScalaValidator`. + +Obtaining a Validator instance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The `ScalaValidator` has an `apply()` function which will return an instance of a `ScalaValidator` +configured with defaults. + +.. code:: scala + + val validator: ScalaValidator = ScalaValidator() + +To apply custom configuration to create a `ScalaValidator`, there is a builder for constructing a +customized validator. + +E.g., to configure a custom size for the descriptor cache: + +.. code:: scala + :emphasize-lines: 3 + + val validator: ScalaValidator = + ScalaValidator.builder + .withDescriptorCacheSize(1024) + .validator + +Or to add custom validations: + +.. code:: scala + :emphasize-lines: 10 + + import com.twitter.util.validation.cfg.ConstraintMapping + + val mapping = ConstraintMapping( + annotationType = classOf[FooConstraintAnnotation], + constraintValidator = classOf[FooConstraintValidator] + ) + + val validator: ScalaValidator = + ScalaValidator.builder + .withConstraintMapping(mapping) + .validator + +or to specify multiple constraint mappings: + +.. code:: scala + :emphasize-lines: 17 + + import com.twitter.util.validation.cfg.ConstraintMapping + + val mappings: Set[ConstraintMapping] = + Set( + ConstraintMapping( + annotationType = classOf[FooConstraintAnnotation], + constraintValidator = classOf[FooConstraintValidator] + ), + ConstraintMapping( + annotationType = classOf[BarConstraintAnnotation], + constraintValidator = classOf[BarConstraintValidator] + ) + ) + + val validator: ScalaValidator = + ScalaValidator.builder + .withConstraintMappings(mappings) + .validator + +You are **not required** to add a constraint mapping if the constraint annotation specifies a +`validatedBy()` class. + +.. important:: + + Attempting to add multiple mappings for the same annotation will result in a `ValidationException `__. + +ScalaValidator API +^^^^^^^^^^^^^^^^^^ + +The `ScalaValidator` contains methods that can be used to either validate entire entities or just +single properties of the entity. All methods but `ScalaValidator#verify` return a +`Set[ConstraintViolation[_]]`. The set is empty if the validation succeeds. Otherwise, a +`ConstraintViolation` instance is added for each violated constraint. In the case of +`ScalaValidator#verify`, when the returned `Set[ConstraintViolation[_]]` is non-empty a +`ConstraintViolationException `__ +is thrown which wraps the `Set[ConstraintViolation[_]]`. + +All the validation methods have a version which takes a `groups: Seq[Class[_]]` parameter that is +used to specify validation groups to be considered when performing the validation. If the parameter +is empty, the default validation group (`jakarta.validation.groups.Default`) is used. + +The topic of validation groups is discussed in detail in Hibernate's `Chapter 5, Grouping constraints `__. + +`ScalaValidator#validate` ++++++++++++++++++++++++++ + +Use `validate` to perform validation of all constraints of a given case class. + +.. code:: scala + :emphasize-lines: 8 + + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{AssertTure, NotEmpty} + + case class Car(@NotEmpty manufacturer: String, @AssertTrue isRegistered: Boolean) + + val car = Car("", false) + + val violations: Set[ConstraintViolation[Car]] = validator.validate(car) + assert(violations.size == 2) + assert(violations.head.getMessage == "must not be empty") + assert(violations.last.getMessage == "not true") + +Using the given definition of a `Car`, the above example shows an instance which fails to satisfy +the `@NotEmpty` constraint on the `manufacturer` property. The validation call therefore returns +one `ConstraintViolation[Car]` object. + +`ScalaValidator#verify` ++++++++++++++++++++++++ + +The `verify` method is the same as `validate` but instead of returning `Set[ConstraintViolation[T]]` +the method throws a `ConstraintViolationException `__. + +.. code:: scala + :emphasize-lines: 9 + + import com.twitter.util.Try + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{AssertTure, NotEmpty} + + case class Car(@NotEmpty manufacturer: String, @AssertTrue isRegistered: Boolean) + + val car = Car("", false) + + val result = Try(validator.verify(car)) + assert(result.isThrow) + +`ScalaValidator#validateValue` +++++++++++++++++++++++++++++++ + +By using the `validateValue` method you can check whether a single field of a given case class can +be validated successfully if the field had the given value: + +.. code:: scala + :emphasize-lines: 7,8,9,10 + + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{AssertTure, NotEmpty} + + case class Car(@NotEmpty manufacturer: String, @AssertTrue isRegistered: Boolean) + + val violations: Set[ConstraintViolation[Car]] = + validator.validateValue( + classOf[Car], + "manufacturer", + "") + assert(violations.size == 1) + assert(violations.head.getMessage == "must not be empty") + +There is also a version of the method which instead of accepting a `Class[T]`, takes a +`CaseClassDescriptor[T]` of a given class. + +.. code:: scala + :emphasize-lines: 6,9,10,11,12 + + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{AssertTure, NotEmpty} + + case class Car(@NotEmpty manufacturer: String, @AssertTrue isRegistered: Boolean) + + val carDescriptor: CaseClassDescriptor[Car] = validator.getConstraintsForClass(classOf[Car]) + + val violations: Set[ConstraintViolation[Car]] = + validator.validateValue( + carDescriptor, + "manufacturer", + "") + assert(violations.size == 1) + assert(violations.head.getMessage == "must not be empty") + +`ScalaValidator#validateProperty` ++++++++++++++++++++++++++++++++++ + +With help of the `validateProperty` you can validate a single named field of a given case class +instance. + +.. code :: scala + :emphasize-lines: 9,10,11 + + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{AssertTure, NotEmpty} + + case class Car(@NotEmpty manufacturer: String, @AssertTrue isRegistered: Boolean) + + val car = Car("", false) // isRegistered is false here which would normally fail `@AssertTrue` validation + + val violations: Set[ConstraintViolation[Car]] = + validator.validateProperty( + car, + "manufacturer") + assert(violations.size == 1) + assert(violations.head.getMessage == "must not be empty") + +.. note:: + + The difference between `validateValue` and `validateProperty` is that the former takes a class + type, a field in the class, and a presumed value in order to perform validation. Thus, no actual + instance of the class is required for performing validation. + + Whereas the latter performs validation of a given field on an instance of a case class. + +.. important:: + + The `@Valid` annotation for cascading validation is not honored by `validateValue` or + `validateProperty`. + +ConstraintViolation +^^^^^^^^^^^^^^^^^^^ + +As mentioned, all methods but `ScalaValidator#verify` return a `Set[ConstraintViolation[_]]`. The +`ConstraintViolation `__ +contains information about the cause and location of the validation failure. + +See the `Hibernate documentation `__ for details. + +Built-in constraints +^^^^^^^^^^^^^^^^^^^^ + +In addition to the constraints defined by the `Jakarta Bean Validation API and the Hibernate Built-in constraints `__ +we provide several useful custom constraints which are listed below. These constraints all apply to +the field/property level. + +* `@CountryCode` + + Checks that the annotated field represents a valid `ISO 3166 `__ country code or a collection of `ISO 3166 `__ country codes. + + - **Supported data types** + + `String`, `Array[String]`, `Iterable[String]` + +* `@OneOf(value=)` + + Checks that the annotated field represents a value or a set of values from the given array of values. Note that the `#toString` representations are checked for equality. + + - **Supported data types** + + `Any`, `Array[Any]`, `Iterable[Any]` + +* `@UUID` + + Checks if the annotated field is a valid `String` representation of a `java.util.UUID `__. + + - **Supported data types** + + `String` + +Declaring and validating method constraints +------------------------------------------- + +Constraints can not only be applied to case classes and their fields, but also to the parameters and +return values of methods within the case class. Following the Jakarta Bean Validation specification, +constraints can be used to specify + +* the preconditions that must be satisfied by the caller before a method or constructor may be invoked (by applying constraints to the parameters of an executable) +* the postconditions that are guaranteed to the caller after a method or constructor invocation returns (by applying constraints to the return value of an executable) + +.. important:: + + In all cases **except when using** `@MethodValidation`, declaring method or constructor + constraints itself does not automatically cause their validation upon validation of a case class + instance. Instead, the `ScalaExecutableValidator` API must be used to perform the validation. + +Declaring method constraints +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`@MethodValidation` constraint +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The `@MethodValidation` constraint annotation is a special method validation constraint provided by +the library. + +Unlike other executable specific constraints, `@MethodValidation`-annotated methods **will be** +**validated along with all field-level and class-level validations** when calling +`ScalaValidate#validate <#id1>`__ or `ScalaValidator#verify <#id2>`__. Note, that the method +validation is strictly performed *after* all field level validations (and *before* class-level +validations). Annotated methods MUST have no specified parameters and MUST return a +`MethodValidationResult`. + +To manually validate a given method or methods, you can use `ScalaExecutableValidator#validateMethod <#scalaexecutablevalidator-validatemethod>`__ +or `ScalaExecutableValidator#validateMethods <#scalaexecutablevalidator-validatemethods>`__. + +You can find details in `“Creating custom constraints - @MethodValidation constraints” <#methodvalidation-constraints>`__ +which shows how to implement a `@MethodValidation` constraint. + +Method Parameter constraints +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can specify the preconditions of a method by adding constraint annotations to its parameters: + +.. code:: scala + + import jakarta.validation.constraints.{Min, NotEmpty, NotNull} + import java.time.LocalDate + + case class RentalStation(@NotEmpty name: String) { + + def rentCar( + @NotNull customer: Customer, + @NotNull @Future startDate: LocalDate, + @Min(1) durationInDays: Int + ): Unit = ??? + } + +The following preconditions are specified here: + +* The `name` passed to the `RentalCar` constructor must not be empty. +* When invoking the `rentCar()` method, the given customer must not be null, the rental’s start date must not be `null` as well as be in the future and finally the rental duration must be at least one day. + +.. note:: + + Constraints may only be applied to instance methods, i.e. declaring constraints on static + methods is not supported. + +Cross-parameter constraints ++++++++++++++++++++++++++++ + +Sometimes validation does not only depend on a single parameter but on several or even all +parameters of a method. This kind of requirement can be fulfilled with help of a cross-parameter +constraint. + +Cross-parameter constraints can be considered as the method validation equivalent to class-level +constraints. Both can be used to implement validation requirements which are based on several +elements. While class-level constraints can apply to several fields of a case class, cross-parameter +constraints can apply to several parameters of a method. + +In contrast to single-parameter constraints, cross-parameter constraints are declared on the method. +In the example below, the cross-parameter constraint `@LuggageCountMatchesPassengerCount` declared +on the `load()` method is used to ensure that no passenger has more than two pieces of luggage. + +.. code:: scala + + case class Car(@NotEmpty manufacturer: String, @AssertTrue isRegistered: Boolean) { + + @LuggageCountMatchesPassengerCount(piecesOfLuggagePerPassenger = 2) + def load(passengers: Seq[Person], luggage: Seq[PieceOfLuggage]): Unit = ??? + } + +We'll see in the next section that return value constraints are also declared on the method level. +Thus, in order to distinguish cross-parameter constraints from return value constraints, the +constraint target is configured in the `ConstraintValidator` implementation using the `@SupportedValidationTarget `__ +annotation. + +More details in `“Creating custom constraints - Cross-parameter constraints” <#id4>`__ which shows +how to implement a cross-parameter constraint. + +Method Return value constraints +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The postconditions of a method or constructor are declared by adding constraint annotations to the +executable: + +.. code:: scala + + case class RentalStation @ValidRentalStation(@NotEmpty name: String) { + + @NotEmpty + @Size(min = 1) + def getCustomers: Seq[Customer] = ??? + } + +In the example above, the following constraints apply to executables (constructor and a method in +this case) of `RentalStation`: +* A new `RentalStation` instance must satisfy the `@ValidRentalStation` constraint +* The `Seq[Customer]` returned by `getCustomers` must not be empty and must contain at least on element + +Cascaded Validation +^^^^^^^^^^^^^^^^^^^ + +Like the cascaded validation of case class fields (see `“Object graphs” <#object-graphs>`__), the +`@Valid` annotation can be used to mark executable parameters and return values for cascaded +validation. When validating a parameter or return value annotated with `@Valid`, the constraints +declared on the parameter or return value object are also validated. + +.. code:: scala + + case class Car(@NotEmpty manufacturer: String, @NotEmpty @Size(min = 2, max = 14) licensePlate: String) + + case class Garage @Valid (@NotEmpty name: String) { + + def checkCar(@NotNull @Valid car: Car): Boolean = ??? + } + +Above, the `car` parameter of the method `Garage#checkCar` as well as the return value of the +`Garage` constructor are marked for cascaded validation. + +When validating the arguments of the `checkCar()` method, the constraints on the properties of the +passed `Car` object are evaluated as well. Similarly, the `@NotEmpty` constraint on the name field +of `Garage` is checked when validating the return value of the `Garage` constructor. + +Generally, the cascaded validation works for executables in the same way as it does for case class +fields. + +In particular, `null` or `None` values are ignored during cascaded validation (which is impossible +for constructor return values) and cascaded validation is performed recursively, i.e. if a parameter +or return value object which is marked for cascaded validation itself has properties marked with +`@Valid`, the constraints declared on the referenced elements will be validated as well. + +Also, the same issues with annotating container elements apply because of the Scala compiler +limitations but also note that more generally, the majority of executable methods proxy to the +underlying Hibernate Java library. + +Method constraints in inheritance hierarchies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note there are some rules for method constraints and inheritance as outlined in the +`Hibernate documentation `__. + +Validating method constraints +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The validation of method constraints is done using the `ScalaExecutableValidator` interface. + +Obtaining a `ScalaExecutableValidator` instance +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The `ScalaValidator` provides a `ScalaExecutableValidator` which implements a set of "executable" +methods that mirrors the bean validation `ExecutableValidator `__ API. + +All of the corollary `ExecutableValidator` methods proxy directly to the underlying Hibernate +Validator for support since it properly handles annotations in these cases and no special logic is +needed. + +The `ScalaExecutableValidator` can be obtained from the `ScalaValidator` by calling +`ScalaValidator#forExecutables` + +.. code:: scala + + import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.executable.ScalaExecutableValidator + + val validator: ScalaValidator = ScalaValidator() + val executableValidator: ScalaExecutableValidator = validator.forExecutables + +`ScalaExecutableValidator` methods +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The `ScalaExecutableValidator` interface offers altogether six methods: + +* `validateMethods()`, `validateMethod()`, `validateParameters()`, and `validateReturnValue()` for method validation +* `validateConstructorParameters()` and `validateConstructorReturnValue()` for constructor validation + +`ScalaExecutableValidator#validateMethods` +++++++++++++++++++++++++++++++++++++++++++ + +Validates all `@MethodValidation`-annotated methods of a given object. + +.. code:: scala + :emphasize-lines: 27 + + import com.twitter.util.validation.MethodValidation + import com.twitter.util.validation.engine.MethodValidationResult + import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.executable.ScalaExecutableValidator + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{Min, NotEmpty} + import java.time.LocalDate + + case class RentalCar( + @NotEmpty make: String, + @NotEmpty model: String, + @Min(2000) modelYear: Int) { + + @MethodValidation(fields = Array("modelYear")) + def onlyNewerCars: MethodValidationResult = { + // only want model years within the last 2 years + val year: Int = LocalDate.now.getYear + if ((year - modelYear) <= 2) MethodValidationResult.Valid() + else MethodValidationResult.Invalid("model year must be within the last 2 years") + } + } + + val rental = RentalCar("Renault", "Ellypse", 2002) + + val validator: ScalaValidator = ScalaValidator() + val executableValidator: ScalaExecutableValidator = validator.forExecutables + + val violations: Set[ConstraintViolation[RentalCar]] = executableValidator.validateMethods(rental) + assert(violations.size == 1) + assert(violations.head.getMessage == "model year must be within the last 2 years") + assert(violations.head.getPropertyPath.toString == "onlyNewerCars.modelYear") + assert(violations.head.getInvalidValue == rental) + +`ScalaExecutableValidator#validateMethod` ++++++++++++++++++++++++++++++++++++++++++ + +Validates the given `@MethodValidation`-annotated method of an object. + +.. code:: scala + :emphasize-lines: 30 + + import com.twitter.util.validation.MethodValidation + import com.twitter.util.validation.engine.MethodValidationResult + import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.executable.ScalaExecutableValidator + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{Min, NotEmpty} + import java.lang.reflect.Method + import java.time.LocalDate + + case class RentalCar( + @NotEmpty make: String, + @NotEmpty model: String, + @Min(2000) modelYear: Int) { + + @MethodValidation(fields = Array("modelYear")) + def onlyNewerCars: MethodValidationResult = { + // only want model years within the last 2 years + val year: Int = LocalDate.now.getYear + if ((year - modelYear) <= 2) MethodValidationResult.Valid() + else MethodValidationResult.Invalid("model year must be within the last 2 years") + } + } + + val rental = RentalCar("Renault", "Nepta", 2006) + + val validator: ScalaValidator = ScalaValidator() + val executableValidator: ScalaExecutableValidator = validator.forExecutables + + val method = classOf[RentalCar].getDeclaredMethod("onlyNewerCars") + + val violations: Set[ConstraintViolation[RentalCar]] = executableValidator.validateMethod(rental, method) + assert(violations.size == 1) + assert(violations.head.getMessage == "model year must be within the last 2 years") + assert(violations.head.getPropertyPath.toString == "onlyNewerCars.modelYear") + assert(violations.head.getInvalidValue == rental) + +`ScalaExecutableValidator#validateParameters` ++++++++++++++++++++++++++++++++++++++++++++++ + +Validates all constraints placed on the parameters of the given method. This method directly proxies +to the `underlying Hibernate validator `__. + +.. code:: scala + :emphasize-lines: 32,33,34,35 + + import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.executable.ScalaExecutableValidator + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{Min, NotEmpty, NotNull, Size} + import java.time.LocalDate + + case class RentalStation(@NotNull name: String) { + def rentCar( + @NotNull customer: Customer, + @NotNull @Future start: LocalDate, + @Min(1) duration: Int + ): Unit = ??? + + @NotEmpty + @Size(min = 1) + def getCustomers: Seq[Customer] = Seq.empty + } + + val rentalStation = RentalStation("Hertz") + + val validator: ScalaValidator = ScalaValidator() + val executableValidator: ScalaExecutableValidator = validator.forExecutables + + val method = + classOf[RentalStation].getMethod( + "rentCar", + classOf[Customer], + classOf[LocalDate], + classOf[Int]) + + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateParameters( + rentalStation, + method, + Array(customer, LocalDate.now(), Integer.valueOf(5))) + assert(violations.size == 1) + assert(violations.head.getMessage == "must be a future date") + assert(violations.head.getPropertyPath.toString == "rentCar.start") + +`ScalaExecutableValidator#validateReturnValue` +++++++++++++++++++++++++++++++++++++++++++++++ + +Validates return value constraints of the given method. This method directly proxies to the +`underlying Hibernate validator `__. + +.. code:: scala + :emphasize-lines: 27,28,29,30 + + import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.executable.ScalaExecutableValidator + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{Min, NotEmpty, NotNull, Size} + import java.time.LocalDate + + case class RentalStation(@NotNull name: String) { + def rentCar( + @NotNull customer: Customer, + @NotNull @Future start: LocalDate, + @Min(1) duration: Int + ): Unit = ??? + + @NotEmpty + @Size(min = 1) + def getCustomers: Seq[Customer] = Seq.empty + } + + val rentalStation = RentalStation("Hertz") + + val validator: ScalaValidator = ScalaValidator() + val executableValidator: ScalaExecutableValidator = validator.forExecutables + + val method = classOf[RentalStation].getMethod("getCustomers") + + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateReturnValue( + rentalStation, + method, + Seq.empty[Customer]) + assert(violations.size == 2) + assert(violations.head.getMessage == "must not be empty") + assert(violations.head.getPropertyPath.toString == "getCustomers.") + assert(violations.last.getMessage == "size must be between 1 and 2147483647") + assert(violations.last.getPropertyPath.toString == "getCustomers.") + +`ScalaExecutableValidator#validateConstructorParameters` +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Validates all constraints placed on the parameters of the given constructor. This method directly +proxies to the `underlying Hibernate validator `__. + +.. code:: scala + :emphasize-lines: 27,28,29 + + package com.example + + import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.executable.ScalaExecutableValidator + import jakarta.validation.ConstraintViolation + import jakarta.validation.constraints.{Min, NotEmpty, NotNull, Size} + import java.time.LocalDate + + case class RentalStation(@NotNull name: String) { + def rentCar( + @NotNull customer: Customer, + @NotNull @Future start: LocalDate, + @Min(1) duration: Int + ): Unit = ??? + + @NotEmpty + @Size(min = 1) + def getCustomers: Seq[Customer] = Seq.empty + } + + val validator: ScalaValidator = ScalaValidator() + val executableValidator: ScalaExecutableValidator = validator.forExecutables + + val constructor = classOf[RentalStation].getConstructor(classOf[String]) + + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateConstructorParameters( + constructor, + Array(null)) + assert(violations.size == 1) + assert(violations.head.getMessage == "must not be null") + assert(violations.head.getPropertyPath.toString == "RentalStation.name") + +`ScalaExecutableValidator#validateConstructorReturnValue` ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +Validates a constructor's return value. This method directly proxies to the +`underlying Hibernate validator `__. + +.. code:: scala + :emphasize-lines: 25,26,27 + + package com.example + + import com.twitter.util.validation.ScalaValidator + import com.twitter.util.validation.executable.ScalaExecutableValidator + import jakarta.validation.ConstraintViolation + + case class CarWithPassengerCount @ValidPassengerCountReturnValue(max = 1) (passengers: Seq[Person]) + + val car = CarWithPassengerCount( + Seq( + Person("abcd1234", "J Doe", DefaultAddress), + Person("abcd1235", "K Doe", DefaultAddress), + Person("abcd1236", "L Doe", DefaultAddress), + Person("abcd1237", "M Doe", DefaultAddress), + Person("abcd1238", "N Doe", DefaultAddress) + ) + ) + + val validator: ScalaValidator = ScalaValidator() + val executableValidator: ScalaExecutableValidator = validator.forExecutables + + val constructor = classOf[CarWithPassengerCount].getConstructor(classOf[Seq[Person]]) + + val violations: Set[ConstraintViolation[CarWithPassengerCount]] = + executableValidator.validateConstructorReturnValue( + constructor, + car) + assert(violations.size == 1) + assert(violations.head.getMessage == "invalid number of passengers") + assert(violations.head.getPropertyPath.toString == "CarWithPassengerCount.") + +Built-in method constraints +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In addition to the built-in case class and field-level constraints discussed in +`“Built-in constraints” <#built-in-constraints>`__, this library provides one method-level +constraint, `@MethodValidation`. This is a generic constraint which allows for implementation of +validation of the (possibly even internal) state of a case class instance. + +This is different from a class-level constraint since being implemented as a method within the case +class means that the method implementation potentially has access to private or internal state for +performing validation whereas a class-level constraint only has access to publicly exposed fields +and methods. + +* `@MethodValidation` + + Used to validate the (possibly internal) state of an entire object. It is important to note that any + annotated method **will be invoked when performing validation** and should thus be side-effect free. + + - **Supported data types** + + No-arg case class method that returns a `MethodValidationResult`. + +Grouping constraints +-------------------- + +See the `Hibernate documentation `__ +which explains constraint grouping in detail. + +Creating custom constraints +--------------------------- + +In cases where none of the `built-in constraints <#built-in-constraints>`__ suffice, you can create +custom constraints to implement your specific validation requirements. + +Creating a simple constraint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To create a custom constraint, the following three steps are required: + +* Create a constraint annotation +* Implement a `ConstraintValidator `__ +* Define a default error message + +The constraint annotation +^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is detailed in the `Hibernate documentation `__ +and all of the steps here are the same for use with this library. The linked documentation creates a +`CheckCase` annotation with two modes for validating a `String` is either uppercase or lowercase. +We'll assume the `CaseMode` enum and the `CheckCase` constraint annotation exists. + +The constraint validator +^^^^^^^^^^^^^^^^^^^^^^^^ + +Having defined the annotation, we now need to create an implementation of a `ConstraintValidator `__, +which is able to validate fields with a `@CheckCase` annotation. To do so, we implement the Jakarta Bean Validation interface `ConstraintValidator `__: + +.. code:: scala + + package com.example.constraint + + import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} + + class CheckCaseConstraintValidator extends ConstraintValidator[CheckCase, String] { + @volatile private[this] var caseMode: CaseMode = _ + + override def initialize(constraintAnnotation: CheckCase): Unit = { + this.caseMode = constraintAnnotation.value() + } + + def isValid( + obj: String, + constraintContext: ConstraintValidatorContext): Boolean = { + if (obj == null) true + else { + caseMode match { + case UPPER => + obj == obj.toUpperCase + case LOWER => + obj == obj.toLowerCase + } + } + } + } + +The `ConstraintValidator `__ +interface defines two type parameters which are set in the implementation. The first one specifies +the annotation type to be validated (in this case `CheckCase`), the second one the type of elements +the validator can handle (in this case, `String`). + +In case a constraint supports several data types, you can create a `ConstraintValidator `__ +implementation over the `Any` or `AnyRef` types and pattern-match within the implementation. If the +implementation receives a type that is unsupported you can throw an `UnexpectedTypeException `__. + +The implementation of the validator is straightforward: the `initialize` method gives you access +to the attribute values of the validated constraint and allows you to store them in a field of the +validator as shown in the example. + +The `isValid` method contains the validator logic. For `@CheckCase` this is the check as to whether +a given string is either completely lower case or upper case, depending on the `caseMode` set from +the constraint annotation value in `initialize`. + +.. important:: + + Note that the Jakarta Bean Validation specification recommends consideration of `null` values as + being valid. If `null` is not a valid value for an element, it should be **explicitly** annotated + with `@NotNull`. + + This library also extends the same consideration for `None` values during validation as + detailed in `With Option[_] <#with-option>`__. + +Thread-safety ++++++++++++++ + +As noted in the `javadocs `__: +access to `isValid` can happen concurrently and thus thread-safety must be ensured by the +implementation. Also note that `isValid` implementations **should not** alter the state of the value +to validate. + +The error message +^^^^^^^^^^^^^^^^^ + +The last piece is an error message which should be used when a `@CheckCase` constraint is violated. +You have a few options. You can specify the default error message directly within the `message()` +attribute of the constraint annotation, e.g., + +.. code:: java + + String message() default "Case mode must be {value}"; + +However, if you want to be able to `internationalize `__ +an error message, it is generally recommended to place the value in a `Resource Bundle `__, +specifying the resource key as the `message()` attribute in the constraint annotation. + +.. code:: java + + String message() default "{com.example.constraint.CheckCase.message}"; + +Create a `properties `__ file, +*ValidationMessages.properties* with the following contents: + +.. code:: text + + com.example.constraint.CheckCase.message=Case mode must be {value} + +Make sure the *ValidationMessages.properties* is packaged on your classpath to be picked up by the +library. + +For details, see the Hibernate documentation: `"Default message interpolation" `__. + +The `TwitterConstraintValidatorContext` ++++++++++++++++++++++++++++++++++++++++ + +First, see the documentation about the `ConstraintValidatorContext `__ +for background on its usage within a `ConstraintValidator `__ +implementation. + +This library provides a utility for customizing any returned `ConstraintViolation` from a +`ConstraintValidator` by using the `c.t.util.validation.constraintvalidation.TwitterConstraintValidatorContext`. + +The `TwitterConstraintValidatorContext` uses the `HibernateConstraintValidatorContext `__ +described in the `Custom contexts `__ +Hibernate documentation. See the referenced documentation for details. + +The `TwitterConstraintValidatorContext` has a simple DSL to create custom constraint violations. The +DSL allows you to set a dynamic `Payload `__, +set message parameters or expression variables for an error message, or specify a fully custom error +message template (that is a message template different than the default specified by the constraint +annotation). + +.. note:: + + As mentioned in the Hibernate `documentation `__: + + *[T]he main difference between message parameters and expression variables is that message parameters* + *are simply interpolated whereas expression variables are interpreted using the* `Expression Language engine `__. + + *In practice, use message parameters if you do not need the advanced features of an Expression Language.* + +By default, Expression Language support **is not enabled** for custom violations created via the +`ConstraintValidatorContext`. However, for some advanced requirements, using Expression Language may +be necessary. If you add an expression variable via the `TwitterConstraintValidatorContext`, +Expression Language support will be enabled when adding a constraint violation. + +.. code:: scala + :emphasize-lines: 25,26,27,28,29 + + import com.twitter.util.validation.constraintvalidation.TwitterConstraintValidatorContext + import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} + + class CheckCaseConstraintValidator extends ConstraintValidator[CheckCase, String] { + @volatile private[this] var caseMode: CaseMode = _ + + override def initialize(constraintAnnotation: CheckCase): Unit = { + this.caseMode = constraintAnnotation.value() + } + + def isValid( + obj: String, + constraintContext: ConstraintValidatorContext): Boolean = { + val valid = if (obj == null) true + else { + caseMode match { + case UPPER => + obj == obj.toUpperCase + case LOWER => + obj == obj.toLowerCase + } + } + + if (!valid) { + TwitterConstraintValidatorContext + .withDynamicPayload(???) + .addMessageParameter("foo", "snafu") + .withMessageTemplate("{foo}") // will be interpolated into the string 'snafu' + .addConstraintViolation(constraintContext) + } + + valid + } + } + +Using the constraint +^^^^^^^^^^^^^^^^^^^^ + +To use the custom `@CheckCase` constraint, annotate a case class field: + +.. code:: scala + :emphasize-lines: 7 + + package com.example + + import jakarta.validation.constraints.{Min, NotEmpty, Size} + + case class Car( + @NotEmpty manufacturer: String, + @NotEmpty @Size(min = 2, max = 14) @CheckCase(CaseMode.UPPER) licensePlate: String, + @Min(2) seatCount: Int) + +During validation, a `licensePlate` field with an invalid case will now cause validation to fail. + +.. code:: scala + + // invalid license plate + val car: Car = Car("Morris", "dd-ab-123", 4) + + val violations: Set[ConstraintViolation[Car]] = validator.validate(car) + assert(violations.size == 1) + assert(violations.head.getMessage == "Case mode must be UPPER") + + // valid license plate + val car: Car = Car("Morris", "DD-AB-123", 4) + assert(validator.validate(car).size == 0) + +Class-level constraints +~~~~~~~~~~~~~~~~~~~~~~~ + +As mentioned previously, constraints can also be applied on the class-level to validate the state of +the entire case class instance. These constraints are defined in the same way as field constraints. + +Here we'll see how to implement the `@ValidPassengerCount` annotation seen in an earlier example: + +.. code:: scala + :emphasize-lines: 6 + + package com.example + + import com.example.constraint.ValidPassengerCount + import jakarta.validation.constraints.Min + + @ValidPassengerCount + case class Car( + @Min(2) seatCount: Int, + passengers: Seq[Person]) + +First the annotation definition could be written as: + +.. code:: java + + package com.example.constraint; + + @Target({ TYPE, ANNOTATION_TYPE }) + @Retention(RUNTIME) + @Constraint(validatedBy = { ValidPassengerCountValidator.class }) + @Documented + public @interface ValidPassengerCount { + + String message() default "{com.example.constraint.ValidPassengerCount.message}"; + + Class[] groups() default { }; + + Class[] payload() default { }; + } + +We define an error message in a *Validation.properties* file: + +.. code:: text + + com.example.constraint.ValidPassengerCount.message=invalid number of passengers + +Then the `ConstraintValidator` implementation which is specified to a `Car` type: + +.. code:: scala + + package com.example + + import com.example.constraint.ValidPassengerCount + import jakarta.validation.{ConstraintValidator + + class ValidPassengerCountValidator extends ConstraintValidator[ValidPassengerCount, Car] { + + def isValid( + car: Car, + constraintContext: ConstraintValidatorContext + ): Boolean = + if (car == null) true + else car.passengers.size <= car.seatCount + } + +Note as the example shows, you need to use the element type `TYPE` in the `@Target `__ +annotation on the constraint annotation. This allows the constraint to be put on type definitions. +The `ValidPassengerCountValidator` receives an instance of a `Car` in the `isValid()` method and can +access the complete object state to decide whether the given instance is valid or not. + +During validation, an instance which does not satisfy the constraint will fail the validation. + +.. code:: scala + + package com.example + + import com.twitter.util.validation.ScalaValidator + import jakarta.validation.ConstraintViolation + + val car = Car( + seatCount = 2, + passengers = Seq( + Person("abcd1234", "J Doe", DefaultAddress), + Person("abcd1235", "K Doe", DefaultAddress), + Person("abcd1236", "L Doe", DefaultAddress), + Person("abcd1237", "M Doe", DefaultAddress), + Person("abcd1238", "N Doe", DefaultAddress) + ) + ) + + val validator: ScalaValidator = ScalaValidator() + + val violations: Set[ConstraintViolation[Car]] = + validator.validate(car) + assert(violations.size == 1) + assert(violations.head.getMessage == "invalid number of passengers") + assert(violations.head.getPropertyPath.toString == "Car") + +Custom property paths +^^^^^^^^^^^^^^^^^^^^^ + +By default for a class-level constraint, any constraint violation is reported on the level of the +annotated type, e.g. in this case the `Car` type. If the class is the top-level, the property path +will be empty, otherwise it will be the field name referencing the class type. + +See the `Hibernate documentation `__ +for details on how to set custom property paths when implementing a class-level constraint. + +Cross-parameter constraints +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Bean validation specification distinguishes two types of constraints: generic constraints which +apply to the annotated element (e.g., a type, field, container element, method parameter or return +value, etc.), and *cross-parameter* constraints. Cross-parameter constraints by contrast, apply to +an array of parameters of a method or constructor and can be used to express validation logic which +depends on several parameter values. + +See the `Hibernate documentation `__ +for details on implementing cross-parameter constraints. + +Constraint composition +~~~~~~~~~~~~~~~~~~~~~~ + +To keep the annotation of case class fields from becoming unwieldy or confusing, you can use a +composing constraint. See the `Hibernate documentation `__ +for details on creating a composing constraint. + +`@MethodValidation` constraints +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. important:: + + The `@MethodValidation` constraint annotation MUST be applied to a no-arg method that returns a + `com.twitter.util.validation.engine.MethodValidationResult`. + + These annotated methods **will be invoked** during validation and should be side-effect free. + +In a case class, define a no-arg method that returns a `MethodValidationResult`. Annotate the method +with `@MethodValidation`, optionally specifying validated fields of the case class for +error-reporting. + +.. code:: scala + :emphasize-lines: 13,14 + + import com.twitter.util.validation.MethodValidation + import com.twitter.util.validation.engine.MethodValidationResult + import jakarta.validation.constraints.{Min, NotEmpty} + import java.time.LocalDate + + case class RentalCar( + @NotEmpty make: String, + @NotEmpty model: String, + @Min(2000) modelYear: Int) { + + @MethodValidation(fields = Array("modelYear")) + def onlyNewerCars: MethodValidationResult = { + // only want model years within the last 2 years + val year: Int = LocalDate.now.getYear + if ((year - modelYear) <= 2) MethodValidationResult.Valid() + else MethodValidationResult.Invalid("model year must be within the last 2 years") + } + } + +Value Extraction +---------------- + +Value extraction is the process of extracting values from a container so that they can be validated. + +It is used when dealing with `container element constraints <#container-element-constraints>`__ and +`cascaded validation <#object-graphs>`__ inside containers. + +Built-in value extractors +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This library provides built-in value extraction for Scala `Iterable[_] <#iterable>`__ types. See the +`Hibernate documentation `__ +for more details on value extraction including how to define and register a new value extractor. + +More resources +-------------- + +* `Hibernate Validator Specifics `__ (from reference guide) +* `Hibernate Validator Reference Guide `__ +* `Jakarta Bean Validation API 3.0.0 Javadocs `__ +* `Hibernate Validator API Javadocs `__ +* `Jakarta Bean Validation `__ diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index a7945b1056..bcacbf9800 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -3,12 +3,14 @@ scala_library( compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/openjdk/jmh:jmh-core", "3rdparty/jvm/org/scala-lang:scala-library", "util/util-core:scala", "util/util-hashing/src/main/scala", "util/util-reflect/src/main/scala/com/twitter/util/reflect", "util/util-stats/src/main/scala", + "util/util-validator/src/main/scala/com/twitter/util/validation", ], ) diff --git a/util-benchmark/src/main/scala/com/twitter/util/ValidationBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/ValidationBenchmark.scala new file mode 100644 index 0000000000..7f8f27143e --- /dev/null +++ b/util-benchmark/src/main/scala/com/twitter/util/ValidationBenchmark.scala @@ -0,0 +1,79 @@ +package com.twitter.util + +import com.twitter.util.validation.{MethodValidation, ScalaValidator} +import com.twitter.util.validation.engine.MethodValidationResult +import jakarta.validation.ValidationException +import jakarta.validation.constraints.{Min, NotEmpty} +import org.openjdk.jmh.annotations.{Benchmark, Scope, State} +import scala.util.control.NonFatal + +private object ValidationBenchmark { + case class User(@NotEmpty id: String, name: String, @Min(18) age: Int) + + case class Users(@NotEmpty users: Seq[User]) { + @MethodValidation + def uniqueUsers: MethodValidationResult = + MethodValidationResult.validIfTrue( + users.map(_.id).distinct.size == users.size, + "user ids are not distinct.") + } +} + +// ./sbt 'project util-benchmark' 'jmh:run ValidationBenchmark' +@State(Scope.Benchmark) +class ValidationBenchmark extends StdBenchAnnotations { + import ValidationBenchmark._ + + private[this] val validator: ScalaValidator = ScalaValidator() + private[this] val validUser: User = User("1234567", "jack", 21) + private[this] val invalidUser: User = User("", "notJack", 13) + private[this] val nestedValidUser: Users = Users(Seq(validUser)) + private[this] val nestedInvalidUser: Users = Users(Seq(invalidUser)) + private[this] val nestedDuplicateUser: Users = Users(Seq(validUser, validUser)) + + @Benchmark + def withValidUser(): Unit = { + validator.verify(validUser) + } + + @Benchmark + def withInvalidUser(): Unit = { + try { + validator.verify(invalidUser) + } catch { + case _: ValidationException => // avoid throwing exceptions so the benchmark can finish + } + } + + @Benchmark + def withNestedValidUser(): Unit = { + validator.verify(nestedValidUser) + } + + @Benchmark + def withNestedInvalidUser(): Unit = { + try { + validator.verify(nestedInvalidUser) + } catch { + case _: ValidationException => // avoid throwing exceptions so the benchmark can finish + } + } + + @Benchmark + def withNestedDuplicateUser(): Unit = { + try { + validator.verify(nestedDuplicateUser) + } catch { + case _: ValidationException => // avoid throwing exceptions so the benchmark can finish + } + } + + override def finalize(): Unit = { + try { + validator.close() + } catch { + case NonFatal(_) => // do nothing + } + super.finalize() + } +} diff --git a/util-benchmark/src/test/scala/BUILD b/util-benchmark/src/test/scala/BUILD new file mode 100644 index 0000000000..0a92dcf982 --- /dev/null +++ b/util-benchmark/src/test/scala/BUILD @@ -0,0 +1,17 @@ +junit_tests( + sources = [ + "**/*.scala", + ], + compiler_option_sets = ["fatal_warnings"], + dependencies = [ + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:scalacheck", + scoped( + "3rdparty/jvm/org/slf4j:slf4j-simple", + scope = "runtime", + ), + "util/util-benchmark/src/main/scala", + ], +) diff --git a/util-benchmark/src/test/scala/com/twitter/util/ValidationBenchmarkTest.scala b/util-benchmark/src/test/scala/com/twitter/util/ValidationBenchmarkTest.scala new file mode 100644 index 0000000000..41bb2b979f --- /dev/null +++ b/util-benchmark/src/test/scala/com/twitter/util/ValidationBenchmarkTest.scala @@ -0,0 +1,21 @@ +package com.twitter.util + +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class ValidationBenchmarkTest extends AnyFunSuite { + private[this] val benchmark = new ValidationBenchmark() + + test("validate case class") { + benchmark.withValidUser() + benchmark.withInvalidUser() + } + + test("validate nested case class") { + benchmark.withNestedValidUser() + benchmark.withNestedInvalidUser() + benchmark.withNestedDuplicateUser() + } +} diff --git a/util-validator/src/main/java/com/twitter/util/validation/BUILD b/util-validator/src/main/java/com/twitter/util/validation/BUILD new file mode 100644 index 0000000000..5109e5e03d --- /dev/null +++ b/util-validator/src/main/java/com/twitter/util/validation/BUILD @@ -0,0 +1,16 @@ +java_library( + sources = ["*.java"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-java", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + ], + exports = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + ], +) diff --git a/util-validator/src/main/java/com/twitter/util/validation/MethodValidation.java b/util-validator/src/main/java/com/twitter/util/validation/MethodValidation.java new file mode 100644 index 0000000000..9aee33853c --- /dev/null +++ b/util-validator/src/main/java/com/twitter/util/validation/MethodValidation.java @@ -0,0 +1,47 @@ +package com.twitter.util.validation; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * A type of [[jakarta.validation.Constraint]] used to annotate case class methods + * to invoke for additional instance state validation. + */ +@Documented +@Constraint(validatedBy = {}) +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +public @interface MethodValidation { + + /** Every constraint annotation must define a message element of type String. */ + String message() default ""; + + /** + * Every constraint annotation must define a groups element that specifies the processing groups + * with which the constraint declaration is associated. + */ + Class[] groups() default {}; + + /** + * Constraint annotations must define a payload element that specifies the payload with which + * the constraint declaration is associated. * + */ + Class[] payload() default {}; + + /** + * Specify the fields validated by the method. Used in error reporting. + */ + String[] fields() default {}; +} diff --git a/util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD b/util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD new file mode 100644 index 0000000000..91c29f7cd3 --- /dev/null +++ b/util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD @@ -0,0 +1,16 @@ +java_library( + sources = ["*.java"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-java-constraints", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + ], + exports = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + ], +) diff --git a/util-validator/src/main/java/com/twitter/util/validation/constraints/CountryCode.java b/util-validator/src/main/java/com/twitter/util/validation/constraints/CountryCode.java new file mode 100644 index 0000000000..9a8f5ba719 --- /dev/null +++ b/util-validator/src/main/java/com/twitter/util/validation/constraints/CountryCode.java @@ -0,0 +1,52 @@ +package com.twitter.util.validation.constraints; + +import java.lang.annotation.Documented; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import com.twitter.util.validation.constraints.CountryCode.List; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = {}) +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +@Repeatable(List.class) +public @interface CountryCode { + + /** Every constraint annotation must define a message element of type String. */ + String message() default "{com.twitter.util.validation.constraints.CountryCode.message}"; + + /** + * Every constraint annotation must define a groups element that specifies the processing groups + * with which the constraint declaration is associated. + */ + Class[] groups() default {}; + + /** + * Constraint annotations must define a payload element that specifies the payload with which + * the constraint declaration is associated. + */ + Class[] payload() default {}; + + /** + * Defines several {@code @CountryCode} annotations on the same element. + */ + @Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) + @Retention(RUNTIME) + @Documented + public @interface List { + CountryCode[] value(); + } +} diff --git a/util-validator/src/main/java/com/twitter/util/validation/constraints/OneOf.java b/util-validator/src/main/java/com/twitter/util/validation/constraints/OneOf.java new file mode 100644 index 0000000000..275d148653 --- /dev/null +++ b/util-validator/src/main/java/com/twitter/util/validation/constraints/OneOf.java @@ -0,0 +1,57 @@ +package com.twitter.util.validation.constraints; + +import java.lang.annotation.Documented; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import com.twitter.util.validation.constraints.OneOf.List; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = {}) +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +@Repeatable(List.class) +public @interface OneOf { + + /** Every constraint annotation must define a message element of type String. */ + String message() default "{com.twitter.util.validation.constraints.OneOf.message}"; + + /** + * Every constraint annotation must define a groups element that specifies the processing groups + * with which the constraint declaration is associated. + */ + Class[] groups() default {}; + + /** + * Constraint annotations must define a payload element that specifies the payload with which + * the constraint declaration is associated. * + */ + Class[] payload() default {}; + + /** + * The array of valid values. + */ + String[] value(); + + /** + * Defines several {@code @OneOf} annotations on the same element. + */ + @Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) + @Retention(RUNTIME) + @Documented + public @interface List { + OneOf[] value(); + } +} diff --git a/util-validator/src/main/java/com/twitter/util/validation/constraints/UUID.java b/util-validator/src/main/java/com/twitter/util/validation/constraints/UUID.java new file mode 100644 index 0000000000..7a5b969db3 --- /dev/null +++ b/util-validator/src/main/java/com/twitter/util/validation/constraints/UUID.java @@ -0,0 +1,52 @@ +package com.twitter.util.validation.constraints; + +import java.lang.annotation.Documented; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import com.twitter.util.validation.constraints.UUID.List; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = {}) +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +@Repeatable(List.class) +public @interface UUID { + + /** Every constraint annotation must define a message element of type String. */ + String message() default "{com.twitter.util.validation.constraints.UUID.message}"; + + /** + * Every constraint annotation must define a groups element that specifies the processing groups + * with which the constraint declaration is associated. + */ + Class[] groups() default {}; + + /** + * Constraint annotations must define a payload element that specifies the payload with which + * the constraint declaration is associated. * + */ + Class[] payload() default {}; + + /** + * Defines several {@code @UUID} annotations on the same element. + */ + @Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) + @Retention(RUNTIME) + @Documented + public @interface List { + UUID[] value(); + } +} diff --git a/util-validator/src/main/resources/BUILD b/util-validator/src/main/resources/BUILD new file mode 100644 index 0000000000..a1246ebdf9 --- /dev/null +++ b/util-validator/src/main/resources/BUILD @@ -0,0 +1,6 @@ +resources( + sources = [ + "**/*.properties", + "**/*.xml", + ], +) diff --git a/util-validator/src/main/resources/META-INF/validation.xml b/util-validator/src/main/resources/META-INF/validation.xml new file mode 100644 index 0000000000..a2678728f0 --- /dev/null +++ b/util-validator/src/main/resources/META-INF/validation.xml @@ -0,0 +1,8 @@ + + com/twitter/util/validation/constraint-mappings.xml + diff --git a/util-validator/src/main/resources/ValidationMessages.properties b/util-validator/src/main/resources/ValidationMessages.properties new file mode 100644 index 0000000000..30c2277f07 --- /dev/null +++ b/util-validator/src/main/resources/ValidationMessages.properties @@ -0,0 +1,3 @@ +com.twitter.util.validation.constraints.CountryCode.message=${validatedValue} not a valid country code +com.twitter.util.validation.constraints.OneOf.message=${validatedValue} not one of {value} +com.twitter.util.validation.constraints.UUID.message=must be a valid UUID diff --git a/util-validator/src/main/resources/com/twitter/util/validation/constraint-mappings.xml b/util-validator/src/main/resources/com/twitter/util/validation/constraint-mappings.xml new file mode 100644 index 0000000000..2622a2bf04 --- /dev/null +++ b/util-validator/src/main/resources/com/twitter/util/validation/constraint-mappings.xml @@ -0,0 +1,32 @@ + + + + com.twitter.util.validation.internal.validators.ISO3166CountryCodeConstraintValidator + + + + + com.twitter.util.validation.internal.validators.OneOfConstraintValidator + + + + + com.twitter.util.validation.internal.validators.UUIDConstraintValidator + + + + + com.twitter.util.validation.internal.validators.notempty.NotEmptyValidatorForIterable + + + + + com.twitter.util.validation.internal.validators.size.SizeValidatorForIterable + + + diff --git a/util-validator/src/main/scala/com/twitter/util/validation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/BUILD new file mode 100644 index 0000000000..0b473eeaa5 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/BUILD @@ -0,0 +1,49 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-core", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/github/ben-manes/caffeine", + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "3rdparty/jvm/org/json4s:json4s-core", + "util/util-core:scala", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + "util/util-slf4j-api/src/main/scala", + "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator/src/main/java/com/twitter/util/validation/constraints", + "util/util-validator/src/main/resources", # loads custom constraint mappings and validation messages + "util/util-validator/src/main/scala/com/twitter/util/validation/cfg", + "util/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation", + "util/util-validator/src/main/scala/com/twitter/util/validation/conversions", + "util/util-validator/src/main/scala/com/twitter/util/validation/engine", + "util/util-validator/src/main/scala/com/twitter/util/validation/executable", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/engine", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/validators", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size", + "util/util-validator/src/main/scala/com/twitter/util/validation/metadata", + "util/util-validator/src/main/scala/org/hibernate/validator/internal/engine", + "util/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean", + ], + exports = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/json4s:json4s-core", + "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator/src/main/java/com/twitter/util/validation/constraints", + "util/util-validator/src/main/scala/com/twitter/util/validation/cfg", + "util/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation", + "util/util-validator/src/main/scala/com/twitter/util/validation/conversions", + "util/util-validator/src/main/scala/com/twitter/util/validation/engine", + "util/util-validator/src/main/scala/com/twitter/util/validation/executable", + "util/util-validator/src/main/scala/com/twitter/util/validation/metadata", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/DescriptorFactory.scala b/util-validator/src/main/scala/com/twitter/util/validation/DescriptorFactory.scala new file mode 100644 index 0000000000..84aa137672 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/DescriptorFactory.scala @@ -0,0 +1,429 @@ +package com.twitter.util.validation + +import com.github.benmanes.caffeine.cache.{Cache, Caffeine} +import com.twitter.util.reflect.{Annotations, Types => ReflectTypes} +import com.twitter.util.validation.engine.MethodValidationResult +import com.twitter.util.validation.internal.Types +import com.twitter.util.validation.metadata.{ + CaseClassDescriptor, + ConstructorDescriptor, + MethodDescriptor, + PropertyDescriptor +} +import com.twitter.util.{Return, Try} +import jakarta.validation.{Constraint, ConstraintDeclarationException, Valid, ValidationException} +import java.lang.annotation.Annotation +import java.lang.reflect.{Constructor, Method, Parameter} +import java.util.function.Function +import org.hibernate.validator.internal.metadata.core.ConstraintHelper +import org.json4s.reflect.{ClassDescriptor, ConstructorParamDescriptor, Reflector, ScalaType} +import org.json4s.{reflect => json4s} +import scala.collection.mutable + +private[validation] object DescriptorFactory { + + def unmangleName(name: String): String = scala.reflect.NameTransformer.decode(name) + def mangleName(name: String): String = scala.reflect.NameTransformer.encode(name) + + /** If the given class is marked for cascaded validation or not. True if the class is a case class and has the `@Valid` annotation */ + private def isCascadedValidation(erasure: Class[_], annotations: Array[Annotation]): Boolean = + Annotations.findAnnotation[Valid](annotations).isDefined && ReflectTypes.isCaseClass(erasure) + + private case class ConstructorParams( + param: ConstructorParamDescriptor, + annotations: Array[Annotation]) + + private def getCascadedScalaType( + scalaType: ScalaType, + annotations: Array[Annotation] + ): Option[ScalaType] = + Types.getContainedScalaType(scalaType) match { + case Some(containedScalaType) + if isCascadedValidation(containedScalaType.erasure, annotations) => + Some(containedScalaType) + case _ => None + } +} + +/** + * Used to describe a given Class[T] as a CaseClassDescriptor[T] + */ +private[validation] class DescriptorFactory( + descriptorCacheSize: Long, + constraintHelper: ConstraintHelper) { + + import DescriptorFactory._ + + // A caffeine cache to store the expensive reflection calls on the same object. Caffeine cache + // uses the `Window TinyLfu` policy to remove evicted keys. + // For more information, check out: https://github.com/ben-manes/caffeine/wiki/Efficiency + private[validation] val caseClassDescriptors: Cache[Class[_], CaseClassDescriptor[_]] = + Caffeine + .newBuilder() + .maximumSize(descriptorCacheSize) + .build[Class[_], CaseClassDescriptor[_]]() + + /** Close and attempt to clean up resources */ + def close(): Unit = { + caseClassDescriptors.invalidateAll() + caseClassDescriptors.cleanUp() + } + + /** Describe a [[Class]] */ + def describe[T]( + clazz: Class[T] + ): CaseClassDescriptor[T] = { + caseClassDescriptors + .get( + clazz, + new Function[Class[_], CaseClassDescriptor[_]] { + def apply(key: Class[_]): CaseClassDescriptor[_] = { + buildDescriptor[T](clazz) + } + } + ).asInstanceOf[CaseClassDescriptor[T]] + } + + /** + * Describe a [[Method]]. + * + * As we do not want to describe every possible case class method, this potentially + * returns a None in the case where the method has no constraint annotation and no + * constrained parameters. + */ + def describe(method: Method): Option[MethodDescriptor] = buildMethodDescriptor(method) + + /** Describe a [[Constructor]]. */ + def describe[T]( + constructor: Constructor[T] + ): ConstructorDescriptor = describe(constructor, None) + + /** Describe a [[Constructor]] given an optional "mix-in" Class. */ + def describe[T]( + constructor: Constructor[T], + mixinClazz: Option[Class[_]] + ): ConstructorDescriptor = { + val parameters: Array[Parameter] = constructor.getParameters + val desc = org.json4s.reflect.ConstructorDescriptor( + params = Seq.tabulate(parameters.length) { index => + val parameter = parameters(index) + org.json4s.reflect.ConstructorParamDescriptor( + name = unmangleName(parameter.getName), + mangledName = parameter.getName, + argIndex = index, + argType = Reflector.scalaTypeOf(parameter.getParameterizedType), + defaultValue = None + ) + }, + constructor = new org.json4s.reflect.Executable(constructor, true), + isPrimary = true + ) + buildConstructorDescriptor[T](constructor.getDeclaringClass, mixinClazz, desc) + } + + /* Private */ + + private[this] def buildDescriptor[T](clazz: Class[T]): CaseClassDescriptor[T] = { + // for case classes, annotations for params only appear on the executable + val clazzDescriptor: ClassDescriptor = Reflector.describe(clazz).asInstanceOf[ClassDescriptor] + // map of parameter name to a class with all annotations for the parameter (including inherited) + val constructorParams: scala.collection.Map[String, ConstructorParams] = + getConstructorParams(clazz, clazzDescriptor) + + val members = new mutable.HashMap[String, PropertyDescriptor]() + // we validate only declared properties of the class + val properties: Seq[json4s.PropertyDescriptor] = clazzDescriptor.properties + var index = 0 + val size = properties.size + while (index < size) { + val property = properties(index) + val (fromConstructorScalaType, annotations) = { + constructorParams.get(property.mangledName) match { + case Some(constructorParam) => + // we already have information for the field as it is an annotated executable parameter + (constructorParam.param.argType, constructorParam.annotations) + case _ => + // we don't have information, use the property information + (property.returnType, property.field.getAnnotations) + } + } + + // create property descriptorFactory + members.put( + property.mangledName, + buildPropertyDescriptor(fromConstructorScalaType, annotations) + ) + index += 1 + } + + val constructors = new mutable.ArrayBuffer[ConstructorDescriptor]() + // create executable descriptors + var jindex = 0 + val jsize = clazzDescriptor.constructors.size + while (jindex < jsize) { + constructors.append( + buildConstructorDescriptor(clazz, None, clazzDescriptor.constructors(jindex)) + ) + jindex += 1 + } + + val methods = new mutable.ArrayBuffer[MethodDescriptor]() + val clazzMethods = clazz.getMethods + if (clazzMethods.nonEmpty) { + var index = 0 + val length = clazzMethods.length + while (index < length) { + val method = clazzMethods(index) + buildMethodDescriptor(method).foreach(methods.append(_)) + index += 1 + } + } + + CaseClassDescriptor( + clazz = clazz, + scalaType = clazzDescriptor.erasure, + annotations = clazzDescriptor.erasure.erasure.getAnnotations.filter(isConstraintAnnotation), + constructors = constructors.toArray, + members = members.toMap, + methods = methods.toArray + ) + } + + /* Exposed for testing */ + // find the executable with parameters that are all declared class fields + private[validation] def findConstructor( + clazzDescriptor: ClassDescriptor + ): org.json4s.reflect.ConstructorDescriptor = { + def isDefaultConstructorDescriptor( + constructorDescriptor: org.json4s.reflect.ConstructorDescriptor + ): Boolean = { + constructorDescriptor.isPrimary || constructorDescriptor.params.forall { param => + Try(clazzDescriptor.erasure.erasure.getDeclaredField(param.mangledName)).isReturn + } + } + + // locate the executable where all params are also declared fields else error + clazzDescriptor.constructors + .find(isDefaultConstructorDescriptor) + .getOrElse(throw new ValidationException( + s"Unable to parse case class for validation: ${clazzDescriptor.erasure.fullName}")) + } + + private[this] def getConstructorParams( + clazz: Class[_], + clazzDescriptor: ClassDescriptor + ): scala.collection.Map[String, ConstructorParams] = { + // find the default executable descriptorFactory (the underlying executable + // may be a executable or a method) + val constructorDescriptor: org.json4s.reflect.ConstructorDescriptor = + findConstructor(clazzDescriptor) + getConstructorParams( + clazz, + constructorDescriptor + ) + } + + private[this] def getConstructorParams( + clazz: Class[_], + constructorDescriptor: org.json4s.reflect.ConstructorDescriptor + ): scala.collection.Map[String, ConstructorParams] = { + val constructorParameters: Array[Parameter] = { + Option(constructorDescriptor.constructor.constructor) match { + case Some(cons) => + cons.getParameters + case _ => + // factory method "executable" + constructorDescriptor.constructor.method.getParameters + } + } + + // find all inherited annotations for every executable param + val allFieldAnnotations = + findAnnotations( + clazz, + clazz.getDeclaredFields.map(_.getName).toSet, + constructorDescriptor.params.map { param => + // annotations can be on the executable param OR they were + // copied to the generated field with a meta annotation. we + // rely on the fact that a given annotation should not show up in + // both arrays the way the Scala compiler currently works, however + // the executable we're iterating through may not represent + // actual declared fields in the class, so we have to apply logic + param.mangledName -> getAnnotations( + param.mangledName, + constructorParameters(param.argIndex), + clazz) + }.toMap + ) + + val result: mutable.HashMap[String, ConstructorParams] = + new mutable.HashMap[String, ConstructorParams]() + var index = 0 + val length = constructorParameters.length + while (index < length) { + val descriptor = constructorDescriptor.params(index) + val filteredAnnotations = allFieldAnnotations(descriptor.mangledName).filter { ann => + Annotations.isAnnotationPresent[Constraint](ann) || + Annotations.equals[Valid](ann) + } + + result.put( + descriptor.mangledName, + ConstructorParams( + descriptor, + filteredAnnotations + ) + ) + index += 1 + } + result + } + + private[this] def getAnnotations( + name: String, + parameter: Parameter, + clazz: Class[_] + ): Array[Annotation] = { + val fromClazzAnnotations: Array[Annotation] = + Try(clazz.getDeclaredField(name)) match { + case Return(declaredField) => + declaredField.getAnnotations + case _ => + Array.empty[Annotation] + } + parameter.getAnnotations ++ fromClazzAnnotations + } + + private[this] def findAnnotations( + clazz: Class[_], + declaredFields: Set[String], + fieldAnnotations: scala.collection.Map[String, Array[Annotation]] + ): scala.collection.Map[String, Array[Annotation]] = { + val collectorMap = new scala.collection.mutable.HashMap[String, Array[Annotation]]() + collectorMap ++= fieldAnnotations + // find inherited annotations + Annotations.findDeclaredAnnotations( + clazz, + declaredFields, + collectorMap + ) + } + + private[this] def buildConstructorDescriptor[T]( + clazz: Class[T], + mixinClazz: Option[Class[_]], + constructorDescriptor: org.json4s.reflect.ConstructorDescriptor + ): ConstructorDescriptor = { + val parameterAnnotations = + mixinClazz match { + case Some(mixin) => + val constructorAnnotationMap = getConstructorParams(clazz, constructorDescriptor) + // augment with mixin class field annotations + constructorAnnotationMap.map { + case (name, params) => + Try(mixin.getDeclaredMethod(name)).toOption match { + case Some(mixinField) => + ( + name, + params.copy(annotations = + params.annotations ++ + mixinField.getAnnotations.filter(isConstraintAnnotation)) + ) + case _ => // do nothing + (name, params) + } + } + case _ => + getConstructorParams(clazz, constructorDescriptor) + } + + val (executable, annotations) = Option(constructorDescriptor.constructor.constructor) match { + case Some(constructor) => + (constructor, constructor.getAnnotations) + case _ => + ( + constructorDescriptor.constructor.method, + constructorDescriptor.constructor.method.getAnnotations) + } + + ConstructorDescriptor( + executable, + annotations = annotations.filter(isConstraintAnnotation), + members = constructorDescriptor.params.map { parameter => + parameter.mangledName -> buildPropertyDescriptor( + parameter.argType, + parameterAnnotations(parameter.mangledName).annotations + ) + }.toMap + ) + } + + /** Create a PropertyDescriptor from the given ScalaType and annotations */ + private[this] def buildPropertyDescriptor[T]( + scalaType: ScalaType, + annotations: Array[Annotation] + ): PropertyDescriptor = { + val cascadedScalaType = getCascadedScalaType(scalaType, annotations) + val isCascaded = // annotated with @Valid and is a case class type + Annotations.findAnnotation(classOf[Valid], annotations).isDefined && + cascadedScalaType.exists(p => ReflectTypes.isCaseClass(p.erasure)) + + PropertyDescriptor( + scalaType = scalaType, + cascadedScalaType = cascadedScalaType, + annotations = annotations.filter(isConstraintAnnotation), + isCascaded = isCascaded + ) + } + + /** Create a MethodDescriptor for a given clazz Method */ + private[this] def buildMethodDescriptor(method: Method): Option[MethodDescriptor] = { + if (isMethodValidation(method) && checkMethodValidationMethod(method)) { + Some( + MethodDescriptor( + method = method, + annotations = method.getDeclaredAnnotations.filter(isConstraintAnnotation), + members = Map.empty + ) + ) + } else if (isConstrainedMethod(method) || hasConstrainedParameters(method)) { + Some( + MethodDescriptor( + method = method, + annotations = method.getDeclaredAnnotations.filter(isConstraintAnnotation), + members = method.getParameters.map { parameter => + parameter.getName -> buildPropertyDescriptor( + Reflector.scalaTypeOf(parameter.getParameterizedType), + parameter.getAnnotations + ) + }.toMap + ) + ) + } else None + } + + private[this] def checkMethodValidationMethod(method: Method): Boolean = { + if (method.getParameterCount != 0) + throw new ConstraintDeclarationException( + s"Methods annotated with @${classOf[MethodValidation].getSimpleName} must not declare any arguments") + if (method.getReturnType != classOf[MethodValidationResult]) + throw new ConstraintDeclarationException( + s"Methods annotated with @${classOf[MethodValidation].getSimpleName} must return a ${classOf[MethodValidationResult]}") + true + } + + private[this] def isConstraintAnnotation(annotation: Annotation): Boolean = + constraintHelper.isConstraintAnnotation(annotation.annotationType()) + + // Array of annotations contains a constraint annotation + private[this] def isConstrainedMethod(method: Method): Boolean = + method.getDeclaredAnnotations.exists(isConstraintAnnotation) + + private[this] def hasConstrainedParameters(method: Method): Boolean = { + method.getParameters.exists(_.getAnnotations.exists(isConstraintAnnotation)) + } + + // Array of annotation contains @MethodValidation + private[this] def isMethodValidation(method: Method): Boolean = + Annotations.findAnnotation[MethodValidation](method.getDeclaredAnnotations).isDefined +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala b/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala new file mode 100644 index 0000000000..bac1195732 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala @@ -0,0 +1,1470 @@ +package com.twitter.util.validation + +import com.twitter.util.logging.Logging +import com.twitter.util.reflect.{Classes, Annotations => ReflectAnnotations, Types => ReflectTypes} +import com.twitter.util.validation.cfg.ConstraintMapping +import com.twitter.util.validation.engine.MethodValidationResult +import com.twitter.util.validation.executable.ScalaExecutableValidator +import com.twitter.util.validation.internal.constraintvalidation.ConstraintValidatorContextFactory +import com.twitter.util.validation.internal.engine.ConstraintViolationFactory +import com.twitter.util.validation.internal.metadata.descriptor.{ + ConstraintDescriptorFactory, + MethodValidationConstraintDescriptor +} +import com.twitter.util.validation.internal.{AnnotationFactory, Types, ValidationContext} +import com.twitter.util.validation.metadata._ +import jakarta.validation._ +import jakarta.validation.constraintvalidation.{SupportedValidationTarget, ValidationTarget} +import jakarta.validation.metadata.ConstraintDescriptor +import jakarta.validation.spi.ValidationProvider +import java.lang.annotation.Annotation +import java.lang.reflect.{Constructor, Executable, InvocationTargetException, Method, Parameter} +import java.util.Collections +import org.hibernate.validator.{HibernateValidator, HibernateValidatorConfiguration} +import org.hibernate.validator.internal.engine.constraintvalidation.ConstraintValidatorManager +import org.hibernate.validator.internal.engine.path.PathImpl +import org.hibernate.validator.internal.engine.{ValidatorFactoryImpl, ValidatorFactoryInspector} +import org.hibernate.validator.internal.metadata.aggregated.{ + CascadingMetaDataBuilder, + ExecutableMetaData +} +import org.hibernate.validator.internal.metadata.descriptor.ConstraintDescriptorImpl +import org.hibernate.validator.internal.metadata.raw.ConstrainedElement.ConstrainedElementKind +import org.hibernate.validator.internal.metadata.raw.{ + ConfigurationSource, + ConstrainedExecutable, + ConstrainedParameter +} +import org.hibernate.validator.internal.properties.javabean.{ + ConstructorCallable, + ExecutableCallable, + MethodCallable +} +import org.json4s.reflect.{Reflector, ScalaType} +import scala.annotation.{tailrec, varargs} +import scala.collection.mutable +import scala.collection.mutable.ListBuffer +import scala.jdk.CollectionConverters._ +import scala.util.control.NonFatal + +object ScalaValidator { + + /** The size of the caffeine cache that is used to store reflection data on a validated case class. */ + private val DefaultDescriptorCacheSize: Long = 128 + + case class Builder private[validation] ( + descriptorCacheSize: Long = DefaultDescriptorCacheSize, + messageInterpolator: Option[MessageInterpolator] = None, + constraintMappings: Set[ConstraintMapping] = Set.empty) { + + /** + * Define the size of the cache that stores annotation data for validated case classes. + */ + def withDescriptorCacheSize(size: Long): ScalaValidator.Builder = + Builder( + descriptorCacheSize = size, + messageInterpolator = this.messageInterpolator, + constraintMappings = this.constraintMappings + ) + + def withMessageInterpolator( + messageInterpolator: MessageInterpolator + ): ScalaValidator.Builder = + Builder( + descriptorCacheSize = this.descriptorCacheSize, + messageInterpolator = Some(messageInterpolator), + constraintMappings = this.constraintMappings + ) + + /** + * Register a Set of [[ConstraintMapping]]. + * + * @note Java users please see the version which takes a [[java.util.Set]] + */ + def withConstraintMappings( + constraintMappings: Set[ConstraintMapping] + ): ScalaValidator.Builder = + Builder( + descriptorCacheSize = this.descriptorCacheSize, + messageInterpolator = this.messageInterpolator, + constraintMappings = constraintMappings + ) + + /** + * Register a Set of [[ConstraintMapping]]. + * + * @note Scala users please see the version which takes a [[Set]] + */ + def withConstraintMappings( + constraintMappings: java.util.Set[ConstraintMapping] + ): ScalaValidator.Builder = + Builder( + descriptorCacheSize = this.descriptorCacheSize, + messageInterpolator = this.messageInterpolator, + constraintMappings = constraintMappings.asScala.toSet + ) + + /** + * Register a [[ConstraintMapping]]. + */ + def withConstraintMapping( + constraintMapping: ConstraintMapping + ): ScalaValidator.Builder = + Builder( + descriptorCacheSize = this.descriptorCacheSize, + messageInterpolator = this.messageInterpolator, + constraintMappings = Set(constraintMapping) + ) + + def validator: ScalaValidator = { + val configuration: HibernateValidatorConfiguration = + Validation + .byProvider[ + HibernateValidatorConfiguration, + ValidationProvider[HibernateValidatorConfiguration]]( + classOf[HibernateValidator] + .asInstanceOf[Class[ValidationProvider[HibernateValidatorConfiguration]]]) + .configure() + + // add user-configured message interpolator + messageInterpolator.map { interpolator => + configuration.messageInterpolator(interpolator) + } + + // add user-configured constraint mappings + constraintMappings.foreach { constraintMapping => + val hibernateConstraintMapping: org.hibernate.validator.cfg.ConstraintMapping = + configuration.createConstraintMapping() + hibernateConstraintMapping + .constraintDefinition(constraintMapping.annotationType.asInstanceOf[Class[Annotation]]) + .includeExistingValidators(constraintMapping.includeExistingValidators) + .validatedBy(constraintMapping.constraintValidator + .asInstanceOf[Class[_ <: ConstraintValidator[Annotation, _]]]) + configuration.addMapping(hibernateConstraintMapping) + } + + val validatorFactory = + configuration.buildValidatorFactory + + new ScalaValidator( + this.descriptorCacheSize, + new ValidatorFactoryInspector(validatorFactory.asInstanceOf[ValidatorFactoryImpl]), + validatorFactory.getValidator + ) + } + } + + def builder: ScalaValidator.Builder = Builder() + + def apply(): ScalaValidator = ScalaValidator.builder.validator +} + +class ScalaValidator private[validation] ( + cacheSize: Long, + validatorFactory: ValidatorFactoryInspector, + val underlying: Validator) + extends ScalaExecutableValidator + with Logging { + + /* Exposed for testing */ + private[validation] val descriptorFactory: DescriptorFactory = + new DescriptorFactory(cacheSize, validatorFactory.constraintHelper) + + private[this] val constraintViolationFactory: ConstraintViolationFactory = + new ConstraintViolationFactory(validatorFactory) + + private[this] val constraintDescriptorFactory: ConstraintDescriptorFactory = + new ConstraintDescriptorFactory(validatorFactory) + + private[this] val constraintValidatorContextFactory: ConstraintValidatorContextFactory = + new ConstraintValidatorContextFactory(validatorFactory) + + private[this] val constraintValidatorManager: ConstraintValidatorManager = + validatorFactory.getConstraintCreationContext.getConstraintValidatorManager + + /** Close resources.*/ + def close(): Unit = { + descriptorFactory.close() + validatorFactory.close() + } + + /** + * Returns the descriptorFactory object describing case class constraints. Descriptors describe constraint + * constraints on a given class and any isCascaded types. + * + * The returned object (and associated objects including CaseClassDescriptors) are immutable. + * + * @param clazz class or interface type evaluated + * @tparam T the clazz type + * + * @return the [[CaseClassDescriptor]] for the specified class + * + * @see [[CaseClassDescriptor]] + */ + def getConstraintsForClass[T](clazz: Class[T]): CaseClassDescriptor[T] = + descriptorFactory.describe[T](clazz) + + /** + * Returns the set of Constraints which validate the given [[Annotation]] class. + * + * For instance if the constraint type is `Class[com.twitter.util.validation.constraints.CountryCode]`, + * the returned set would contain an instance of the `ISO3166CountryCodeConstraintValidator`. + * + * @param annotationType the Class[_] of the [[Annotation]] for which to find all supporting constraint validators. + * + * @return the set of supporting constraint validators for a given [[Annotation]]. + */ + def findConstraintValidators[A <: Annotation]( + annotationType: Class[A] + ): Set[ConstraintValidatorType] = { + // compute from constraint, otherwise compute from registry + val validatedBy = findValidatedBy(annotationType) + if (validatedBy.isEmpty) { + validatorFactory.constraintHelper + .getAllValidatorDescriptors(annotationType).asScala + .map(_.newInstance(validatorFactory.constraintValidatorFactory)).toSet + } else validatedBy + } + + /** + * Checks whether the specified [[Annotation]] is a valid constraint constraint. A constraint + * constraint has to fulfill the following conditions: + * + * - Must be annotated with [[jakarta.validation.Constraint]] + * - Define a `message` parameter + * - Define a `groups` parameter + * - Define a `payload` parameter + * + * @param annotation The [[Annotation]] to test. + * + * @return true if the constraint fulfills the above conditions, false otherwise. + */ + def isConstraintAnnotation(annotation: Annotation): Boolean = + isConstraintAnnotation(annotation.annotationType()) + + /** + * Checks whether the specified [[Annotation]] clazz is a valid constraint constraint. A constraint + * constraint has to fulfill the following conditions: + * + * - Must be annotated with [[jakarta.validation.Constraint]] + * - Define a `message` parameter + * - Define a `groups` parameter + * - Define a `payload` parameter + * + * @param annotationType The [[Annotation]] class to test. + * + * @return true if the constraint fulfills the above conditions, false otherwise. + */ + def isConstraintAnnotation(annotationType: Class[_ <: Annotation]): Boolean = + validatorFactory.constraintHelper.isConstraintAnnotation(annotationType) + + /** + * Validates all constraint constraints on an object. + * + * @param obj the object to validate. + * + * @throws ValidationException - if any constraints produce a violation. + * @throws IllegalArgumentException - if object is null. + */ + @throws[ValidationException] + def verify(obj: Any, groups: Class[_]*): Unit = { + val invalidResult = validate(obj, groups: _*) + if (invalidResult.nonEmpty) + throw constraintViolationFactory.newConstraintViolationException(invalidResult) + } + + /** + * Validates all constraint constraints on an object. + * + * @param obj the object to validate. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type of the object to validate. + * + * @return constraint violations or an empty set if none + * + * @throws IllegalArgumentException - if object is null. + */ + def validate[T]( + obj: T, + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (obj == null) throw new IllegalArgumentException("value must not be null.") + val context = ValidationContext[T](None, None, None, None, PathImpl.createRootPath()) + validate[T]( + maybeDescriptor = None, + context = context, + value = obj, + groups = groups + ) + } + + /** + * Validates all constraint constraints on the `fieldName` field of the given class `beanType` + * if the `fieldName` field value were `value`. + * + * ConstraintViolation objects return `null` for ConstraintViolation.getRootBean() and + * ConstraintViolation.getLeafBean(). + * + * ==Usage== + * Validate value of "" for the "id" field in case class `MyClass` (enabling group "Checks"): + * {{{ + * case class MyCaseClass(@NotEmpty(groups = classOf[Checks]) id) + * validator.validateFieldValue(classOf[MyCaseClass], "id", "", Seq(classOf[Checks])) + * }}} + * + * @param beanType the case class type. + * @param propertyName field to validate. + * @param value field value to validate. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type of the object to validate + * + * @return constraint violations or an empty set if none + * + * @throws IllegalArgumentException - if beanType is null, if fieldName is null, empty or not a valid object property. + */ + def validateValue[T]( + beanType: Class[T], + propertyName: String, + value: Any, + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (beanType == null) throw new IllegalArgumentException("beanType must not be null.") + val descriptor = getConstraintsForClass(beanType) + validateValue[T](descriptor, propertyName, value, groups: _*) + } + + /** + * Validates all constraint constraints on the `fieldName` field of the class described + * by the given [[CaseClassDescriptor]] if the `fieldName` field value were `value`. + * + * @param descriptor the [[CaseClassDescriptor]] of the described class. + * @param propertyName field to validate. + * @param value field value to validate. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type of the object to validate + * + * @return constraint violations or an empty set if none + * + * @throws IllegalArgumentException - if fieldName is null, empty or not a valid object property. + */ + def validateValue[T]( + descriptor: CaseClassDescriptor[T], + propertyName: String, + value: Any, + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (descriptor == null) + throw new IllegalArgumentException("descriptor must not be null.") + if (propertyName == null || propertyName.isEmpty) + throw new IllegalArgumentException("fieldName must not be null or empty.") + + val mangledName = DescriptorFactory.mangleName(propertyName) + descriptor.members.get(mangledName) match { + case Some(propertyDescriptor) => + val context = ValidationContext( + Some(propertyName), + Some(descriptor.clazz), + None, + None, + PathImpl.createRootPath()) + validate( + maybeDescriptor = Some(propertyDescriptor), + context = context, + value = value, + groups = groups + ) + case _ => + throw new IllegalArgumentException(s"$propertyName is not a field of ${descriptor.clazz}.") + } + } + + /** + * Validates all constraint constraints on the `fieldName` field of the given object. + * + * ==Usage== + * Validate the "id" field in case class `MyClass` (enabling the validation group "Checks"): + * {{{ + * case class MyClass(@NotEmpty(groups = classOf[Checks]) id) + * val i = MyClass("") + * validator.validateProperty(i, "id", Seq(classOf[Checks])) + * }}} + * + * @param obj object to validate. + * @param propertyName property to validate (used in error reporting). + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type of the object to validate. + * + * @return constraint violations or an empty set if none. + * + * @throws IllegalArgumentException - if object is null, if fieldName is null, empty or not a valid object property. + */ + def validateProperty[T]( + obj: T, + propertyName: String, + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (obj == null) throw new IllegalArgumentException("obj must not be null.") + if (propertyName == null || propertyName.isEmpty) + throw new IllegalArgumentException("fieldName must not be null or empty.") + + val caseClassDescriptor = getConstraintsForClass(obj.getClass) + + val mangledName = DescriptorFactory.mangleName(propertyName) + caseClassDescriptor.members.get(mangledName) match { + case Some(_) => + val context = ValidationContext( + Some(propertyName), + Some(obj.getClass.asInstanceOf[Class[T]]), + Some(obj), + Some(obj), + PathImpl.createRootPath()) + validate( + maybeDescriptor = Some(caseClassDescriptor), + context = context, + value = obj, + groups = groups + ) + case _ => + throw new IllegalArgumentException( + s"$propertyName is not a field of ${caseClassDescriptor.clazz}.") + } + } + + /** Returns the contract for validating parameters and return values of methods and constructors. */ + def forExecutables: ScalaExecutableValidator = this + + /** + * Returns an instance of the specified type allowing access to provider-specific APIs. + * + * If the Jakarta Bean Validation provider implementation does not support the specified class, ValidationException is thrown. + * + * @param clazz the class of the object to be returned + * @tparam U the type of the object to be returned + * + * @return an instance of the specified class + * + * @throws ValidationException - if the provider does not support the call. + */ + @throws[ValidationException] + def unwrap[U](clazz: Class[U]): U = { + // If this were an implementation that implemented the specification + // interface, this is where users would be able to cast the validator from the + // interface type into our specific implementation type in order to call methods + // only available on the implementation. However, since the specification uses Java + // collection types we do not directly implement and instead expose our feature-compatible + // ScalaValidator directly. We try to remain true to the spirit of the specification + // interface and thus implement unwrap but only to return this implementation. + if (clazz.isAssignableFrom(classOf[ScalaValidator])) { + this.asInstanceOf[U] + } else if (clazz.isAssignableFrom(classOf[ScalaExecutableValidator])) { + this.asInstanceOf[U] + } else { + throw new ValidationException(s"Type ${clazz.getName} not supported for unwrapping.") + } + } + + /* Executable Validator Methods */ + + /** @inheritdoc */ + def validateMethods[T]( + obj: T, + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (obj == null) + throw new IllegalArgumentException("The object to be validated must not be null.") + val caseClassDescriptor = getConstraintsForClass(obj.getClass) + validateMethods[T]( + rootClazz = Some(obj.getClass.asInstanceOf[Class[T]]), + root = Some(obj), + methods = caseClassDescriptor.methods, + obj = obj, + groups = groups + ) + } + + /** @inheritdoc */ + def validateMethod[T]( + obj: T, + method: Method, + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (obj == null) + throw new IllegalArgumentException("The object to be validated must not be null.") + if (method == null) + throw new IllegalArgumentException("The method to be validated must not be null.") + val caseClassDescriptor = getConstraintsForClass(obj.getClass) + caseClassDescriptor.methods.find(_.method.getName == method.getName) match { + case Some(methodDescriptor) => + validateMethods[T]( + rootClazz = Some(obj.getClass.asInstanceOf[Class[T]]), + root = Some(obj), + methods = Array(methodDescriptor), + obj = obj, + groups = groups + ) + case _ => + throw new IllegalArgumentException( + s"${method.getName} is not method of ${caseClassDescriptor.clazz}.") + } + } + + /** @inheritdoc */ + def validateParameters[T]( + obj: T, + method: Method, + parameterValues: Array[Any], + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (obj == null) + throw new IllegalArgumentException("The object to be validated must not be null.") + if (method == null) + throw new IllegalArgumentException("The method to be validated must not be null.") + + val caseClassDescriptor = getConstraintsForClass(obj.getClass) + caseClassDescriptor.methods.find(_.method.getName == method.getName) match { + case Some(methodDescriptor: ExecutableDescriptor) => + validateExecutableParameters[T]( + executableDescriptor = methodDescriptor, + root = Some(obj), + leaf = Some(obj), + parameterValues = parameterValues, + parameterNames = None, + groups = groups + ) + case _ => + throw new IllegalArgumentException( + s"${method.getName} is not method of ${caseClassDescriptor.clazz}.") + } + } + + /** @inheritdoc */ + def validateReturnValue[T]( + obj: T, + method: Method, + returnValue: Any, + groups: Class[_]* + ): Set[ConstraintViolation[T]] = underlying + .forExecutables().validateReturnValue( + obj, + method, + returnValue, + groups: _* + ).asScala.toSet + + /** @inheritdoc */ + def validateConstructorParameters[T]( + constructor: Constructor[T], + parameterValues: Array[Any], + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (constructor == null) + throw new IllegalArgumentException("The executable to be validated must not be null.") + if (parameterValues == null) + throw new IllegalArgumentException("The constructor parameter array cannot not be null.") + + val parameterCount = constructor.getParameterCount + val parameterValuesLength = parameterValues.length + if (parameterCount != parameterValuesLength) + throw new ValidationException( + s"Wrong number of parameters. Method or constructor $constructor expects $parameterCount parameters, but got $parameterValuesLength.") + + validateExecutableParameters[T]( + executableDescriptor = descriptorFactory.describe[T](constructor), + root = None, + leaf = None, + parameterValues = parameterValues, + parameterNames = None, + groups = groups + ) + } + + /** @inheritdoc */ + def validateMethodParameters[T]( + method: Method, + parameterValues: Array[Any], + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (method == null) + throw new IllegalArgumentException("The executable to be validated must not be null.") + if (parameterValues == null) + throw new IllegalArgumentException("The method parameter array cannot not be null.") + + val parameterCount = method.getParameterCount + val parameterValuesLength = parameterValues.length + if (parameterCount != parameterValuesLength) + throw new ValidationException( + s"Wrong number of parameters. Method or constructor $method expects $parameterCount parameters, but got $parameterValuesLength.") + + descriptorFactory.describe(method) match { + case Some(descriptor: ExecutableDescriptor) => + validateExecutableParameters[T]( + executableDescriptor = descriptor, + root = None, + leaf = None, + parameterValues = parameterValues, + parameterNames = None, + groups = groups + ) + case _ => Set.empty[ConstraintViolation[T]] + } + } + + /** @inheritdoc */ + def validateConstructorReturnValue[T]( + constructor: Constructor[T], + createdObject: T, + groups: Class[_]* + ): Set[ConstraintViolation[T]] = + underlying + .forExecutables().validateConstructorReturnValue( + constructor, + createdObject, + groups: _*).asScala.toSet + + /* Private */ + + /** @inheritdoc */ + private[twitter] def validateExecutableParameters[T]( + executable: Executable, + parameterValues: Array[Any], + parameterNames: Array[String], + mixinClazz: Option[Class[_]], + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + if (executable == null) + throw new IllegalArgumentException("The executable to be validated must not be null.") + if (parameterValues == null) + throw new IllegalArgumentException("The constructor parameter array cannot not be null.") + + val parameterCount = executable.getParameterCount + val parameterValuesLength = parameterValues.length + if (parameterCount != parameterValuesLength) + throw new ValidationException( + s"Wrong number of parameters. Method or constructor $executable expects $parameterCount parameters, but got $parameterValuesLength.") + + executable match { + case constructor: Constructor[_] => + validateExecutableParameters[T]( + executableDescriptor = + descriptorFactory.describe[T](constructor.asInstanceOf[Constructor[T]], mixinClazz), + root = None, + leaf = None, + parameterValues = parameterValues, + parameterNames = Some(parameterNames), + groups = groups + ) + case _ => + descriptorFactory.describe(executable.asInstanceOf[Method]) match { + case Some(descriptor: ExecutableDescriptor) => + validateExecutableParameters[T]( + executableDescriptor = descriptor, + root = None, + leaf = None, + parameterValues = parameterValues, + parameterNames = Some(parameterNames), + groups = groups + ) + case _ => Set.empty[ConstraintViolation[T]] + } + } + } + + // note: Prefer while loop over Scala for loop for better performance. The scala for loop + // performance is optimized in 2.13.0 if we enable scalac: https://github.com/scala/bug/issues/1338 + + /** + * Evaluates all known [[ConstraintValidator]] instances supporting the constraints represented by the + * given mapping of [[Annotation]] to constraint attributes placed on a field `fieldName` if + * the `fieldName` field value were `value`. The `fieldName` is used in error reporting (as the + * the resulting property path for any resulting [[ConstraintViolation]]). + * + * E.g., the given mapping could be Map(jakarta.validation.constraint.Size -> Map("min" -> 0, "max" -> 1000)) + * + * ConstraintViolation objects return `null` for ConstraintViolation.getRootBeanClass(), + * ConstraintViolation.getRootBean() and ConstraintViolation.getLeafBean(). + * + * @param constraints the mapping of constraint [[Annotation]] to attributes to evaluate + * @param fieldName field to validate (used in error reporting). + * @param value field value to validate. + * @param groups the list of groups targeted for validation (defaults to Default). + * + * @return constraint violations or an empty set if none + * + * @throws IllegalArgumentException - if fieldName is null, empty or not a valid object property. + * + * @note Scala users should prefer the version which takes a [[Map]] of constraints and + * a [[Seq]] of groups. + */ + @varargs + private[twitter] def validateFieldValue( + constraints: java.util.Map[Class[_ <: Annotation], java.util.Map[String, Any]], + fieldName: String, + value: Any, + groups: Class[_]* + ): java.util.Set[ConstraintViolation[Any]] = { + if (fieldName == null || fieldName.isEmpty) + throw new IllegalArgumentException("fieldName must not be null or empty.") + val annotations = constraints.asScala.map { + case (constraintAnnotationType, attributes) + if isConstraintAnnotation(constraintAnnotationType) => + AnnotationFactory.newInstance(constraintAnnotationType, attributes.asScala.toMap) + } + validateFieldValue(fieldName, annotations.toArray, value, groups.toSeq).asJava + } + + /** + * Evaluates all known [[ConstraintValidator]] instances supporting the constraints represented by the + * given mapping of [[Annotation]] to constraint attributes placed on a field `fieldName` if + * the `fieldName` field value were `value`. The `fieldName` is used in error reporting (as the + * the resulting property path for any resulting [[ConstraintViolation]]). + * + * E.g., the given mapping could be Map(jakarta.validation.constraint.Size -> Map("min" -> 0, "max" -> 1000)) + * + * ConstraintViolation objects return `null` for ConstraintViolation.getRootBeanClass(), + * ConstraintViolation.getRootBean() and ConstraintViolation.getLeafBean(). + * + * @param constraints the mapping of constraint [[Annotation]] to attributes to evaluate + * @param fieldName field to validate (used in error reporting). + * @param value field value to validate. + * @param groups the list of groups targeted for validation (defaults to Default). + * + * @return constraint violations or an empty set if none + * + * @throws IllegalArgumentException - if fieldName is null, empty or not a valid object property. + * + * @note Java users should prefer the version which takes a [[java.util.Map]] of constraints and + * a [[java.util.List]] of groups. + */ + private[twitter] def validateFieldValue( + constraints: Map[Class[_ <: Annotation], Map[String, Any]], + fieldName: String, + value: Any, + groups: Class[_]* + ): Set[ConstraintViolation[Any]] = { + if (fieldName == null || fieldName.isEmpty) + throw new IllegalArgumentException("fieldName must not be null or empty.") + + val annotations = constraints.map { + case (constraintAnnotationType, attributes) => + AnnotationFactory.newInstance(constraintAnnotationType, attributes) + } + + validateFieldValue(fieldName, annotations.toArray, value, groups) + } + + /** Validate the given fieldName with the given constraint constraints, value, and groups. */ + private[this] def validateFieldValue( + fieldName: String, + constraints: Array[Annotation], + value: Any, + groups: Seq[Class[_]] + ): Set[ConstraintViolation[Any]] = + if (constraints.nonEmpty) { + val results = new mutable.ListBuffer[ConstraintViolation[Any]]() + var index = 0 + val length = constraints.length + while (index < length) { + val annotation = constraints(index) + val path = PathImpl.createRootPath + path.addPropertyNode(fieldName) + val clazz = classOf[Any] + val scalaType = Reflector.scalaTypeOf(value.getClass) + val context = ValidationContext(Some(fieldName), Some(clazz), None, None, path) + + results.appendAll( + isValid[Any]( + context = context, + constraint = annotation, + scalaType = scalaType, + value = value, + groups = groups + ) + ) + index += 1 + } + results.toSet + } else Set.empty[ConstraintViolation[Any]] + + private[this] def findValidatedBy[A <: Annotation]( + annotationType: Class[A] + ): Set[ConstraintValidatorType] = { + if (annotationType.isAnnotationPresent(classOf[Constraint])) { + val constraintAnnotation = annotationType.getAnnotation(classOf[Constraint]) + val clazzes: Array[Class[ConstraintValidatorType]] = + constraintAnnotation.validatedBy().map(_.asInstanceOf[Class[ConstraintValidatorType]]) + + clazzes.map { clazz => + val instance = validatorFactory.constraintValidatorFactory.getInstance(clazz) + if (instance == null) + throw new ValidationException( + s"Constraint factory returned null when trying to create instance of $clazz.") + instance + }.toSet + } else Set.empty + } + + // https://beanvalidation.org/2.0/spec/#constraintsdefinitionimplementation-validationimplementation + // From the documentation: + // + // While not mandatory, it is considered a good practice to split the core constraint + // validation from the "not null" constraint validation (for example, an @Email constraint + // should return "true" on a `null` object, i.e. it will not also assert the @NotNull validation). + // A "null" can have multiple meanings but is commonly used to express that a value does + // not make sense, is not available or is simply unknown. Those constraints on the + // value are orthogonal in most cases to other constraints. For example a String, + // if present, must be an email but can be null. Separating both concerns is a good + // practice. + // + // Thus, we "ignore" any value that is null and not annotated with @NotNull. + private[this] def ignorable(value: Any, annotation: Annotation): Boolean = value match { + case null if !ReflectAnnotations.equals[jakarta.validation.constraints.NotNull](annotation) => + true + case _ => false + } + + /** Validate Executable field-level parameters (including cross-field parameters) */ + private[this] def validateExecutableParameters[T]( + executableDescriptor: ExecutableDescriptor, + root: Option[T], + leaf: Option[Any], + parameterValues: Array[Any], + parameterNames: Option[Array[String]], + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = { + if (executableDescriptor.members.size != parameterValues.length) + throw new IllegalArgumentException( + s"Invalid number of arguments for method ${executableDescriptor.executable.getName}.") + val results = new mutable.ListBuffer[ConstraintViolation[T]]() + val parameters: Array[Parameter] = executableDescriptor.executable.getParameters + val parameterNamesList: Array[String] = + parameterNames match { + case Some(names) => + names + case _ => + val parameterNameProviderNames: java.util.List[String] = + validatorFactory.validatorFactoryScopedContext.getParameterNameProvider + .getParameterNames(executableDescriptor.executable) + Array.tabulate(parameterNameProviderNames.size()) { index => + parameterNameProviderNames.get(index) + } + } + + // executable parameter constraints + var index = 0 + val parametersLength = parameterNamesList.length + while (index < parametersLength) { + val parameter = parameters(index) + val parameterValue = parameterValues(index) + // only for property path use to affect error reporting + val parameterName = parameterNamesList(index) + + val parameterPath = + PathImpl.createPathForExecutable(getExecutableMetaData(executableDescriptor.executable)) + parameterPath.addParameterNode(parameterName, index) + + val propertyDescriptor = executableDescriptor.members(parameter.getName) + val validateFieldContext = + ValidationContext[T]( + Some(parameterName), + Some(executableDescriptor.executable.getDeclaringClass.asInstanceOf[Class[T]]), + root, + leaf, + parameterPath) + val parameterViolations = validateField[T]( + validateFieldContext, + propertyDescriptor, + parameterValue, + groups + ) + if (parameterViolations.nonEmpty) results.appendAll(parameterViolations) + + index += 1 + } + results.toSet + + // executable cross-parameter constraints + val crossParameterPath = PathImpl + .createPathForExecutable(getExecutableMetaData(executableDescriptor.executable)) + crossParameterPath.addCrossParameterNode() + val constraints = executableDescriptor.annotations + index = 0 + val constraintsLength = constraints.length + while (index < constraintsLength) { + val constraint = constraints(index) + val scalaType = Reflector.scalaTypeOf(parameterValues.getClass) + + val validators = + findConstraintValidators(constraint.annotationType()) + .filter(parametersValidationTargetFilter) + + val validatorsIterator = validators.iterator + while (validatorsIterator.hasNext) { + val validator = validatorsIterator.next().asInstanceOf[ConstraintValidator[Annotation, _]] + val constraintDescriptor = constraintDescriptorFactory + .newConstraintDescriptor( + name = executableDescriptor.executable.getName, + clazz = scalaType.erasure, + declaringClazz = executableDescriptor.executable.getDeclaringClass, + annotation = constraint, + constrainedElementKind = ConstrainedElementKind.METHOD + ) + if (groupsEnabled(constraintDescriptor, groups)) { + // initialize validator + validator.initialize(constraint) + val validateCrossParameterContext = + ValidationContext[T]( + None, + Some(executableDescriptor.executable.getDeclaringClass.asInstanceOf[Class[T]]), + root, + leaf, + crossParameterPath) + results.appendAll( + isValid( + context = validateCrossParameterContext, + constraint = constraint, + scalaType = scalaType, + value = parameterValues, + constraintDescriptorOption = Some(constraintDescriptor), + validatorOption = Some(validator.asInstanceOf[ConstraintValidator[Annotation, Any]]), + groups = groups + ) + ) + } + } + index += 1 + } + results.toSet + } + + /** Validate method-level constraints, i.e., @MethodValidation annotated methods */ + private[this] def validateMethods[T]( + rootClazz: Option[Class[T]], + root: Option[T], + methods: Array[MethodDescriptor], + obj: T, + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = { + val results = new mutable.ListBuffer[ConstraintViolation[T]]() + if (methods.nonEmpty) { + var index = 0 + val length = methods.length + while (index < length) { + val methodDescriptor = methods(index) + val validateMethodContext = + ValidationContext(None, rootClazz, root, Some(obj), PathImpl.createRootPath()) + results.appendAll( + validateMethod(validateMethodContext, methodDescriptor, obj, groups) + ) + index += 1 + } + } + results.toSet + } + + // BEGIN: Recursive validation methods ----------------------------------------------------------- + + /** Validate field-level constraints */ + private[this] def validateField[T]( + context: ValidationContext[T], + propertyDescriptor: PropertyDescriptor, + fieldValue: Any, + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = { + val constraints: Array[Annotation] = propertyDescriptor.annotations + val results = new mutable.ListBuffer[ConstraintViolation[T]]() + var index = 0 + val length = constraints.length + while (index < length) { + val constraint = constraints(index) + results.appendAll( + isValid[T]( + context, + constraint = constraint, + scalaType = propertyDescriptor.scalaType, + value = fieldValue, + groups = groups + ) + ) + index += 1 + } + + // Cannot cascade a null or None value + fieldValue match { + case null | None => // do nothing + case _ => + if (propertyDescriptor.isCascaded) { + results.appendAll( + validateCascadedProperty[T](context, propertyDescriptor, fieldValue, groups) + ) + } + } + + if (results.isEmpty) Set.empty[ConstraintViolation[T]] + else results.toSet + } + + /** Validate cascaded field-level properties */ + private[this] def validateCascadedProperty[T]( + context: ValidationContext[T], + propertyDescriptor: PropertyDescriptor, + clazzInstance: Any, + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = propertyDescriptor.cascadedScalaType match { + case Some(cascadedScalaType) => + // Update the found value with the current landscape, for example, the retrieved case class + // descriptor may be scala type Foo, but this property is a Seq[Foo] or an Option[Foo] and + // we want to carry that property typing through. + val caseClassDescriptor = + descriptorFactory + .describe[T](cascadedScalaType.erasure.asInstanceOf[Class[T]]) + .copy(scalaType = propertyDescriptor.scalaType) + + val caseClassPath = PathImpl.createCopy(context.path) + caseClassDescriptor.scalaType match { + case argType if argType.isCollection => + val results = mutable.ListBuffer[ConstraintViolation[T]]() + val collectionValue: Iterable[_] = clazzInstance.asInstanceOf[Iterable[_]] + val collectionValueIterator = collectionValue.iterator + var index = 0 + while (collectionValueIterator.hasNext) { + val instanceValue = collectionValueIterator.next() + // apply the index to the parent path, then use this to recompute paths of members and methods + val indexedPropertyPath = { + val indexedPath = PathImpl.createCopyWithoutLeafNode(caseClassPath) + indexedPath.addPropertyNode( + s"${caseClassPath.getLeafNode.asString()}[${index.toString}]") + indexedPath + } + val violations = validate[T]( + maybeDescriptor = Some( + caseClassDescriptor + .copy( + scalaType = caseClassDescriptor.scalaType.typeArgs.head, + members = caseClassDescriptor.members, + methods = caseClassDescriptor.methods + ) + ), + context = context.copy(path = indexedPropertyPath), + value = instanceValue, + groups = groups + ) + if (violations.nonEmpty) results.appendAll(violations) + index += 1 + } + if (results.nonEmpty) results.toSet + else Set.empty[ConstraintViolation[T]] + case argType if argType.isOption => + val optionValue = clazzInstance.asInstanceOf[Option[_]] + validate[T]( + maybeDescriptor = Some( + caseClassDescriptor.copy(scalaType = caseClassDescriptor.scalaType.typeArgs.head)), + context = context.copy(path = caseClassPath), + value = optionValue, + groups = groups + ) + case _ => + validate[T]( + maybeDescriptor = Some(caseClassDescriptor), + context = context.copy(path = caseClassPath), + value = clazzInstance, + groups = groups + ) + } + case _ => Set.empty[ConstraintViolation[T]] + } + + /** Validate @MethodValidation annotated methods */ + private[this] def validateMethod[T]( + context: ValidationContext[T], + methodDescriptor: MethodDescriptor, + clazzInstance: Any, + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = + ReflectAnnotations.findAnnotation[MethodValidation](methodDescriptor.annotations) match { + case Some(annotation) => + // if we're unable to invoke the method, we want this propagate + val constraintDescriptor = new MethodValidationConstraintDescriptor(annotation) + if (groupsEnabled(constraintDescriptor, groups)) { + try { + val result = + methodDescriptor.method.invoke(clazzInstance).asInstanceOf[MethodValidationResult] + val pathWithMethodName = PathImpl.createCopy(context.path) + pathWithMethodName.addPropertyNode(methodDescriptor.method.getName) + parseMethodValidationFailures[T]( + rootClazz = context.rootClazz, + root = context.root, + leaf = Some(clazzInstance), + path = pathWithMethodName, + annotation = annotation.asInstanceOf[MethodValidation], + method = methodDescriptor.method, + result = result, + value = clazzInstance, + constraintDescriptor + ) + } catch { + case e: InvocationTargetException if e.getCause != null => + throw e.getCause + } + } else Set.empty[ConstraintViolation[T]] + case _ => Set.empty[ConstraintViolation[T]] + } + + /** Validate class-level constraints */ + private[this] def validateClazz[T]( + context: ValidationContext[T], + caseClassDescriptor: CaseClassDescriptor[T], + clazzInstance: Any, + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = { + val constraints = caseClassDescriptor.annotations + if (constraints.nonEmpty) { + val results = new mutable.ListBuffer[ConstraintViolation[T]]() + var index = 0 + val length = constraints.length + while (index < length) { + val annotation = constraints(index) + results.appendAll( + isValid[T]( + context = context, + constraint = annotation, + scalaType = caseClassDescriptor.scalaType, + value = clazzInstance, + groups = groups + ) + ) + index += 1 + } + results.toSet + } else Set.empty[ConstraintViolation[T]] + } + + /** Recursively validate full case class */ + @tailrec + private[this] def validate[T]( + maybeDescriptor: Option[Descriptor], + context: ValidationContext[T], + value: Any, + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = maybeDescriptor match { + case Some(descriptor) => + descriptor match { + case p: PropertyDescriptor => + val propertyPath = PathImpl.createCopy(context.path) + context.fieldName.map(propertyPath.addPropertyNode) + // validateField recurses back through validate for cascaded properties + validateField[T](context.copy(path = propertyPath), p, value, groups) + case c: CaseClassDescriptor[_] => + val memberViolations = { + val results = new mutable.ListBuffer[ConstraintViolation[T]]() + if (c.members.nonEmpty) { + val keys: Iterable[String] = c.members.keys + val keyIterator = keys.iterator + while (keyIterator.hasNext) { + val name = keyIterator.next() + val propertyDescriptor = c.members(name) + val propertyPath = PathImpl.createCopy(context.path) + propertyPath.addPropertyNode(DescriptorFactory.unmangleName(name)) + // validateField recurses back through validate here for cascaded properties + val fieldResults = + validateField[T]( + context.copy(fieldName = Some(name), path = propertyPath), + propertyDescriptor, + findFieldValue(value, c.clazz, name), + groups) + if (fieldResults.nonEmpty) results.appendAll(fieldResults) + } + } + results.toSet + } + val methodViolations = { + val results = new ListBuffer[ConstraintViolation[T]]() + if (c.methods.nonEmpty) { + var index = 0 + val length = c.methods.length + while (index < length) { + val methodResults = value match { + case null | None => + Set.empty + case Some(obj) => + validateMethod[T](context.copy(fieldName = None), c.methods(index), obj, groups) + case _ => + validateMethod[T]( + context.copy(fieldName = None), + c.methods(index), + value, + groups) + } + + if (methodResults.nonEmpty) results.appendAll(methodResults) + index += 1 + } + } + results.toSet + } + val clazzViolations = + validateClazz[T](context, c.asInstanceOf[CaseClassDescriptor[T]], value, groups) + + // put them all together + memberViolations ++ methodViolations ++ clazzViolations + } + case _ => + val clazz: Class[T] = value.getClass.asInstanceOf[Class[T]] + if (ReflectTypes.notCaseClass(clazz)) + throw new ValidationException(s"$clazz is not a valid case class.") + val caseClassDescriptor: CaseClassDescriptor[T] = descriptorFactory.describe(clazz) + val context = ValidationContext( + fieldName = None, + rootClazz = Some(clazz), + root = Some(value.asInstanceOf[T]), + leaf = Some(value), + path = PathImpl.createRootPath()) + validate[T](Some(caseClassDescriptor), context, value, groups) + } + + // END: Recursive validation methods ------------------------------------------------------------- + + @tailrec + private[this] def findFieldValue[T]( + obj: Any, + clazz: Class[T], + name: String + ): Any = obj match { + case null | None => + obj + case Some(v) => + findFieldValue(v, clazz, name) + case _ => + try { + val field = clazz.getDeclaredField(name) + field.setAccessible(true) + Option(field.get(obj)).orNull.asInstanceOf[T] + } catch { + case NonFatal(t) => + throw new ValidationException(t) + } + } + + private[this] def isValid[T]( + context: ValidationContext[T], + constraint: Annotation, + scalaType: ScalaType, + value: Any, + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = + isValid(context, constraint, scalaType, value, None, None, groups) + + private[this] def isValid[T]( + context: ValidationContext[T], + constraint: Annotation, + scalaType: ScalaType, + value: Any, + constraintDescriptorOption: Option[ConstraintDescriptorImpl[Annotation]], + validatorOption: Option[ConstraintValidator[Annotation, Any]], + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = value match { + case _: Option[_] => + isValidOption( + context, + constraint, + scalaType, + value, + constraintDescriptorOption, + validatorOption, + groups) + case _ => + val constraintDescriptor = constraintDescriptorOption match { + case Some(descriptor) => + descriptor + case _ => + constraintDescriptorFactory.newConstraintDescriptor( + name = context.fieldName.orNull, + clazz = scalaType.erasure, + declaringClazz = context.rootClazz.getOrElse(scalaType.erasure), + annotation = constraint + ) + } + + if (!ignorable(value, constraint) && groupsEnabled(constraintDescriptor, groups)) { + val refinedScalaType = + Types.refineScalaType(value, scalaType) + + val constraintValidator: ConstraintValidator[Annotation, Any] = validatorOption match { + case Some(validator) => + validator // should already be initialized + case _ => + constraintValidatorManager + .getInitializedValidator[Annotation]( + Types.getJavaType(refinedScalaType), + constraintDescriptor, + validatorFactory.constraintValidatorFactory, + validatorFactory.validatorFactoryScopedContext.getConstraintValidatorInitializationContext + ).asInstanceOf[ConstraintValidator[Annotation, Any]] + } + + if (constraintValidator == null) { + val configuration = + if (context.path.toString.isEmpty) Classes.simpleName(scalaType.erasure) + else context.path.toString + throw new UnexpectedTypeException( + s"No validator could be found for constraint '${constraint.annotationType()}' " + + s"validating type '${scalaType.erasure.getName}'. " + + s"Check configuration for '$configuration'") + } + // create validator context + val constraintValidatorContext: ConstraintValidatorContext = + constraintValidatorContextFactory.newConstraintValidatorContext( + context.path, + constraintDescriptor) + // compute if valid + if (constraintValidator.isValid(value, constraintValidatorContext)) Set.empty + else { + constraintViolationFactory.buildConstraintViolations[T]( + rootClazz = context.rootClazz, + root = context.root, + leaf = context.leaf, + path = context.path, + invalidValue = value, + constraintDescriptor = constraintDescriptor, + constraintValidatorContext = constraintValidatorContext + ) + } + } else Set.empty + } + + private[this] def isValidOption[T]( + context: ValidationContext[T], + constraint: Annotation, + scalaType: ScalaType, + value: Any, + constraintDescriptorOption: Option[ConstraintDescriptorImpl[Annotation]], + validatorOption: Option[ConstraintValidator[Annotation, Any]], + groups: Seq[Class[_]] + ): Set[ConstraintViolation[T]] = value match { + case Some(actualVal) => + isValid[T]( + context, + constraint, + scalaType, + actualVal, + constraintDescriptorOption, + validatorOption, + groups) + case _ => + Set.empty[ConstraintViolation[T]] + } + + private[this] def groupsEnabled( + constraintDescriptor: ConstraintDescriptor[_], + groups: Seq[Class[_]] + ): Boolean = { + val groupsFromAnnotation: java.util.Set[Class[_]] = constraintDescriptor.getGroups + if (groups.isEmpty && groupsFromAnnotation.isEmpty) true + else if (groups.isEmpty && + groupsFromAnnotation.contains(classOf[jakarta.validation.groups.Default])) true + else if (groups.contains(classOf[jakarta.validation.groups.Default]) && + groupsFromAnnotation.isEmpty) true + else groups.exists(groupsFromAnnotation.contains) + } + + private[this] def parametersValidationTargetFilter( + constraintValidator: ConstraintValidator[_, _] + ): Boolean = + ReflectAnnotations + .findAnnotation( + classOf[SupportedValidationTarget], + constraintValidator.getClass.getAnnotations) match { + case Some(annotation: SupportedValidationTarget) => + annotation.value().contains(ValidationTarget.PARAMETERS) + case _ => false + } + + private[this] def getExecutableMetaData[T](executable: Executable): ExecutableMetaData = { + val callable: ExecutableCallable[_] = executable match { + case constructor: Constructor[_] => + new ConstructorCallable(constructor) + case method: Method => + new MethodCallable(method) + } + val builder = new ExecutableMetaData.Builder( + executable.getDeclaringClass, + new ConstrainedExecutable( + ConfigurationSource.ANNOTATION, + callable, + callable.getParameters + .map(p => + new ConstrainedParameter( + ConfigurationSource.ANNOTATION, + callable, + p.getType, + p.getIndex)).asJava, + Collections.emptySet(), + Collections.emptySet(), + Collections.emptySet(), + CascadingMetaDataBuilder.nonCascading + ), + validatorFactory.getConstraintCreationContext, + validatorFactory.getExecutableHelper, + validatorFactory.getExecutableParameterNameProvider, + validatorFactory.getMethodValidationConfiguration + ) + + builder.build() + } + + private[validation] def parseMethodValidationFailures[T]( + rootClazz: Option[Class[T]], + root: Option[T], + leaf: Option[Any], + path: PathImpl, + annotation: MethodValidation, + method: Method, + result: MethodValidationResult, + value: Any, + constraintDescriptor: ConstraintDescriptor[_] + ): Set[ConstraintViolation[T]] = { + def methodValidationConstraintViolation( + validationPath: PathImpl, + message: String, + payload: Payload + ): ConstraintViolation[T] = { + constraintViolationFactory.newConstraintViolation[T]( + messageTemplate = annotation.message(), + interpolatedMessage = message, + path = validationPath, + invalidValue = value.asInstanceOf[T], + rootClazz = rootClazz.orNull, + root = root.getOrElse(null.asInstanceOf[T]), + leaf = leaf.orNull, + constraintDescriptor = constraintDescriptor, + payload = payload + ) + } + + result match { + case invalid: MethodValidationResult.Invalid => + val results = mutable.HashSet[ConstraintViolation[T]]() + val annotationFields: Array[String] = annotation.fields.filter(_.nonEmpty) + if (annotationFields.nonEmpty) { + var index = 0 + val length = annotationFields.length + while (index < length) { + val fieldName = annotationFields(index) + val parameterPath = PathImpl.createCopy(path) + parameterPath.addParameterNode(fieldName, index) + results.add( + methodValidationConstraintViolation( + parameterPath, + invalid.message, + invalid.payload.orNull) + ) + index += 1 + } + if (results.nonEmpty) results.toSet + else Set.empty[ConstraintViolation[T]] + } else { + Set(methodValidationConstraintViolation(path, invalid.message, invalid.payload.orNull)) + } + case _ => Set.empty[ConstraintViolation[T]] + } + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD new file mode 100644 index 0000000000..d2a3af5b5b --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD @@ -0,0 +1,17 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-cfg", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-core:scala", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + "util/util-slf4j-api/src/main/scala", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/cfg/ConstraintMapping.scala b/util-validator/src/main/scala/com/twitter/util/validation/cfg/ConstraintMapping.scala new file mode 100644 index 0000000000..ae8bb8eac6 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/cfg/ConstraintMapping.scala @@ -0,0 +1,41 @@ +package com.twitter.util.validation.cfg + +import jakarta.validation.ConstraintValidator +import java.lang.annotation.Annotation + +/** + * Simple configuration class for defining constraint mappings for ScalaValidator configuration. + * ==Usage== + * {{{ + * val customConstraintMapping: ConstraintMapping = + * ConstraintMapping(classOf[Annotation], classOf[ConstraintValidator]) + * + * val validator: ScalaValidator = + * ScalaValidator.builder + * .withConstraintMapping(customConstraintMapping) + * .validator + * }}} + * + * or multiple mappings + * + * {{{ + * val customConstraintMapping1: ConstraintMapping = ??? + * val customConstraintMapping2: ConstraintMapping = ??? + * + * val validator: ScalaValidator = + * ScalaValidator.builder + * .withConstraintMappings(Set(customConstraintMapping1, customConstraintMapping2)) + * .validator + * }}} + * + * @param annotationType the `Class[Annotation]` of the constraint annotation. + * @param constraintValidator the implementing ConstraintValidator class for the given constraint annotation. + * @param includeExistingValidators if this is an additional validator for the given constraint annotation type + * or if this should replace all existing validators for the given constraint annotation + * type. Default is true (additional). + * @note adding multiple constraint mappings for the same annotation type will result in a ValidationException being thrown. + */ +case class ConstraintMapping( + annotationType: Class[_ <: Annotation], + constraintValidator: Class[_ <: ConstraintValidator[_ <: Annotation, _]], + includeExistingValidators: Boolean = true) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD new file mode 100644 index 0000000000..0662ef0e36 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD @@ -0,0 +1,16 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-constraintvalidation", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-core:scala", + "util/util-slf4j-api/src/main/scala", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/TwitterConstraintValidatorContext.scala b/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/TwitterConstraintValidatorContext.scala new file mode 100644 index 0000000000..4894abd34f --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/TwitterConstraintValidatorContext.scala @@ -0,0 +1,221 @@ +package com.twitter.util.validation.constraintvalidation + +import jakarta.validation.{ConstraintValidatorContext, Payload, ValidationException} +import org.hibernate.validator.constraintvalidation.HibernateConstraintValidatorContext + +object TwitterConstraintValidatorContext { + + def builder: TwitterConstraintValidatorContext.Builder = + new Builder(None, None, Map.empty[String, AnyRef], Map.empty[String, AnyRef]) + + /** + * Simple utility to build a non-default [[jakarta.validation.ConstraintViolation]] + * from a [[jakarta.validation.ConstraintValidatorContext]]. + */ + class Builder private[validation] ( + messageTemplate: Option[String], + payload: Option[Payload], + messageParameters: Map[String, AnyRef], + expressionVariables: Map[String, AnyRef]) { + + /** + * Specify a new un-interpolated constraint message. + * + * @param messageTemplate an un-interpolated constraint message. + * + * @return a [[TwitterConstraintValidatorContext.Builder]] to allow for method chaining. + */ + def withMessageTemplate(messageTemplate: String): Builder = + new Builder( + messageTemplate = Some(messageTemplate), + payload = this.payload, + messageParameters = this.messageParameters, + expressionVariables = this.expressionVariables) + + /** + * Allows to set an object that may further describe the violation. The user is responsible to + * ensure that this payload is serializable in case the [[jakarta.validation.ConstraintViolation]] + * has to be serialized. + * + * @param payload payload an object representing additional information about the violation + * + * @return a [[TwitterConstraintValidatorContext.Builder]] to allow for method chaining. + */ + def withDynamicPayload(payload: Payload): Builder = + new Builder( + messageTemplate = this.messageTemplate, + payload = Some(payload), + messageParameters = this.messageParameters, + expressionVariables = this.expressionVariables) + + /** + * Allows setting of an additional named parameter which can be interpolated in the constraint violation message. The + * variable will be available for interpolation for all constraint violations generated for this constraint. + * This includes the default one as well as all violations created by this TwitterConstraintValidatorContext. + * To create multiple constraint violations with different variable values, this method can be called + * between successive calls to [[TwitterConstraintValidatorContext#addConstraintViolation]]. + * + * @param name the name under which to bind the parameter, cannot be `null`. + * @param value the value to be bound to the specified name + * + * @return a [[TwitterConstraintValidatorContext.Builder]] to allow for method chaining. + */ + def addMessageParameter(name: String, value: AnyRef): Builder = + new Builder( + messageTemplate = this.messageTemplate, + payload = this.payload, + messageParameters = this.messageParameters + (name -> value), + expressionVariables = this.expressionVariables + ) + + /** + * Allows setting of an additional expression variable which will be available as an EL variable during interpolation. The + * variable will be available for interpolation for all constraint violations generated for this constraint. + * This includes the default one as well as all violations created by this TwitterConstraintValidatorContext. + * To create multiple constraint violations with different variable values, this method can be called + * between successive calls to [[TwitterConstraintValidatorContext#addConstraintViolation]]. + * + * @param name the name under which to bind the expression variable, cannot be `null`. + * @param value the value to be bound to the specified name + * + * @return a [[TwitterConstraintValidatorContext.Builder]] to allow for method chaining. + */ + def addExpressionVariable(name: String, value: AnyRef): Builder = + new Builder( + messageTemplate = this.messageTemplate, + payload = this.payload, + messageParameters = this.messageParameters, + expressionVariables = this.expressionVariables + (name -> value) + ) + + /** + * Replaces the default constraint violation report with one built via this method using the + * given [[TwitterConstraintValidatorContext]]. + * + * @param context the [[TwitterConstraintValidatorContext]] to use in building a [[jakarta.validation.ConstraintViolation]] + * + * @throws ValidationException - if the given [[TwitterConstraintValidatorContext]] cannot be unwrapped as expected. + */ + @throws[ValidationException] + def addConstraintViolation(context: ConstraintValidatorContext): Unit = { + if (context != null) { + val hibernateContext: HibernateConstraintValidatorContext = + context.unwrap(classOf[HibernateConstraintValidatorContext]) + // custom payload + payload.foreach(hibernateContext.withDynamicPayload) + // set message parameters + messageParameters.foreach { + case (name, value) => + hibernateContext.addMessageParameter(name, value) + } + // set expression variables + expressionVariables.foreach { + case (name, value) => + hibernateContext.addExpressionVariable(name, value) + } + // custom message + messageTemplate.foreach { template => + hibernateContext.disableDefaultConstraintViolation() + + if (expressionVariables.nonEmpty) { + hibernateContext + .buildConstraintViolationWithTemplate(template) + .enableExpressionLanguage + .addConstraintViolation() + } else { + hibernateContext + .buildConstraintViolationWithTemplate(template) + .addConstraintViolation() + } + } + } + } + } + + /** + * Specify a new un-interpolated constraint message. + * + * @param messageTemplate an un-interpolated constraint message. + * + * @return a [[TwitterConstraintValidatorContext.Builder]] to allow for method chaining. + */ + def withMessageTemplate(messageTemplate: String): TwitterConstraintValidatorContext.Builder = + builder.withMessageTemplate(messageTemplate) + + /** + * Allows to set an object that may further describe the violation. The user is responsible to + * ensure that this payload is serializable in case the [[jakarta.validation.ConstraintViolation]] + * has to be serialized. + * + * @param payload payload an object representing additional information about the violation + * + * @return a [[TwitterConstraintValidatorContext.Builder]] to allow for method chaining. + */ + def withDynamicPayload(payload: Payload): TwitterConstraintValidatorContext.Builder = + builder.withDynamicPayload(payload) + + /** + * Allows setting of an additional named parameter which can be interpolated in the constraint violation message. The + * variable will be available for interpolation for all constraint violations generated for this constraint. + * This includes the default one as well as all violations created by this TwitterConstraintValidatorContext. + * To create multiple constraint violations with different variable values, this method can be called + * between successive calls to [[TwitterConstraintValidatorContext#addConstraintViolation]]. E.g., + * within a ConstraintValidator instance: + * {{{ + * def isValid(value: String, constraintValidatorContext: TwitterConstraintValidatorContext): Boolean = { + * + * TwitterConstraintValidatorContext + * .addMessageParameter("foo", "bar") + * .withMessageTemplate("{foo}") + * .addConstraintViolation(context) + * + * TwitterConstraintValidatorContext + * .addMessageParameter("foo", "snafu") + * .withMessageTemplate("{foo}") + * .addConstraintViolation(context) + * + * false + * }}} + * + * @param name the name under which to bind the parameter, cannot be `null`. + * @param value the value to be bound to the specified name + * + * @return a [[TwitterConstraintValidatorContext.Builder]] to allow for method chaining. + */ + def addMessageParameter(name: String, value: AnyRef): TwitterConstraintValidatorContext.Builder = + builder.addMessageParameter(name, value) + + /** + * Allows setting of an additional expression variable which will be available as an EL variable during interpolation. The + * variable will be available for interpolation for all constraint violations generated for this constraint. + * This includes the default one as well as all violations created by this TwitterConstraintValidatorContext. + * To create multiple constraint violations with different variable values, this method can be called + * between successive calls to [[TwitterConstraintValidatorContext#addConstraintViolation]]. E.g., + * within a ConstraintValidator instance: + * {{{ + * def isValid(value: String, constraintValidatorContext: TwitterConstraintValidatorContext): Boolean = { + * + * TwitterConstraintValidatorContext + * .addExpressionVariable("foo", "bar") + * .withMessageTemplate("${foo}") + * .addConstraintViolation(context) + * + * TwitterConstraintValidatorContext + * .addExpressionVariable("foo", "snafu") + * .withMessageTemplate("${foo}") + * .addConstraintViolation(context) + * + * false + * }}} + * + * @param name the name under which to bind the expression variable, cannot be `null`. + * @param value the value to be bound to the specified name + * + * @return a [[TwitterConstraintValidatorContext.Builder]] to allow for method chaining. + */ + def addExpressionVariable( + name: String, + value: AnyRef + ): TwitterConstraintValidatorContext.Builder = + builder.addExpressionVariable(name, value) +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD new file mode 100644 index 0000000000..9cf37ac72a --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-conversions", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-validator/src/main/scala/com/twitter/util/validation/engine", + ], + exports = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/conversions/ConstraintViolationOps.scala b/util-validator/src/main/scala/com/twitter/util/validation/conversions/ConstraintViolationOps.scala new file mode 100644 index 0000000000..00c71e8e25 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/conversions/ConstraintViolationOps.scala @@ -0,0 +1,22 @@ +package com.twitter.util.validation.conversions + +import com.twitter.util.validation.engine.ConstraintViolationHelper +import jakarta.validation.{ConstraintViolation, Payload} + +object ConstraintViolationOps { + + implicit class RichConstraintViolation[T](val self: ConstraintViolation[T]) extends AnyVal { + + /** + * Returns a String created by concatenating the [[ConstraintViolation#getPropertyPath]] and + * the [[ConstraintViolation#getMessage]] separated by a colon (:). E.g. if a given violation + * has a `getPropertyPath.toString` value of "foo" and a `getMessage` value of "does not exist", + * this would return "foo: does not exist". + */ + def getMessageWithPath: String = ConstraintViolationHelper.messageWithPath(self) + + /** Lookup the dynamic payload of the given class type */ + def getDynamicPayload[P <: Payload](clazz: Class[P]): Option[P] = + ConstraintViolationHelper.dynamicPayload[P](self, clazz) + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/conversions/PathOps.scala b/util-validator/src/main/scala/com/twitter/util/validation/conversions/PathOps.scala new file mode 100644 index 0000000000..534c67c1a8 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/conversions/PathOps.scala @@ -0,0 +1,20 @@ +package com.twitter.util.validation.conversions + +import jakarta.validation.Path +import org.hibernate.validator.internal.engine.path.PathImpl + +object PathOps { + + implicit class RichPath(val self: Path) extends AnyVal { + def getLeafNode: Path.Node = self match { + case pathImpl: PathImpl => + pathImpl.getLeafNode + case _ => + var node = self.iterator().next() + while (self.iterator().hasNext) { + node = self.iterator().next() + } + node + } + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD new file mode 100644 index 0000000000..98f85d2879 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-engine", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-core:scala", + "util/util-slf4j-api/src/main/scala", + "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/engine", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/engine/ConstraintViolationHelper.scala b/util-validator/src/main/scala/com/twitter/util/validation/engine/ConstraintViolationHelper.scala new file mode 100644 index 0000000000..55d4f68a9e --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/engine/ConstraintViolationHelper.scala @@ -0,0 +1,41 @@ +package com.twitter.util.validation.engine + +import com.twitter.util.{Return, Try} +import com.twitter.util.validation.internal.engine.ConstraintViolationFactory +import jakarta.validation.{ConstraintViolation, Payload} +import org.hibernate.validator.engine.HibernateConstraintViolation + +/** Helper for [[jakarta.validation.ConstraintViolation]] */ +object ConstraintViolationHelper { + + /** + * Sorts the given [[Set]] by natural order of String created by concatenating the + * [[ConstraintViolation#getPropertyPath]] and [[ConstraintViolation#getMessage]] separated + * by a colon (:). + * + * @see [[messageWithPath]] + */ + def sortViolations[T](violations: Set[ConstraintViolation[T]]): Seq[ConstraintViolation[T]] = + ConstraintViolationFactory.sortSet[T](violations) + + /** + * Returns a String created by concatenating the [[ConstraintViolation#getPropertyPath]] and + * the [[ConstraintViolation#getMessage]] separated by a colon (:). E.g. if a given violation + * has a `getPropertyPath.toString` value of "foo" and a `getMessage` value of "does not exist", + * this would return "foo: does not exist". + */ + def messageWithPath(violation: ConstraintViolation[_]): String = + ConstraintViolationFactory.getMessage(violation) + + /** Lookup the dynamic payload of the given class type */ + def dynamicPayload[P <: Payload]( + violation: ConstraintViolation[_], + clazz: Class[P] + ): Option[P] = { + Try(violation.unwrap(classOf[HibernateConstraintViolation[_]])) match { + case Return(hibernateConstraintViolation) => + Option(hibernateConstraintViolation.getDynamicPayload[P](clazz)) + case _ => None + } + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/engine/MethodValidationResult.scala b/util-validator/src/main/scala/com/twitter/util/validation/engine/MethodValidationResult.scala new file mode 100644 index 0000000000..9fd4ac8033 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/engine/MethodValidationResult.scala @@ -0,0 +1,88 @@ +package com.twitter.util.validation.engine + +import com.twitter.util.logging.Logger +import com.twitter.util.{Return, Throw, Try} +import jakarta.validation.Payload + +sealed trait MethodValidationResult { + def payload: Option[Payload] + def isValid: Boolean +} + +object MethodValidationResult { + private[this] val logger: Logger = Logger(MethodValidationResult.getClass) + + case object EmptyPayload extends Payload + + case object Valid extends MethodValidationResult { + override final val payload: Option[Payload] = None + override final val isValid: Boolean = true + } + + case class Invalid( + message: String, + payload: Option[Payload] = None) + extends MethodValidationResult { + override final val isValid: Boolean = false + } + + /** + * Utility for evaluating a condition in order to return a [[MethodValidationResult]]. Returns + * [[MethodValidationResult.Valid]] when the condition is `true`, otherwise if the condition + * evaluates to `false` or throws an exception a [[MethodValidationResult.Invalid]] will be returned. + * In the case of an exception, the `exception.getMessage` is used in place of the given message. + * + * @note This will not allow a non-fatal exception to escape. Instead a [[MethodValidationResult.Invalid]] + * will be returned when a non-fatal exception is encountered when evaluating `condition`. As + * this equates failure to execute the condition function to a return of `false`. + * @param condition function to evaluate for validation. + * @param message function to evaluate for a message when the given condition is `false`. + * @param payload [[Payload]] to use for when the given condition is `false`. + * + * @return a [[MethodValidationResult.Valid]] when the condition is `true` otherwise a [[MethodValidationResult.Invalid]]. + */ + def validIfTrue( + condition: => Boolean, + message: => String, + payload: Option[Payload] = None, + ): MethodValidationResult = Try(condition) match { + case Return(result) if result => + Valid + case Return(result) if !result => + Invalid(message, payload) + case Throw(e) => + logger.warn(e.getMessage, e) + Invalid(e.getMessage, payload) + } + + /** + * Utility for evaluating the negation of a condition in order to return a [[MethodValidationResult]]. + * Returns [[MethodValidationResult.Valid]] when the condition is `false`, otherwise if the condition + * evaluates to `true` or throws an exception a [[MethodValidationResult.Invalid]] will be returned. + * In the case of an exception, the `exception.getMessage` is used in place of the given message. + * + * @note This will not allow a non-fatal exception to escape. Instead a [[MethodValidationResult.Valid]] + * will be returned when a non-fatal exception is encountered when evaluating `condition`. As + * this equates failure to execute the condition to a return of `false`. + * @param condition function to evaluate for validation. + * @param message function to evaluate for a message when the given condition is `true`. + * @param payload [[Payload]] to use for when the given condition is `true`. + * + * @return a [[MethodValidationResult.Valid]] when the condition is `false` or when the condition evaluation + * throws a NonFatal exception otherwise a [[MethodValidationResult.Invalid]]. + */ + def validIfFalse( + condition: => Boolean, + message: => String, + payload: Option[Payload] = None + ): MethodValidationResult = + Try(condition) match { + case Return(result) if !result => + Valid + case Return(result) if result => + Invalid(message, payload) + case Throw(e) => + logger.warn(e.getMessage, e) + Valid + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD new file mode 100644 index 0000000000..0930df7bae --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD @@ -0,0 +1,18 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-executable", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "util/util-core:scala", + "util/util-slf4j-api/src/main/scala", + ], + exports = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/executable/ScalaExecutableValidator.scala b/util-validator/src/main/scala/com/twitter/util/validation/executable/ScalaExecutableValidator.scala new file mode 100644 index 0000000000..5fd113983c --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/executable/ScalaExecutableValidator.scala @@ -0,0 +1,178 @@ +package com.twitter.util.validation.executable + +import jakarta.validation.{ConstraintViolation, ValidationException} +import java.lang.reflect.{Constructor, Executable, Method} + +/** + * Scala version of the Bean Specification [[jakarta.validation.executable.ExecutableValidator]]. + * + * Validates parameters and return values of methods and constructors. Implementations of this interface must be thread-safe. + */ +trait ScalaExecutableValidator { + + /** + * Validates all methods annotated with `@MethodValidation` of the given object. + * + * @param obj the object on which the methods to validate are invoked. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type hosting the methods to validate. + * + * @return a set with the constraint violations caused by this validation; will be empty if no error occurs, but never null. + * + * @throws IllegalArgumentException - if null is passed for the object to validate. + * @throws ValidationException - if a non recoverable error happens during the validation process. + */ + @throws[ValidationException] + def validateMethods[T]( + obj: T, + groups: Class[_]* + ): Set[ConstraintViolation[T]] + + /** + * Validates the given method annotated with `@MethodValidation` of the given object. + * + * @param obj the object on which the method to validate is invoked. + * @param method the `@MethodValidation` annotated method to invoke for validation. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type hosting the method to validate. + * + * @return a set with the constraint violations caused by this validation; will be empty if no error occurs, but never null. + * + * @throws IllegalArgumentException - if null is passed for the object to validate or if the given method is not a method of the object class type. + * @throws ValidationException - if a non recoverable error happens during the validation process. + */ + @throws[ValidationException] + def validateMethod[T]( + obj: T, + method: Method, + groups: Class[_]* + ): Set[ConstraintViolation[T]] + + /** + * Validates all constraints placed on the parameters of the given method. + * + * @param obj the object on which the method to validate is invoked. + * @param method the method for which the parameter constraints is validated. + * @param parameterValues the values provided by the caller for the given method's parameters. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type hosting the method to validate. + * + * @return a set with the constraint violations caused by this validation; will be empty if no error occurs, but never null. + * + * @throws IllegalArgumentException - if null is passed for any of the parameters or if parameters don't match with each other. + * @throws ValidationException - if a non recoverable error happens during the validation process. + */ + @throws[ValidationException] + def validateParameters[T]( + obj: T, + method: Method, + parameterValues: Array[Any], + groups: Class[_]* + ): Set[ConstraintViolation[T]] + + /** + * Validates all return value constraints of the given method. + * + * @param obj the object on which the method to validate is invoked. + * @param method the method for which the return value constraints is validated. + * @param returnValue the value returned by the given method. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type hosting the method to validate. + * + * @return a set with the constraint violations caused by this validation; will be empty if no error occurs, but never null. + * + * @throws IllegalArgumentException - if null is passed for any of the parameters or if parameters don't match with each other. + * @throws ValidationException - if a non recoverable error happens during the validation process. + */ + @throws[ValidationException] + def validateReturnValue[T]( + obj: T, + method: Method, + returnValue: Any, + groups: Class[_]* + ): Set[ConstraintViolation[T]] + + /** + * Validates all constraints placed on the parameters of the given constructor. + * + * @param constructor the constructor for which the parameter constraints is validated. + * @param parameterValues the values provided by the caller for the given constructor's parameters. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type hosting the constructor to validate. + * + * @return a set with the constraint violations caused by this validation; Will be empty if no error occurs, but never null. + * + * @throws IllegalArgumentException - if null is passed for any of the parameters or if parameters don't match with each other. + * @throws ValidationException - if a non recoverable error happens during the validation process. + */ + @throws[ValidationException] + def validateConstructorParameters[T]( + constructor: Constructor[T], + parameterValues: Array[Any], + groups: Class[_]* + ): Set[ConstraintViolation[T]] + + /** + * Validates all constraints placed on the parameters of the given method. + * + * @param method the method for which the parameter constraints is validated. + * @param parameterValues the values provided by the caller for the given method's parameters. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type defining the method to validate. + * + * @return a set with the constraint violations caused by this validation; Will be empty if no error occurs, but never null. + * + * @throws IllegalArgumentException - if null is passed for any of the parameters or if parameters don't match with each other. + * @throws ValidationException - if a non recoverable error happens during the validation process. + */ + @throws[ValidationException] + def validateMethodParameters[T]( + method: Method, + parameterValues: Array[Any], + groups: Class[_]* + ): Set[ConstraintViolation[T]] + + /** + * Validates all return value constraints of the given constructor. + * + * @param constructor the constructor for which the return value constraints is validated. + * @param createdObject the object instantiated by the given method. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type hosting the constructor to validate. + * + * @return a set with the constraint violations caused by this validation; will be empty, if no error occurs, but never null. + * + * @throws IllegalArgumentException - if null is passed for any of the parameters or if parameters don't match with each other. + * @throws ValidationException - if a non recoverable error happens during the validation process. + */ + @throws[ValidationException] + def validateConstructorReturnValue[T]( + constructor: Constructor[T], + createdObject: T, + groups: Class[_]* + ): Set[ConstraintViolation[T]] + + /** + * Validates all constraints placed on the parameters of the given executable. + * + * @param executable the executable for which the parameter constraints is validated. + * @param parameterValues the values provided by the caller for the given executable's parameters. + * @param parameterNames the parameter names to use for error reporting. + * @param mixinClazz the optional mix-in class to also consider for annotation processing. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type defining the executable to validate. + * + * @return a set with the constraint violations caused by this validation; Will be empty if no error occurs, but never null. + * + * @throws IllegalArgumentException - if null is passed for any of the parameters or if parameters don't match with each other. + * @throws ValidationException - if a non recoverable error happens during the validation process. + */ + @throws[ValidationException] + private[twitter] def validateExecutableParameters[T]( + executable: Executable, + parameterValues: Array[Any], + parameterNames: Array[String], + mixinClazz: Option[Class[_]], + groups: Class[_]* + ): Set[ConstraintViolation[T]] +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/AnnotationFactory.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/AnnotationFactory.scala new file mode 100644 index 0000000000..c84eff1458 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/AnnotationFactory.scala @@ -0,0 +1,38 @@ +package com.twitter.util.validation.internal + +import java.lang.annotation.Annotation +import java.lang.reflect.Method +import java.util +import sun.reflect.annotation.AnnotationParser + +private[validation] object AnnotationFactory { + + /** + * Creates a new Annotation (AnnotationProxy) instance of + * the given annotation type with the custom values applied. + * + * @see [[AnnotationParser.annotationForMap]] + */ + def newInstance[A <: Annotation]( + annotationType: Class[A], + customValues: Map[String, Any] = Map.empty + ): A = { + val attributes: util.Map[String, Object] = new util.HashMap[String, Object]() + // Extract default values from annotation + val methods: Array[Method] = annotationType.getDeclaredMethods + var index = 0 + val length = methods.length + while (index < length) { + val method = methods(index) + attributes.put(method.getName, method.getDefaultValue) + index += 1 + } + // Populate custom values + val keysIterator = customValues.keysIterator + while (keysIterator.hasNext) { + val key = keysIterator.next() + attributes.put(key, customValues(key).asInstanceOf[AnyRef]) + } + AnnotationParser.annotationForMap(annotationType, attributes).asInstanceOf[A] + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD new file mode 100644 index 0000000000..ff39c6a888 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-internal", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "3rdparty/jvm/org/json4s:json4s-core", + "util/util-core:scala", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + "util/util-slf4j-api/src/main/scala", + "util/util-validator/src/main/scala/com/twitter/util/validation/metadata", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/Types.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/Types.scala new file mode 100644 index 0000000000..394dd4a179 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/Types.scala @@ -0,0 +1,71 @@ +package com.twitter.util.validation.internal + +import org.json4s.reflect.{Reflector, ScalaType} + +private[validation] object Types { + + /** For certain scala types, return the scala type of the first type arg */ + def getContainedScalaType( + scalaType: ScalaType + ): Option[ScalaType] = scalaType match { + case argType if argType.isMap || argType.isMutableMap => + // Maps not supported + None + case argType if argType.isCollection || argType.isOption => + // handles Option and Iterable collections + if (argType.typeArgs.isEmpty) Some(Reflector.scalaTypeOf(classOf[Object])) + else Some(argType.typeArgs.head) + case _ => + Some(scalaType) + } + + // if the case class is generically typed, the originally parsed scala type + // may be an object thus we try to refine it based on the given value + def refineScalaType(value: Any, scalaType: ScalaType): ScalaType = { + if (scalaType.erasure == classOf[Object] && value != null) { + if (classOf[Option[_]].isAssignableFrom(value.getClass)) + Reflector.scalaTypeOf(classOf[Option[_]]) + else + Reflector.scalaTypeOf(value.getClass) + } else if (scalaType.isOption) { + if (scalaType.typeArgs.nonEmpty) scalaType.typeArgs.head + else Reflector.scalaTypeOf(classOf[Object]) + } else scalaType + } + + /** Translate a Scala type to a corresponding Java class */ + def getJavaType(scalaType: ScalaType): Class[_] = scalaType match { + case argType if argType.isOption => + if (argType.typeArgs.nonEmpty) argType.typeArgs.head.erasure + else classOf[Object] + case argType + if argType.isPrimitive && argType.simpleName == java.lang.Byte.TYPE.getSimpleName => + classOf[java.lang.Byte] + case argType + if argType.isPrimitive && argType.simpleName == java.lang.Short.TYPE.getSimpleName => + classOf[java.lang.Short] + case argType + if argType.isPrimitive && argType.simpleName == java.lang.Character.TYPE.getSimpleName => + classOf[java.lang.Character] + case argType + if argType.isPrimitive && argType.simpleName == java.lang.Integer.TYPE.getSimpleName => + classOf[java.lang.Integer] + case argType + if argType.isPrimitive && argType.simpleName == java.lang.Long.TYPE.getSimpleName => + classOf[java.lang.Long] + case argType + if argType.isPrimitive && argType.simpleName == java.lang.Float.TYPE.getSimpleName => + classOf[java.lang.Float] + case argType + if argType.isPrimitive && argType.simpleName == java.lang.Double.TYPE.getSimpleName => + classOf[java.lang.Double] + case argType + if argType.isPrimitive && argType.simpleName == java.lang.Boolean.TYPE.getSimpleName => + classOf[java.lang.Boolean] + case argType + if argType.isPrimitive && argType.simpleName == java.lang.Void.TYPE.getSimpleName => + classOf[java.lang.Void] + case argType => + argType.erasure + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/ValidationContext.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/ValidationContext.scala new file mode 100644 index 0000000000..f730837c08 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/ValidationContext.scala @@ -0,0 +1,12 @@ +package com.twitter.util.validation.internal + +import org.hibernate.validator.internal.engine.path.PathImpl + +/** Utility class to carry necessary context for validation */ +private[validation] case class ValidationContext[T]( + fieldName: Option[String], + rootClazz: Option[Class[T]], + root: Option[T], + leaf: Option[Any], + path: PathImpl, + isFailFast: Boolean = false) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD new file mode 100644 index 0000000000..1479d2195f --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD @@ -0,0 +1,15 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-internal-constraintvalidation", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-validator/src/main/scala/org/hibernate/validator/internal/engine", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/ConstraintValidatorContextFactory.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/ConstraintValidatorContextFactory.scala new file mode 100644 index 0000000000..9227b6fd4a --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/ConstraintValidatorContextFactory.scala @@ -0,0 +1,26 @@ +package com.twitter.util.validation.internal.constraintvalidation + +import jakarta.validation.ConstraintValidatorContext +import jakarta.validation.metadata.ConstraintDescriptor +import org.hibernate.validator.internal.engine.ValidatorFactoryInspector +import org.hibernate.validator.internal.engine.constraintvalidation.{ + ConstraintValidatorContextImpl => HibernateConstraintValidatorContextImpl +} +import org.hibernate.validator.internal.engine.path.PathImpl + +private[validation] class ConstraintValidatorContextFactory( + validatorFactory: ValidatorFactoryInspector) { + + def newConstraintValidatorContext( + path: PathImpl, + constraintDescriptor: ConstraintDescriptor[_] + ): ConstraintValidatorContext = + new HibernateConstraintValidatorContextImpl( + validatorFactory.validatorFactoryScopedContext.getClockProvider, + path, + constraintDescriptor, + validatorFactory.validatorFactoryScopedContext.getConstraintValidatorPayload, + validatorFactory.validatorFactoryScopedContext.getConstraintExpressionLanguageFeatureLevel, + validatorFactory.validatorFactoryScopedContext.getCustomViolationExpressionLanguageFeatureLevel + ) +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD new file mode 100644 index 0000000000..b6fb9b349f --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD @@ -0,0 +1,17 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-internal-engine", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor", + "util/util-validator/src/main/scala/org/hibernate/validator/internal/engine", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/ConstraintViolationFactory.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/ConstraintViolationFactory.scala new file mode 100644 index 0000000000..e59da4626d --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/ConstraintViolationFactory.scala @@ -0,0 +1,167 @@ +package com.twitter.util.validation.internal.engine + +import jakarta.validation.metadata.ConstraintDescriptor +import jakarta.validation.{ + ConstraintValidatorContext, + ConstraintViolation, + ConstraintViolationException, + Payload +} +import java.util.Collections +import org.hibernate.validator.internal.engine.constraintvalidation.{ + ConstraintValidatorContextImpl, + ConstraintViolationCreationContext +} +import org.hibernate.validator.internal.engine.path.PathImpl +import org.hibernate.validator.internal.engine.{ + MessageInterpolatorContext, + ValidatorFactoryInspector, + ConstraintViolationImpl => HibernateConstraintViolationImpl +} +import scala.collection.{SortedSet, mutable} +import scala.jdk.CollectionConverters._ + +private[validation] object ConstraintViolationFactory { + private[this] val MessageWithPathTemplate = "%s: %s" + + def sortSet[T](unsortedSet: Set[ConstraintViolation[T]]): Seq[ConstraintViolation[T]] = + (SortedSet.empty[ConstraintViolation[T]](new ConstraintViolationOrdering) ++ unsortedSet).toSeq + + /* Private */ + + def getMessage(violation: ConstraintViolation[_]): String = + MessageWithPathTemplate.format(violation.getPropertyPath.toString, violation.getMessage) + + private[this] class ConstraintViolationOrdering[T] extends Ordering[ConstraintViolation[T]] { + override def compare(x: ConstraintViolation[T], y: ConstraintViolation[T]): Int = + getMessage(x).compare(getMessage(y)) + } +} + +/** A wrapper/helper for [[ConstraintViolation]] instances */ +private[validation] class ConstraintViolationFactory(validatorFactory: ValidatorFactoryInspector) { + import ConstraintViolationFactory._ + + /** Instantiate a new [[ConstraintViolationException]] from the given Set of violations */ + def newConstraintViolationException( + violations: Set[ConstraintViolation[Any]] + ): ConstraintViolationException = { + val errors: Seq[ConstraintViolation[Any]] = sortSet(violations) + + val message: String = + "\nValidation Errors:\t\t" + errors + .map(getMessage).mkString(", ") + "\n\n" + new ConstraintViolationException(message, errors.toSet.asJava) + } + + /** + * Performs message interpolation given the constraint descriptor and constraint validator context + * to create a set of [[ConstraintViolation]] from the given context and parameters. + */ + def buildConstraintViolations[T]( + rootClazz: Option[Class[T]], + root: Option[T], + leaf: Option[Any], + path: PathImpl, + invalidValue: Any, + constraintDescriptor: ConstraintDescriptor[_], + constraintValidatorContext: ConstraintValidatorContext + ): Set[ConstraintViolation[T]] = { + val results = mutable.HashSet[ConstraintViolation[T]]() + val constraintViolationCreationContexts = + constraintValidatorContext + .asInstanceOf[ConstraintValidatorContextImpl] + .getConstraintViolationCreationContexts + + var index = 0 + val size = constraintViolationCreationContexts.size() + while (index < size) { + val constraintViolationCreationContext = constraintViolationCreationContexts.get(index) + val messageTemplate = constraintViolationCreationContext.getMessage + val interpolatedMessage = + validatorFactory.messageInterpolator + .interpolate( + messageTemplate, + new MessageInterpolatorContext( + constraintDescriptor, + invalidValue, + root.map(_.getClass.asInstanceOf[Class[T]]).orNull, + constraintViolationCreationContext.getPath, + constraintViolationCreationContext.getMessageParameters, + constraintViolationCreationContext.getExpressionVariables, + constraintViolationCreationContext.getExpressionLanguageFeatureLevel, + constraintViolationCreationContext.isCustomViolation + ) + ) + results.add( + newConstraintViolation[T]( + messageTemplate, + interpolatedMessage, + path, + invalidValue, + rootClazz.orNull, + root.map(_.asInstanceOf[T]).getOrElse(null.asInstanceOf[T]), + leaf.orNull, + constraintDescriptor, + constraintViolationCreationContext + ) + ) + index += 1 + } + + if (results.nonEmpty) results.toSet + else Set.empty[ConstraintViolation[T]] + } + + /** Creates a new [[ConstraintViolation]] from the given context and parameters. */ + def newConstraintViolation[T]( + messageTemplate: String, + interpolatedMessage: String, + path: PathImpl, + invalidValue: Any, + rootClazz: Class[T], + root: T, + leaf: Any, + constraintDescriptor: ConstraintDescriptor[_], + payload: Payload + ): ConstraintViolation[T] = + HibernateConstraintViolationImpl.forBeanValidation[T]( + messageTemplate, + Collections.emptyMap[String, Object](), + Collections.emptyMap[String, Object](), + interpolatedMessage, + rootClazz, + root, + leaf, + invalidValue, + path, + constraintDescriptor, + payload + ) + + /** Creates a new [[ConstraintViolation]] from the given context and parameters. */ + def newConstraintViolation[T]( + messageTemplate: String, + interpolatedMessage: String, + path: PathImpl, + invalidValue: Any, + rootClazz: Class[T], + root: T, + leaf: Any, + constraintDescriptor: ConstraintDescriptor[_], + constraintViolationCreationContext: ConstraintViolationCreationContext + ): ConstraintViolation[T] = + HibernateConstraintViolationImpl.forBeanValidation[T]( + messageTemplate, + constraintViolationCreationContext.getMessageParameters, + constraintViolationCreationContext.getExpressionVariables, + interpolatedMessage, + rootClazz, + root, + leaf, + invalidValue, + path, + constraintDescriptor, + constraintViolationCreationContext.getDynamicPayload + ) +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD new file mode 100644 index 0000000000..7d135e23c6 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD @@ -0,0 +1,16 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-internal-metadata-descriptor", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator/src/main/scala/org/hibernate/validator/internal/engine", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/ConstraintDescriptorFactory.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/ConstraintDescriptorFactory.scala new file mode 100644 index 0000000000..e757cc9c0d --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/ConstraintDescriptorFactory.scala @@ -0,0 +1,52 @@ +package com.twitter.util.validation.internal.metadata.descriptor + +import jakarta.validation.groups.Default +import java.lang.annotation.Annotation +import java.lang.reflect.Type +import org.hibernate.validator.internal.engine.ValidatorFactoryInspector +import org.hibernate.validator.internal.metadata.core.ConstraintOrigin +import org.hibernate.validator.internal.metadata.descriptor.{ + ConstraintDescriptorImpl => HibernateConstraintDescriptorImpl +} +import org.hibernate.validator.internal.metadata.descriptor.ConstraintDescriptorImpl.ConstraintType +import org.hibernate.validator.internal.metadata.location.ConstraintLocation.ConstraintLocationKind +import org.hibernate.validator.internal.metadata.raw.ConstrainedElement +import org.hibernate.validator.internal.metadata.raw.ConstrainedElement.ConstrainedElementKind +import org.hibernate.validator.internal.properties.Constrainable +import org.hibernate.validator.internal.util.annotation.ConstraintAnnotationDescriptor + +private[validation] class ConstraintDescriptorFactory( + validatorFactory: ValidatorFactoryInspector) { + + def newConstraintDescriptor( + name: String, + clazz: Class[_], + declaringClazz: Class[_], + annotation: Annotation, + constrainedElementKind: ConstrainedElementKind = ConstrainedElementKind.FIELD + ) = new HibernateConstraintDescriptorImpl( + validatorFactory.constraintHelper, + mkConstrainable(name, clazz, declaringClazz, constrainedElementKind), + new ConstraintAnnotationDescriptor.Builder(annotation).build(), + ConstraintLocationKind.FIELD, + classOf[Default], + ConstraintOrigin.DEFINED_LOCALLY, + ConstraintType.GENERIC + ) + + /* Private */ + + private[this] def mkConstrainable( + name: String, + clazz: Class[_], + declaringClazz: Class[_], + constrainedElementKind: ConstrainedElementKind = ConstrainedElementKind.FIELD + ): Constrainable = new Constrainable { + override def getName: String = name + override def getDeclaringClass: Class[_] = declaringClazz + override def getTypeForValidatorResolution: Type = clazz + override def getType: Type = clazz + override def getConstrainedElementKind: ConstrainedElement.ConstrainedElementKind = + constrainedElementKind + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/MethodValidationConstraintDescriptor.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/MethodValidationConstraintDescriptor.scala new file mode 100644 index 0000000000..11e6b27df9 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/MethodValidationConstraintDescriptor.scala @@ -0,0 +1,41 @@ +package com.twitter.util.validation.internal.metadata.descriptor + +import com.twitter.util.validation.MethodValidation +import jakarta.validation.metadata.{ConstraintDescriptor, ValidateUnwrappedValue} +import jakarta.validation.{ConstraintTarget, ConstraintValidator, Payload, ValidationException} +import java.util + +private[validation] class MethodValidationConstraintDescriptor(annotation: MethodValidation) + extends ConstraintDescriptor[MethodValidation] { + override def getAnnotation: MethodValidation = annotation + override def getMessageTemplate: String = annotation.message() + override def getGroups: util.Set[Class[_]] = { + val target = new util.HashSet[Class[_]] + annotation.groups().foreach(target.add) + target + } + override def getPayload: util.Set[Class[_ <: Payload]] = { + val target = new util.HashSet[Class[_ <: Payload]] + annotation.payload().foreach(target.add) + target + } + override def getValidationAppliesTo: ConstraintTarget = ConstraintTarget.PARAMETERS + override def getConstraintValidatorClasses: util.List[Class[ + _ <: ConstraintValidator[MethodValidation, _] + ]] = util.Collections.emptyList() + override def getAttributes: util.Map[String, AnyRef] = { + val target = new util.HashMap[String, AnyRef]() + var i = 0 + annotation.fields().foreach { key => + target.put(key, Integer.valueOf(i)) + i += 1 + } + target + } + override def getComposingConstraints: util.Set[ConstraintDescriptor[_]] = + util.Collections.emptySet() + override def isReportAsSingleViolation: Boolean = true + override def getValueUnwrapping: ValidateUnwrappedValue = ValidateUnwrappedValue.DEFAULT + override def unwrap[U](clazz: Class[U]): U = throw new ValidationException( + s"${clazz.getName} is unsupported") +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD new file mode 100644 index 0000000000..d0354352bf --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD @@ -0,0 +1,18 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-internal-validators", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-core:scala", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + "util/util-validator/src/main/java/com/twitter/util/validation/constraints", + "util/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidator.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidator.scala new file mode 100644 index 0000000000..bd3c56270b --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidator.scala @@ -0,0 +1,63 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.validation.constraints.CountryCode +import com.twitter.util.validation.constraintvalidation.TwitterConstraintValidatorContext +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} +import java.util.Locale + +private object ISO3166CountryCodeConstraintValidator { + + /** @see [[https://www.iso.org/iso-3166-country-codes.html ISO 3166]] */ + val CountryCodes: Set[String] = Locale.getISOCountries.toSet +} + +private[validation] class ISO3166CountryCodeConstraintValidator + extends ConstraintValidator[CountryCode, Any] { + import ISO3166CountryCodeConstraintValidator._ + + @volatile private[this] var countryCode: CountryCode = _ + + override def initialize(constraintAnnotation: CountryCode): Unit = { + this.countryCode = constraintAnnotation + } + + override def isValid( + obj: Any, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = obj match { + case null => true + case typedValue: Array[Any] => + validationResult(typedValue, constraintValidatorContext) + case typedValue: Iterable[Any] => + validationResult(typedValue, constraintValidatorContext) + case anyValue => + validationResult(Seq(anyValue.toString), constraintValidatorContext) + } + + /* Private */ + + private[this] def validationResult( + value: Iterable[Any], + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = { + val invalidCountryCodes = findInvalidCountryCodes(value) + // an empty value is not a valid country code + val valid = if (value.isEmpty) false else invalidCountryCodes.isEmpty + + if (!valid) { + TwitterConstraintValidatorContext + .addExpressionVariable("validatedValue", mkString(value)) + .withMessageTemplate(countryCode.message()) + .addConstraintViolation(constraintValidatorContext) + } + + valid + } + + private[this] def findInvalidCountryCodes(values: Iterable[Any]): Set[String] = { + val uppercaseCountryCodes = values.toSet.map { value: Any => + value.toString.toUpperCase + } + uppercaseCountryCodes.diff(CountryCodes) + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidator.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidator.scala new file mode 100644 index 0000000000..9f2bea72e4 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidator.scala @@ -0,0 +1,57 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.validation.constraints.OneOf +import com.twitter.util.validation.constraintvalidation.TwitterConstraintValidatorContext +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} + +private[validation] class OneOfConstraintValidator extends ConstraintValidator[OneOf, Any] { + + @volatile private[this] var oneOf: OneOf = _ + @volatile private[this] var values: Set[String] = _ + + override def initialize(constraintAnnotation: OneOf): Unit = { + this.oneOf = constraintAnnotation + this.values = constraintAnnotation.value().toSet + } + + override def isValid( + obj: Any, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = obj match { + case null => true + case arrayValue: Array[_] => + validationResult(arrayValue, values, constraintValidatorContext) + case traversableValue: Iterable[_] => + validationResult(traversableValue, values, constraintValidatorContext) + case anyValue => + validationResult(Seq(anyValue.toString), values, constraintValidatorContext) + } + + /* Private */ + + private[this] def validationResult( + value: Iterable[_], + oneOfValues: Set[String], + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = { + val invalidValues = findInvalidValues(value, oneOfValues) + // an empty value is not one of the given values + val valid = if (value.isEmpty) false else invalidValues.isEmpty + if (!valid) { + TwitterConstraintValidatorContext + .addExpressionVariable("validatedValue", mkString(value)) + .withMessageTemplate(oneOf.message()) + .addConstraintViolation(constraintValidatorContext) + } + + valid + } + + private[this] def findInvalidValues( + value: Iterable[_], + oneOfValues: Set[String] + ): Set[String] = { + val valueAsStrings = value.map(_.toString).toSet + valueAsStrings.diff(oneOfValues) + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/UUIDConstraintValidator.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/UUIDConstraintValidator.scala new file mode 100644 index 0000000000..2220a6facd --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/UUIDConstraintValidator.scala @@ -0,0 +1,17 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.Try +import com.twitter.util.validation.constraints.UUID +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} +import java.util.{UUID => JUUID} + +private[validation] class UUIDConstraintValidator extends ConstraintValidator[UUID, String] { + + override def isValid( + obj: String, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = { + if (obj == null) true + else Try(JUUID.fromString(obj)).isReturn + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD new file mode 100644 index 0000000000..10a0d985d5 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD @@ -0,0 +1,16 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-internal-validators-notempty", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-core:scala", + "util/util-validator/src/main/java/com/twitter/util/validation/constraints", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/NotEmptyValidatorForIterable.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/NotEmptyValidatorForIterable.scala new file mode 100644 index 0000000000..cb00473018 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/NotEmptyValidatorForIterable.scala @@ -0,0 +1,23 @@ +package com.twitter.util.validation.internal.validators.notempty + +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} +import jakarta.validation.constraints.NotEmpty + +/** + * Intended to work exactly like [[org.hibernate.validator.internal.constraintvalidators.bv.notempty.NotEmptyValidatorForArray]] + * but for Scala [[Iterable]] types. + * + * Check that the [[Iterable]] is not null and not empty. + * + * @see [[org.hibernate.validator.internal.constraintvalidators.bv.notempty.NotEmptyValidatorForArray]] + */ +private[validation] class NotEmptyValidatorForIterable + extends ConstraintValidator[NotEmpty, Iterable[_]] { + override def isValid( + obj: Iterable[_], + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = { + if (obj == null) false + else obj.nonEmpty + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/package.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/package.scala new file mode 100644 index 0000000000..0dee50a7ea --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/package.scala @@ -0,0 +1,11 @@ +package com.twitter.util.validation.internal + +package object validators { + + private[validation] def mkString(value: Iterable[_]): String = { + val trimmed = value.map(_.toString.trim).filter(_.nonEmpty) + if (trimmed.nonEmpty && trimmed.size > 1) s"${trimmed.mkString("[", ", ", "]")}" + else if (trimmed.nonEmpty) trimmed.head + else "" + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD new file mode 100644 index 0000000000..e13c3e5655 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD @@ -0,0 +1,16 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-internal-validators-size", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-core:scala", + "util/util-validator/src/main/java/com/twitter/util/validation/constraints", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/SizeValidatorForIterable.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/SizeValidatorForIterable.scala new file mode 100644 index 0000000000..b4d168b5e5 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/SizeValidatorForIterable.scala @@ -0,0 +1,38 @@ +package com.twitter.util.validation.internal.validators.size + +import jakarta.validation.constraints.Size +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} + +/** + * Intended to work exactly like [[org.hibernate.validator.internal.constraintvalidators.bv.size.SizeValidatorForArray]] + * but for Scala [[Iterable]] types. + * + * Check that the length of an [[Iterable]] is between `min` and `max`. + * + * @see [[org.hibernate.validator.internal.constraintvalidators.bv.size.SizeValidatorForArray]] + */ +private[validation] class SizeValidatorForIterable extends ConstraintValidator[Size, Iterable[_]] { + + @volatile private[this] var min: Int = _ + @volatile private[this] var max: Int = _ + + override def initialize(constraintAnnotation: Size): Unit = { + this.min = constraintAnnotation.min() + this.max = constraintAnnotation.max() + validateParameters() + } + + override def isValid( + obj: Iterable[_], + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = { + if (obj == null) true + else obj.size >= min && obj.size <= max + } + + private def validateParameters(): Unit = { + if (min < 0) throw new IllegalArgumentException("The min parameter cannot be negative.") + if (max < 0) throw new IllegalArgumentException("The max parameter cannot be negative.") + if (max < min) throw new IllegalArgumentException("The length cannot be negative.") + } +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD new file mode 100644 index 0000000000..939df4b445 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-metadata", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/github/ben-manes/caffeine", + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "3rdparty/jvm/org/json4s:json4s-core", + "util/util-core:scala", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + "util/util-slf4j-api/src/main/scala", + ], +) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/metadata/CaseClassDescriptor.scala b/util-validator/src/main/scala/com/twitter/util/validation/metadata/CaseClassDescriptor.scala new file mode 100644 index 0000000000..8611c96423 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/metadata/CaseClassDescriptor.scala @@ -0,0 +1,13 @@ +package com.twitter.util.validation.metadata + +import java.lang.annotation.Annotation +import org.json4s.reflect.ScalaType + +case class CaseClassDescriptor[T] private[validation] ( + clazz: Class[T], + scalaType: ScalaType, + annotations: Array[Annotation], + constructors: Array[ConstructorDescriptor], + members: Map[String, PropertyDescriptor], + methods: Array[MethodDescriptor]) + extends Descriptor diff --git a/util-validator/src/main/scala/com/twitter/util/validation/metadata/ConstructorDescriptor.scala b/util-validator/src/main/scala/com/twitter/util/validation/metadata/ConstructorDescriptor.scala new file mode 100644 index 0000000000..d66c9fd29c --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/metadata/ConstructorDescriptor.scala @@ -0,0 +1,10 @@ +package com.twitter.util.validation.metadata + +import java.lang.annotation.Annotation +import java.lang.reflect.Executable + +case class ConstructorDescriptor private[validation] ( + override val executable: Executable, + annotations: Array[Annotation], + members: Map[String, PropertyDescriptor]) + extends ExecutableDescriptor(executable) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/metadata/Descriptor.scala b/util-validator/src/main/scala/com/twitter/util/validation/metadata/Descriptor.scala new file mode 100644 index 0000000000..7d6f72f03d --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/metadata/Descriptor.scala @@ -0,0 +1,4 @@ +package com.twitter.util.validation.metadata + +/** Marker trait for Descriptor Metadata */ +trait Descriptor diff --git a/util-validator/src/main/scala/com/twitter/util/validation/metadata/ExecutableDescriptor.scala b/util-validator/src/main/scala/com/twitter/util/validation/metadata/ExecutableDescriptor.scala new file mode 100644 index 0000000000..8157d4aac9 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/metadata/ExecutableDescriptor.scala @@ -0,0 +1,10 @@ +package com.twitter.util.validation.metadata + +import java.lang.annotation.Annotation +import java.lang.reflect.Executable + +abstract class ExecutableDescriptor private[validation] (val executable: Executable) + extends Descriptor { + def annotations: Array[Annotation] + def members: Map[String, PropertyDescriptor] +} diff --git a/util-validator/src/main/scala/com/twitter/util/validation/metadata/MethodDescriptor.scala b/util-validator/src/main/scala/com/twitter/util/validation/metadata/MethodDescriptor.scala new file mode 100644 index 0000000000..7484ea02d8 --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/metadata/MethodDescriptor.scala @@ -0,0 +1,10 @@ +package com.twitter.util.validation.metadata + +import java.lang.annotation.Annotation +import java.lang.reflect.Method + +case class MethodDescriptor private[validation] ( + method: Method, + annotations: Array[Annotation], + members: Map[String, PropertyDescriptor]) + extends ExecutableDescriptor(method) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/metadata/PropertyDescriptor.scala b/util-validator/src/main/scala/com/twitter/util/validation/metadata/PropertyDescriptor.scala new file mode 100644 index 0000000000..6f94d7920f --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/metadata/PropertyDescriptor.scala @@ -0,0 +1,11 @@ +package com.twitter.util.validation.metadata + +import java.lang.annotation.Annotation +import org.json4s.reflect.ScalaType + +case class PropertyDescriptor private[validation] ( + scalaType: ScalaType, + cascadedScalaType: Option[ScalaType], + annotations: Array[Annotation], + isCascaded: Boolean = false) + extends Descriptor diff --git a/util-validator/src/main/scala/com/twitter/util/validation/package.scala b/util-validator/src/main/scala/com/twitter/util/validation/package.scala new file mode 100644 index 0000000000..d146cd8cde --- /dev/null +++ b/util-validator/src/main/scala/com/twitter/util/validation/package.scala @@ -0,0 +1,10 @@ +package com.twitter.util + +import jakarta.validation.ConstraintValidator +import java.lang.annotation.Annotation + +package object validation { + type ConstraintValidatorType = ConstraintValidator[_ <: Annotation, _] + type ConstraintAnnotationClazz = Class[_ <: Annotation] + type ConstraintValidatorClazz = Class[_ <: ConstraintValidatorType] +} diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD b/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD new file mode 100644 index 0000000000..0926cb69c2 --- /dev/null +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD @@ -0,0 +1,18 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-internal-hibernate-engine", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + ], + exports = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + ], +) diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/engine/ValidatorFactoryInspector.scala b/util-validator/src/main/scala/org/hibernate/validator/internal/engine/ValidatorFactoryInspector.scala new file mode 100644 index 0000000000..1acde1ef9f --- /dev/null +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/engine/ValidatorFactoryInspector.scala @@ -0,0 +1,41 @@ +package org.hibernate.validator.internal.engine + +import jakarta.validation.{ConstraintValidatorFactory, MessageInterpolator} +import java.lang.reflect.Field +import org.hibernate.validator.internal.metadata.core.ConstraintHelper +import org.hibernate.validator.internal.util.{ExecutableHelper, ExecutableParameterNameProvider} + +/** Allows access to the configured [[ConstraintHelper]] and [[ConstraintValidatorFactory]] */ +class ValidatorFactoryInspector(underlying: ValidatorFactoryImpl) { + + def constraintHelper: ConstraintHelper = + underlying.getConstraintCreationContext.getConstraintHelper + + def constraintValidatorFactory: ConstraintValidatorFactory = + underlying.getConstraintValidatorFactory + + def validatorFactoryScopedContext: ValidatorFactoryScopedContext = + underlying.getValidatorFactoryScopedContext + + def messageInterpolator: MessageInterpolator = + underlying.getMessageInterpolator + + def getConstraintCreationContext: ConstraintCreationContext = + underlying.getConstraintCreationContext + + def getExecutableHelper: ExecutableHelper = { + val field: Field = classOf[ValidatorFactoryImpl].getDeclaredField("executableHelper") + field.setAccessible(true) + field.get(underlying).asInstanceOf[ExecutableHelper] + } + + def getExecutableParameterNameProvider: ExecutableParameterNameProvider = + underlying.getExecutableParameterNameProvider + + def getMethodValidationConfiguration: MethodValidationConfiguration = + underlying.getMethodValidationConfiguration + + /** Close underlying ValidatorFactoryImpl */ + def close(): Unit = + underlying.close() +} diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD new file mode 100644 index 0000000000..a6c291af78 --- /dev/null +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-validation-internal-hibernate-properties-javabeans", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + ], + exports = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + ], +) diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/ConstructorCallable.scala b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/ConstructorCallable.scala new file mode 100644 index 0000000000..b52f66cad3 --- /dev/null +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/ConstructorCallable.scala @@ -0,0 +1,20 @@ +package org.hibernate.validator.internal.properties.javabean + +import com.twitter.util.reflect.Classes +import java.lang.reflect.{Constructor, Method, TypeVariable} +import org.hibernate.validator.internal.metadata.raw.ConstrainedElement.ConstrainedElementKind + +class ConstructorCallable(constructor: Constructor[_]) + extends ExecutableCallable[Constructor[_]](constructor, true) { + + override def getName: String = Classes.simpleName(getDeclaringClass) + + def getConstrainedElementKind: ConstrainedElementKind = ConstrainedElementKind.CONSTRUCTOR + + def getTypeParameters: Array[TypeVariable[Class[_]]] = + executable + .asInstanceOf[Method] + .getReturnType + .getTypeParameters + .asInstanceOf[Array[TypeVariable[Class[_]]]] +} diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/ExecutableCallable.scala b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/ExecutableCallable.scala new file mode 100644 index 0000000000..de74eac668 --- /dev/null +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/ExecutableCallable.scala @@ -0,0 +1,186 @@ +package org.hibernate.validator.internal.properties.javabean + +import com.twitter.util.reflect.Classes +import jakarta.validation.ValidationException +import java.lang.reflect.{Constructor, Executable, Method, Modifier, Parameter, Type, TypeVariable} +import org.hibernate.validator.internal.properties.{Callable, Signature} +import org.hibernate.validator.internal.util.{ + ExecutableHelper, + ExecutableParameterNameProvider, + ReflectionHelper, + TypeHelper +} +import scala.collection.mutable + +private object ExecutableCallable { + + def getParameters(executable: Executable): Seq[JavaBeanParameter] = { + if (executable.getParameterCount == 0) Seq.empty[JavaBeanParameter] + else { + val parameters = new mutable.ArrayBuffer[JavaBeanParameter](executable.getParameterCount) + + val parameterArray = executable.getParameters + val parameterTypes = executable.getParameterTypes + val genericParameterTypes = executable.getGenericParameterTypes + if (parameterTypes.length == genericParameterTypes.length) { + var index = 0; + val length = parameterArray.length + while (index < length) { + parameters.append( + new JavaBeanParameter( + index, + parameterArray(index), + parameterTypes(index), + getErasedTypeIfTypeVariable(genericParameterTypes(index)))) + + index += 1 + } + } else { + // in this case, we have synthetic or implicit parameters + val hasParameterModifierInfo = isAnyParameterCarryingMetadata(parameterArray) + if (!hasParameterModifierInfo) + throw new ValidationException("") + + var explicitlyDeclaredParameterIndex = 0 + var index = 0; + val length = parameterArray.length + while (index < length) { + if (explicitlyDeclaredParameterIndex < genericParameterTypes.length + && isExplicit(parameterArray(index)) + && parameterTypesMatch( + parameterTypes(index), + genericParameterTypes(explicitlyDeclaredParameterIndex))) { + // in this case we have a parameter that is present and matches ("most likely") to the one in the generic parameter types list + parameters.append( + new JavaBeanParameter( + index, + parameterArray(index), + parameterTypes(index), + getErasedTypeIfTypeVariable(genericParameterTypes(explicitlyDeclaredParameterIndex)) + ) + ) + explicitlyDeclaredParameterIndex += 1 + } else { + // in this case, the parameter is not present in genericParameterTypes, or the types doesn't match + parameters.append( + new JavaBeanParameter( + index, + parameterArray(index), + parameterTypes(index), + parameterTypes(index))) + } + + index += 1 + } + } + + parameters.toSeq + } + } + + private[this] def getErasedTypeIfTypeVariable(genericType: Type): Type = genericType match { + case _: TypeVariable[_] => + TypeHelper.getErasedType(genericType) + case _ => + genericType + } + + private[this] def isAnyParameterCarryingMetadata(parameterArray: Array[Parameter]): Boolean = { + parameterArray.exists(p => p.isSynthetic || p.isImplicit) + } + + private[this] def parameterTypesMatch(paramType: Class[_], genericParamType: Type): Boolean = { + TypeHelper.getErasedType(genericParamType).equals(paramType) + } + + private[this] def isExplicit(parameter: Parameter): Boolean = { + !parameter.isSynthetic && !parameter.isImplicit + } +} + +abstract class ExecutableCallable[T <: Executable]( + val executable: T, + hasReturnValue: Boolean) + extends Callable { + + private val `type`: Type = ReflectionHelper.typeOf(executable) + private val parameters: Seq[JavaBeanParameter] = ExecutableCallable.getParameters(executable) + private val typeForValidatorResolution: Type = ReflectionHelper.boxedType(this.`type`) + + def getParameters: Seq[JavaBeanParameter] = this.parameters + override def getName: String = executable.getName + override def getDeclaringClass: Class[_] = executable.getDeclaringClass + override def getTypeForValidatorResolution: Type = this.typeForValidatorResolution + override def getType: Type = this.`type` + override def hasReturnValue: Boolean = hasReturnValue + override def hasParameters: Boolean = this.parameters.nonEmpty + override def getParameterCount: Int = this.parameters.size + override def getParameterGenericType(index: Int): Type = this.parameters(index).getGenericType + override def getParameterTypes: Array[Class[_]] = executable.getParameterTypes + override def getParameterName( + parameterNameProvider: ExecutableParameterNameProvider, + parameterIndex: Int + ): String = parameterNameProvider.getParameterNames(executable).get(parameterIndex) + + override def isPrivate: Boolean = Modifier.isPrivate(executable.getModifiers) + override def getSignature: Signature = { + val simpleName = executable match { + case c: Constructor[_] => + Classes.simpleName(c.getDeclaringClass) + case _ => executable.getName + } + ExecutableHelper.getSignature(simpleName, executable.getParameterTypes) + } + + override def overrides( + executableHelper: ExecutableHelper, + superTypeMethod: Callable + ): Boolean = executableHelper.overrides( + executable.asInstanceOf[Method], + superTypeMethod.asInstanceOf[ExecutableCallable[T]].executable.asInstanceOf[Method] + ) + + override def isResolvedToSameMethodInHierarchy( + executableHelper: ExecutableHelper, + mainSubType: Class[_], + superTypeMethod: Callable + ): Boolean = + executableHelper + .isResolvedToSameMethodInHierarchy( + mainSubType, + executable.asInstanceOf[Method], + superTypeMethod.asInstanceOf[ExecutableCallable[T]].executable.asInstanceOf[Method] + ) + + override def equals(o: Any): Boolean = { + if (this == o) { + true + } else if (o != null && this.getClass == o.getClass) { + val that: ExecutableCallable[T] = o.asInstanceOf[ExecutableCallable[T]] + if (this.hasReturnValue != that.hasReturnValue) { + false + } else if (!this.executable.equals(that.executable)) { + false + } else { + if (this.typeForValidatorResolution != that.typeForValidatorResolution) false + else this.`type` == that.`type` + } + } else { + false + } + } + + override def hashCode: Int = { + var result = executable.hashCode + result = 31 * result + this.typeForValidatorResolution.hashCode + result = 31 * result + (if (this.hasReturnValue) 1 else 0) + result = 31 * result + this.`type`.hashCode + result + } + + override def toString: String = + ExecutableHelper.getExecutableAsString( + this.getDeclaringClass.getSimpleName + "#" + executable.getName, + executable.getParameterTypes: _* + ) +} diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/MethodCallable.scala b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/MethodCallable.scala new file mode 100644 index 0000000000..f096a527d5 --- /dev/null +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/MethodCallable.scala @@ -0,0 +1,13 @@ +package org.hibernate.validator.internal.properties.javabean + +import java.lang.reflect.{Method, TypeVariable} +import org.hibernate.validator.internal.metadata.raw.ConstrainedElement.ConstrainedElementKind + +class MethodCallable(method: Method) + extends ExecutableCallable[Method](method, method.getGenericReturnType != classOf[Unit]) { + + def getConstrainedElementKind: ConstrainedElementKind = ConstrainedElementKind.METHOD + + def getTypeParameters: Array[TypeVariable[Class[_]]] = + method.getReturnType.getTypeParameters.asInstanceOf[Array[TypeVariable[Class[_]]]] +} diff --git a/util-validator/src/test/java/com/twitter/util/validation/BUILD b/util-validator/src/test/java/com/twitter/util/validation/BUILD new file mode 100644 index 0000000000..102b9f1969 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/BUILD @@ -0,0 +1,14 @@ +java_library( + sources = [ + "*.java", + "constraints/*.java", + ], + compiler_option_sets = ["fatal_warnings"], + strict_deps = True, + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + ], + exports = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + ], +) diff --git a/util-validator/src/test/java/com/twitter/util/validation/CarMake.java b/util-validator/src/test/java/com/twitter/util/validation/CarMake.java new file mode 100644 index 0000000000..ef1fdd3468 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/CarMake.java @@ -0,0 +1,8 @@ +package com.twitter.util.validation; + +/** + * Test Java Enum + */ +public enum CarMake { + Ford, Audi, Volkswagen, Tesla, Honda; +} diff --git a/util-validator/src/test/java/com/twitter/util/validation/DoesNothing.java b/util-validator/src/test/java/com/twitter/util/validation/DoesNothing.java new file mode 100644 index 0000000000..6b74435c6b --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/DoesNothing.java @@ -0,0 +1,16 @@ +package com.twitter.util.validation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +public @interface DoesNothing {} diff --git a/util-validator/src/test/java/com/twitter/util/validation/constraints/InvalidConstraint.java b/util-validator/src/test/java/com/twitter/util/validation/constraints/InvalidConstraint.java new file mode 100644 index 0000000000..23b0926846 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/constraints/InvalidConstraint.java @@ -0,0 +1,29 @@ +package com.twitter.util.validation.constraints; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = {}) +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +public @interface InvalidConstraint { + + String message() default "invalid"; + + Class[] groups() default {}; + + Class[] payload() default {}; +} diff --git a/util-validator/src/test/java/com/twitter/util/validation/constraints/ValidPassengerCount.java b/util-validator/src/test/java/com/twitter/util/validation/constraints/ValidPassengerCount.java new file mode 100644 index 0000000000..34b18f8419 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/constraints/ValidPassengerCount.java @@ -0,0 +1,46 @@ +package com.twitter.util.validation.constraints; + +import java.lang.annotation.Documented; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = {}) +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +@Repeatable(ValidPassengerCount.List.class) +public @interface ValidPassengerCount { + + String message() default "number of passenger(s) is not valid"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + /** + * The max number of passengers. + */ + long max(); + + /** + * Defines several {@code @OneOf} annotations on the same element. + */ + @Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) + @Retention(RUNTIME) + @Documented + public @interface List { + ValidPassengerCount[] value(); + } +} diff --git a/util-validator/src/test/java/com/twitter/util/validation/constraints/ValidPassengerCountReturnValue.java b/util-validator/src/test/java/com/twitter/util/validation/constraints/ValidPassengerCountReturnValue.java new file mode 100644 index 0000000000..4065b33ef1 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/constraints/ValidPassengerCountReturnValue.java @@ -0,0 +1,46 @@ +package com.twitter.util.validation.constraints; + +import java.lang.annotation.Documented; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = {}) +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +@Repeatable(ValidPassengerCountReturnValue.List.class) +public @interface ValidPassengerCountReturnValue { + + String message() default "number of passenger(s) is not valid"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + /** + * The max number of passengers. + */ + long max(); + + /** + * Defines several {@code @OneOf} annotations on the same element. + */ + @Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) + @Retention(RUNTIME) + @Documented + public @interface List { + ValidPassengerCountReturnValue[] value(); + } +} diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD b/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD new file mode 100644 index 0000000000..f9b0d1d1a7 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD @@ -0,0 +1,24 @@ +junit_tests( + sources = ["**/*.java"], + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/slf4j:slf4j-api", + scoped( + "3rdparty/jvm/org/slf4j:slf4j-simple", + scope = "runtime", + ), + "util/util-slf4j-api/src/main/scala", + "util/util-validator/src/main/scala/com/twitter/util/validation", + "util/util-validator/src/main/scala/com/twitter/util/validation/cfg", + "util/util-validator/src/main/scala/com/twitter/util/validation/conversions", + "util/util-validator/src/main/scala/com/twitter/util/validation/engine", + "util/util-validator/src/main/scala/com/twitter/util/validation/executable", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/validators", + "util/util-validator/src/main/scala/com/twitter/util/validation/metadata", + "util/util-validator/src/test/java/com/twitter/util/validation", + ], +) diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/Car.java b/util-validator/src/test/java/com/twitter/util/validation/tests/Car.java new file mode 100644 index 0000000000..26543766e9 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/tests/Car.java @@ -0,0 +1,52 @@ +package com.twitter.util.validation.tests; + +import java.util.Collections; +import java.util.List; + +import com.twitter.util.validation.constraints.ValidPassengerCount; + +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; + +@ValidPassengerCount(max = 2) +public class Car { + @NotNull + private final String manufacturer; + + @NotNull + @Size(min = 2, max = 14) + private final String licensePlate; + + @Min(2) + private final int seatCount; + + private List passengers; + + public Car(String manufacturer, String licencePlate, int seatCount) { + this.manufacturer = manufacturer; + this.licensePlate = licencePlate; + this.seatCount = seatCount; + this.passengers = Collections.emptyList(); + } + + public void load(List passengers) { + this.passengers = passengers; + } + + public String getManufacturer() { + return manufacturer; + } + + public String getLicensePlate() { + return licensePlate; + } + + public int getSeatCount() { + return seatCount; + } + + public List getPassengers() { + return passengers; + } +} diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/Person.java b/util-validator/src/test/java/com/twitter/util/validation/tests/Person.java new file mode 100644 index 0000000000..bfc5872813 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/tests/Person.java @@ -0,0 +1,16 @@ +package com.twitter.util.validation.tests; + +import jakarta.validation.constraints.NotNull; + +public class Person { + @NotNull + private String name; + + public Person(String name) { + this.name = name; + } + + public String getName() { + return name; + } +} diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/ScalaValidatorJavaTest.java b/util-validator/src/test/java/com/twitter/util/validation/tests/ScalaValidatorJavaTest.java new file mode 100644 index 0000000000..3fa03564a9 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/tests/ScalaValidatorJavaTest.java @@ -0,0 +1,155 @@ +package com.twitter.util.validation.tests; + +import java.lang.annotation.Annotation; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.Assert; +import org.junit.Test; + +import com.twitter.util.validation.ScalaValidator; +import com.twitter.util.validation.cfg.ConstraintMapping; +import com.twitter.util.validation.constraints.ValidPassengerCount; + +import jakarta.validation.ConstraintViolation; + +public class ScalaValidatorJavaTest extends Assert { + private final ConstraintMapping mapping = + ConstraintMapping.apply( + ValidPassengerCount.class, + ValidPassengerCountConstraintValidator.class, false + ); + + private final ScalaValidator validator = + ScalaValidator.builder() + .withConstraintMapping(mapping) + .validator(); + + @Test + public void testConstructors() { + Assert.assertNotNull(ScalaValidator.apply()); + Assert.assertNotNull(ScalaValidator.builder().validator()); + } + + @Test + public void testBuilder() { + Assert.assertNotNull(ScalaValidator.builder().validator()); + + Set mappings = new HashSet<>(); + mappings.add(mapping); + + Assert.assertNotNull( + ScalaValidator.builder() + .withConstraintMapping(mapping) + .validator() + ); + + Assert.assertNotNull( + ScalaValidator.builder() + .withConstraintMappings(mappings) + .validator() + ); + } + + @Test + public void testUnderlyingValidator() { + Car car = new Car("Renault", "DD-AB-123", 4); + + Set> violations = validator.underlying().validate(car); + Assert.assertEquals(violations.size(), 0); + + List passengers = new ArrayList(); + passengers.add(new Person("R Smith")); + passengers.add(new Person("S Smith")); + passengers.add(new Person("T Smith")); + passengers.add(new Person("U Smith")); + + car = new Car("Renault", "DD-AB-123", 2); + car.load(passengers); + + violations = validator.underlying().validate(car); + Assert.assertEquals(violations.size(), 1); + } + + @Test + public void testValidateFieldValue() { + Map sizeConstraintValues = new HashMap<>(); + sizeConstraintValues.put("min", 5); + sizeConstraintValues.put("max", 7); + + Map, Map> constraints = new HashMap<>(); + constraints.put(jakarta.validation.constraints.Size.class, sizeConstraintValues); + + Set> violations = + validator.validateFieldValue( + constraints, + "javaSetIntField", + newSet(6) + ); + + Assert.assertEquals(violations.size(), 0); + + Set value = newSet(9); + violations = + validator.validateFieldValue( + constraints, + "javaSetIntField", + value + ); + + Assert.assertEquals(violations.size(), 1); + ConstraintViolation violation = violations.iterator().next(); + Assert.assertEquals(violation.getPropertyPath().toString(), "javaSetIntField"); + Assert.assertEquals(violation.getMessage(), "size must be between 5 and 7"); + Assert.assertEquals(violation.getInvalidValue(), value); + } + + @Test + public void testValidateFieldValueWithGroups() { + Class[] groups = {TestConstraintGroup.class}; + Map sizeConstraintValues = new HashMap<>(); + sizeConstraintValues.put("min", 5); + sizeConstraintValues.put("max", 7); + sizeConstraintValues.put("groups", groups); + + Map, Map> constraints = new HashMap<>(); + constraints.put(jakarta.validation.constraints.Size.class, sizeConstraintValues); + + Set value = newSet(9); + // TestConstraintGroup is not active + Set> violations = + validator.validateFieldValue( + constraints, + "javaSetIntField", + value + ); + Assert.assertEquals(violations.size(), 0); + + // TestConstraintGroup is active + violations = + validator.validateFieldValue( + constraints, + "javaSetIntField", + value, + TestConstraintGroup.class + ); + ConstraintViolation violation = violations.iterator().next(); + Assert.assertEquals(violation.getPropertyPath().toString(), "javaSetIntField"); + Assert.assertEquals(violation.getMessage(), "size must be between 5 and 7"); + Assert.assertEquals(violation.getInvalidValue(), value); + } + + public Set newSet(Integer total) { + Set results = new HashSet<>(); + for (int i = 0; i < total; i++) { + results.add(i); + } + + return results; + } +} diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/TestConstraintGroup.java b/util-validator/src/test/java/com/twitter/util/validation/tests/TestConstraintGroup.java new file mode 100644 index 0000000000..100deedc95 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/tests/TestConstraintGroup.java @@ -0,0 +1,4 @@ +package com.twitter.util.validation.tests; + +public interface TestConstraintGroup { +} diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/ValidPassengerCountConstraintValidator.java b/util-validator/src/test/java/com/twitter/util/validation/tests/ValidPassengerCountConstraintValidator.java new file mode 100644 index 0000000000..4e4fb5f062 --- /dev/null +++ b/util-validator/src/test/java/com/twitter/util/validation/tests/ValidPassengerCountConstraintValidator.java @@ -0,0 +1,25 @@ +package com.twitter.util.validation.tests; + +import com.twitter.util.validation.constraints.ValidPassengerCount; + +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; + +public class ValidPassengerCountConstraintValidator implements ConstraintValidator { + + private long max; + + @Override + public void initialize(ValidPassengerCount constraintAnnotation) { + this.max = constraintAnnotation.max(); + } + + @Override + public boolean isValid(Car car, ConstraintValidatorContext context) { + if ( car == null ) { + return true; + } + + return car.getPassengers().size() <= this.max; + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/AssertViolationTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/AssertViolationTest.scala new file mode 100644 index 0000000000..9061806fc9 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/AssertViolationTest.scala @@ -0,0 +1,59 @@ +package com.twitter.util.validation + +import com.twitter.util.validation.engine.ConstraintViolationHelper +import jakarta.validation.ConstraintViolation +import java.lang.annotation.Annotation +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner +import scala.reflect.runtime.universe._ + +object AssertViolationTest { + case class WithViolation[T]( + path: String, + message: String, + invalidValue: Any, + rootBeanClazz: Class[T], + root: T, + annotation: Option[Annotation] = None) +} + +@RunWith(classOf[JUnitRunner]) +abstract class AssertViolationTest extends AnyFunSuite with Matchers { + import AssertViolationTest._ + + protected def validator: ScalaValidator + + protected def assertViolations[T: TypeTag]( + obj: T, + groups: Seq[Class[_]] = Seq.empty, + withViolations: Seq[WithViolation[T]] = Seq.empty + ): Unit = { + val violations: Set[ConstraintViolation[T]] = validator.validate(obj, groups: _*) + assertViolations(violations, withViolations) + } + + protected def assertViolations[T: TypeTag]( + violations: Set[ConstraintViolation[T]], + withViolations: Seq[WithViolation[T]] + ): Unit = { + violations.size should equal(withViolations.size) + val sortedViolations: Seq[ConstraintViolation[T]] = + ConstraintViolationHelper.sortViolations(violations) + for ((constraintViolation, index) <- sortedViolations.zipWithIndex) { + val withViolation = withViolations(index) + constraintViolation.getMessage should equal(withViolation.message) + if (constraintViolation.getPropertyPath == null) withViolation.path should be(null) + else constraintViolation.getPropertyPath.toString should equal(withViolation.path) + constraintViolation.getInvalidValue should equal(withViolation.invalidValue) + constraintViolation.getRootBeanClass should equal(withViolation.rootBeanClazz) + withViolation.root.getClass.getName == constraintViolation.getRootBean + .asInstanceOf[T].getClass.getName should be(true) + constraintViolation.getLeafBean should not be null + withViolation.annotation.foreach { ann => + constraintViolation.getConstraintDescriptor.getAnnotation should equal(ann) + } + } + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/BUILD b/util-validator/src/test/scala/com/twitter/util/validation/BUILD new file mode 100644 index 0000000000..7e6f5d9030 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/BUILD @@ -0,0 +1,34 @@ +junit_tests( + sources = [ + "**/*.java", + "**/*.scala", + ], + compiler_option_sets = ["fatal_warnings"], + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", + "3rdparty/jvm/com/github/ben-manes/caffeine", + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "3rdparty/jvm/org/json4s:json4s-core", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:scalacheck", + scoped( + "3rdparty/jvm/org/slf4j:slf4j-simple", + scope = "runtime", + ), + "util/util-core:scala", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + "util/util-slf4j-api/src/main/scala", + "util/util-validator/src/main/scala/com/twitter/util/validation", + "util/util-validator/src/main/scala/com/twitter/util/validation/cfg", + "util/util-validator/src/main/scala/com/twitter/util/validation/conversions", + "util/util-validator/src/main/scala/com/twitter/util/validation/engine", + "util/util-validator/src/main/scala/com/twitter/util/validation/executable", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal", + "util/util-validator/src/main/scala/com/twitter/util/validation/internal/validators", + "util/util-validator/src/main/scala/com/twitter/util/validation/metadata", + "util/util-validator/src/test/java/com/twitter/util/validation", + ], +) diff --git a/util-validator/src/test/scala/com/twitter/util/validation/OptionUnwrappingTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/OptionUnwrappingTest.scala new file mode 100644 index 0000000000..634d346a25 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/OptionUnwrappingTest.scala @@ -0,0 +1,170 @@ +package com.twitter.util.validation + +import com.twitter.util.validation.AssertViolationTest._ +import com.twitter.util.validation.caseclasses._ + +/** + * Options are "unwrapped" by default for validation, meaning any supplied constraint + * annotation is applied to the contained type/value of the Option. The unwrapping + * happens through the Validator and not in the individual ConstraintValidator thus we + * test though a Validator here and why these tests are not in the individual ConstraintValidator + * tests. + */ +class OptionUnwrappingTest extends AssertViolationTest { + + override protected val validator: ScalaValidator = ScalaValidator.builder.validator + + test("CountryCode") { + assertViolations( + obj = CountryCodeOptionExample(Some("11")), + withViolations = Seq( + WithViolation( + "countryCode", + "11 not a valid country code", + "11", + classOf[CountryCodeOptionExample], + CountryCodeOptionExample(Some("11"))) + ) + ) + assertViolations(obj = CountryCodeOptionExample(None)) + + assertViolations( + obj = CountryCodeOptionSeqExample(Some(Seq("11", "22"))), + withViolations = Seq( + WithViolation( + "countryCode", + "[11, 22] not a valid country code", + Seq("11", "22"), + classOf[CountryCodeOptionSeqExample], + CountryCodeOptionSeqExample(Some(Seq("11", "22"))) + ) + ) + ) + assertViolations(obj = CountryCodeOptionSeqExample(None)) + + assertViolations( + obj = CountryCodeOptionArrayExample(Some(Array("11", "22"))), + withViolations = Seq( + WithViolation( + "countryCode", + "[11, 22] not a valid country code", + Array("11", "22"), + classOf[CountryCodeOptionArrayExample], + CountryCodeOptionArrayExample(Some(Array("11", "22"))) + ) + ) + ) + assertViolations(obj = CountryCodeOptionArrayExample(None)) + + assertViolations( + obj = CountryCodeOptionInvalidTypeExample(Some(12345L)), + withViolations = Seq( + WithViolation( + "countryCode", + "12345 not a valid country code", + 12345L, + classOf[CountryCodeOptionInvalidTypeExample], + CountryCodeOptionInvalidTypeExample(Some(12345L))) + ) + ) + assertViolations(obj = CountryCodeOptionInvalidTypeExample(None)) + } + + test("OneOf") { + assertViolations( + obj = OneOfOptionExample(Some("g")), + withViolations = Seq( + WithViolation( + "enumValue", + "g not one of [a, B, c]", + "g", + classOf[OneOfOptionExample], + OneOfOptionExample(Some("g"))) + ) + ) + assertViolations(OneOfOptionExample(None)) + + assertViolations( + obj = OneOfOptionSeqExample(Some(Seq("g", "h", "i"))), + withViolations = Seq( + WithViolation( + "enumValue", + "[g, h, i] not one of [a, B, c]", + Seq("g", "h", "i"), + classOf[OneOfOptionSeqExample], + OneOfOptionSeqExample(Some(Seq("g", "h", "i")))) + ) + ) + assertViolations(OneOfOptionSeqExample(None)) + + assertViolations( + obj = OneOfOptionInvalidTypeExample(Some(10L)), + withViolations = Seq( + WithViolation( + "enumValue", + "10 not one of [a, B, c]", + 10L, + classOf[OneOfOptionInvalidTypeExample], + OneOfOptionInvalidTypeExample(Some(10L))) + ) + ) + assertViolations(OneOfOptionSeqExample(None)) + assertViolations(OneOfOptionInvalidTypeExample(None)) + } + + test("UUID") { + assertViolations( + obj = UUIDOptionExample(Some("abcd1234")), + withViolations = Seq( + WithViolation( + "uuid", + "must be a valid UUID", + "abcd1234", + classOf[UUIDOptionExample], + UUIDOptionExample(Some("abcd1234"))) + ) + ) + assertViolations(obj = UUIDOptionExample(None)) + } + + test("NestedOptionExample") { + assertViolations( + obj = NestedOptionExample( + "abcd1234", + "Foo Var", + None + ), + ) + + val value = NestedOptionExample( + "", + "Foo Var", + Some( + User( + "", + "Bar Var", + "H" + ) + ) + ) + assertViolations( + obj = value, + withViolations = Seq( + WithViolation( + "id", + "must not be empty", + "", + classOf[NestedOptionExample], + value + ), + WithViolation( + "user.gender", + "H not one of [F, M, Other]", + "H", + classOf[NestedOptionExample], + value), + WithViolation("user.id", "must not be empty", "", classOf[NestedOptionExample], value) + ) + ) + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/ScalaValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/ScalaValidatorTest.scala new file mode 100644 index 0000000000..fafeb3abb5 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/ScalaValidatorTest.scala @@ -0,0 +1,1507 @@ +package com.twitter.util.validation + +import com.twitter.util.validation.AssertViolationTest.WithViolation +import com.twitter.util.validation.caseclasses._ +import com.twitter.util.validation.cfg.ConstraintMapping +import com.twitter.util.validation.constraints.{ + CountryCode, + InvalidConstraint, + StateConstraint, + StateConstraintPayload, + ValidPassengerCount +} +import com.twitter.util.validation.conversions.ConstraintViolationOps._ +import com.twitter.util.validation.internal.AnnotationFactory +import com.twitter.util.validation.metadata.{CaseClassDescriptor, PropertyDescriptor} +import com.twitter.util.validation.internal.validators.{ + ISO3166CountryCodeConstraintValidator, + InvalidConstraintValidator, + NotEmptyAnyConstraintValidator, + NotEmptyPathConstraintValidator, + ValidPassengerCountConstraintValidator +} +import jakarta.validation.constraints.{Min, NotEmpty, Pattern, Size} +import jakarta.validation.{ + ConstraintValidator, + ConstraintViolation, + ConstraintViolationException, + UnexpectedTypeException, + Validation, + ValidationException +} +import java.lang.annotation.Annotation +import java.time.LocalDate +import java.util +import java.util.{UUID => JUUID} +import org.hibernate.validator.HibernateValidatorConfiguration +import org.json4s.reflect.{ClassDescriptor, ConstructorDescriptor, Reflector} +import org.scalatest.BeforeAndAfterAll +import scala.jdk.CollectionConverters._ + +private object ScalaValidatorTest { + val DefaultAddress: Address = + Address(line1 = "1234 Main St", city = "Anywhere", state = "CA", zipcode = "94102") + + val CustomConstraintMappings: Set[ConstraintMapping] = + Set( + ConstraintMapping( + classOf[ValidPassengerCount], + classOf[ValidPassengerCountConstraintValidator], + includeExistingValidators = false + ), + ConstraintMapping( + classOf[InvalidConstraint], + classOf[InvalidConstraintValidator] + ) + ) +} + +class ScalaValidatorTest extends AssertViolationTest with BeforeAndAfterAll { + import ScalaValidatorTest._ + + override protected val validator: ScalaValidator = + ScalaValidator.builder + .withConstraintMappings(CustomConstraintMappings) + .validator + + override protected def afterAll(): Unit = { + validator.close() + super.afterAll() + } + + test("ScalaValidator#with custom message interpolator") { + val withCustomMessageInterpolator = ScalaValidator.builder + .withMessageInterpolator(new WrongMessageInterpolator) + .validator + try { + val car = SmallCar(null, "DD-AB-123", 4) + val violations = withCustomMessageInterpolator.validate(car) + assertViolations( + violations = violations, + withViolations = Seq( + WithViolation( + "manufacturer", + "Whatever you entered was wrong", + null, + classOf[SmallCar], + car) + ) + ) + } finally { + withCustomMessageInterpolator.close() + } + } + + test("ScalaValidator#nested case class definition") { + val value = OuterObject.InnerObject.SuperNestedCaseClass(4L) + + val violations = validator.validate(value) + violations.size should equal(1) + val violation = violations.head + violation.getPropertyPath.toString should be("id") + violation.getMessage should be("must be greater than or equal to 5") + violation.getRootBeanClass should equal(classOf[OuterObject.InnerObject.SuperNestedCaseClass]) + violation.getRootBean should equal(value) + violation.getInvalidValue should equal(4L) + } + + test("ScalaValidator#unicode field name") { + val value = UnicodeNameCaseClass(5, "") + + assertViolations( + obj = value, + withViolations = Seq( + WithViolation( + "name", + "must not be empty", + "", + classOf[UnicodeNameCaseClass], + value, + getValidationAnnotations(classOf[UnicodeNameCaseClass], "name").lastOption + ), + WithViolation( + "winning-id", + "must be less than or equal to 3", + 5, + classOf[UnicodeNameCaseClass], + value, + getValidationAnnotations(classOf[UnicodeNameCaseClass], "winning-id").lastOption + ), + ) + ) + } + + test("ScalaValidator#correctly handle not cascading for a non-case class type") { + val value = WithBogusCascadedField("DD-AB-123", Seq(1, 2, 3)) + assertViolations(value) + } + + test("ScalaValidator#with InvalidConstraint") { + val value = AlwaysFails("fails") + + val e = intercept[RuntimeException] { + validator.validate(value) + } + e.getMessage should equal("validator foo error") + } + + test("ScalaValidator#with failing MethodValidation") { + // fails the @NotEmpty on the id field but the method validation blows up + val value = AlwaysFailsMethodValidation("") + + val e = intercept[ValidationException] { + validator.validate(value) + } + e.getMessage should equal("oh noes!") + } + + test("ScalaValidator#isConstraintValidator") { + val stateConstraint = + AnnotationFactory.newInstance(classOf[StateConstraint], Map.empty) + validator.isConstraintAnnotation(stateConstraint) should be(true) + } + + test("ScalaValidator#constraint mapping 1") { + val value = PathNotEmpty(Path.Empty, "abcd1234") + + // no validator registered `@NotEmpty` for Path type + intercept[UnexpectedTypeException] { + validator.validate(value) + } + + // define an additional constraint validator for `@NotEmpty` which works for + // Path types. note we include all existing validators + val pathCustomConstraintMapping = + ConstraintMapping(classOf[NotEmpty], classOf[NotEmptyPathConstraintValidator]) + + val withPathConstraintValidator: ScalaValidator = + ScalaValidator.builder + .withConstraintMapping(pathCustomConstraintMapping).validator + + try { + val violations = withPathConstraintValidator.validate(value) + violations.size should equal(1) + violations.head.getPropertyPath.toString should be("path") + violations.head.getMessage should be("must not be empty") + violations.head.getRootBeanClass should equal(classOf[PathNotEmpty]) + violations.head.getRootBean should equal(value) + violations.head.getInvalidValue should equal(Path.Empty) + + // valid instance + withPathConstraintValidator + .validate(PathNotEmpty(Path("node"), "abcd1234")) + .isEmpty should be(true) + } finally { + withPathConstraintValidator.close() + } + + // define an additional constraint validator for `@NotEmpty` which works for + // Path types. note we do include all existing validators which will cause validation to + // fail since we can no longer find a validator for `@NotEmpty` which works for string + val pathCustomConstraintMapping2 = + ConstraintMapping( + classOf[NotEmpty], + classOf[NotEmptyPathConstraintValidator], + includeExistingValidators = false) + + val withPathConstraintValidator2: ScalaValidator = + ScalaValidator.builder + .withConstraintMapping(pathCustomConstraintMapping2).validator + try { + intercept[UnexpectedTypeException] { + withPathConstraintValidator2.validate(value) + } + } finally { + withPathConstraintValidator2.close() + } + } + + test("ScalaValidator#constraint mapping 2") { + // attempting to add a mapping for the same annotation multiple times will result in an error + val pathCustomConstraintMapping1 = + ConstraintMapping(classOf[NotEmpty], classOf[NotEmptyPathConstraintValidator]) + val pathCustomConstraintMapping2 = + ConstraintMapping(classOf[NotEmpty], classOf[NotEmptyAnyConstraintValidator]) + + val e1 = intercept[ValidationException] { + ScalaValidator.builder + .withConstraintMappings(Set(pathCustomConstraintMapping1, pathCustomConstraintMapping2)) + .validator + } + e1.getMessage.endsWith( + "jakarta.validation.constraints.NotEmpty is configured more than once via the programmatic constraint definition API.") should be( + true) + + val pathCustomConstraintMapping3 = + ConstraintMapping(classOf[NotEmpty], classOf[NotEmptyPathConstraintValidator]) + val pathCustomConstraintMapping4 = + ConstraintMapping( + classOf[NotEmpty], + classOf[NotEmptyAnyConstraintValidator], + includeExistingValidators = false) + + val e2 = intercept[ValidationException] { + ScalaValidator.builder + .withConstraintMappings(Set(pathCustomConstraintMapping3, pathCustomConstraintMapping4)) + .validator + } + + e2.getMessage.endsWith( + "jakarta.validation.constraints.NotEmpty is configured more than once via the programmatic constraint definition API.") should be( + true) + + val pathCustomConstraintMapping5 = + ConstraintMapping( + classOf[NotEmpty], + classOf[NotEmptyPathConstraintValidator], + includeExistingValidators = false) + val pathCustomConstraintMapping6 = + ConstraintMapping(classOf[NotEmpty], classOf[NotEmptyAnyConstraintValidator]) + + val e3 = intercept[ValidationException] { + ScalaValidator.builder + .withConstraintMappings(Set(pathCustomConstraintMapping5, pathCustomConstraintMapping6)) + .validator + } + + e3.getMessage.endsWith( + "jakarta.validation.constraints.NotEmpty is configured more than once via the programmatic constraint definition API.") should be( + true) + + val pathCustomConstraintMapping7 = + ConstraintMapping( + classOf[NotEmpty], + classOf[NotEmptyPathConstraintValidator], + includeExistingValidators = false) + val pathCustomConstraintMapping8 = + ConstraintMapping( + classOf[NotEmpty], + classOf[NotEmptyAnyConstraintValidator], + includeExistingValidators = false) + + val e4 = intercept[ValidationException] { + ScalaValidator.builder + .withConstraintMappings(Set(pathCustomConstraintMapping7, pathCustomConstraintMapping8)) + .validator + } + + e4.getMessage.endsWith( + "jakarta.validation.constraints.NotEmpty is configured more than once via the programmatic constraint definition API.") should be( + true) + } + + test("ScalaValidator#payload") { + val address = + Address(line1 = "1234 Main St", city = "Anywhere", state = "PA", zipcode = "94102") + val violations = validator.validate(address) + violations.size should equal(1) + val violation = violations.head + val payload = violation.getDynamicPayload(classOf[StateConstraintPayload]).orNull + payload == null should be(false) + payload.isInstanceOf[StateConstraintPayload] should be(true) + payload should equal(StateConstraintPayload("PA", "CA")) + } + + test("ScalaValidator#method validation payload") { + val user = NestedUser( + id = "abcd1234", + person = Person("abcd1234", "R. Franklin", DefaultAddress), + gender = "F", + job = "" + ) + val violations = validator.validate(user) + violations.size should equal(1) + val violation = violations.head + val payload = violation.getDynamicPayload(classOf[NestedUserPayload]).orNull + payload == null should be(false) + payload.isInstanceOf[NestedUserPayload] should be(true) + payload should equal(NestedUserPayload("abcd1234", "")) + } + + test("ScalaValidator#newInstance") { + val stateConstraint = + AnnotationFactory.newInstance(classOf[StateConstraint], Map.empty) + validator.isConstraintAnnotation(stateConstraint) should be(true) + + val minConstraint = AnnotationFactory.newInstance( + classOf[Min], + Map("value" -> 1L) + ) + minConstraint.groups().isEmpty should be(true) + minConstraint.message() should equal("{jakarta.validation.constraints.Min.message}") + minConstraint.payload().isEmpty should be(true) + minConstraint.value() should equal(1L) + validator.isConstraintAnnotation(minConstraint) should be(true) + + val patternConstraint = AnnotationFactory.newInstance( + classOf[Pattern], + Map("regexp" -> ".*", "flags" -> Array[Pattern.Flag](Pattern.Flag.CASE_INSENSITIVE)) + ) + patternConstraint.groups().isEmpty should be(true) + patternConstraint.message() should equal("{jakarta.validation.constraints.Pattern.message}") + patternConstraint.payload().isEmpty should be(true) + patternConstraint.regexp() should equal(".*") + patternConstraint.flags() should equal(Array[Pattern.Flag](Pattern.Flag.CASE_INSENSITIVE)) + validator.isConstraintAnnotation(patternConstraint) should be(true) + } + + test("ScalaValidator#constraint validators by annotation class") { + val validators = validator.findConstraintValidators(classOf[CountryCode]) + validators.size should equal(1) + validators.head.getClass.getName should be( + classOf[ISO3166CountryCodeConstraintValidator].getName) + + validators.head + .asInstanceOf[ConstraintValidator[CountryCode, String]].isValid("US", null) should be(true) + } + + test("ScalaValidator#nulls") { + val car = SmallCar(null, "DD-AB-123", 4) + assertViolations( + obj = car, + withViolations = Seq( + WithViolation("manufacturer", "must not be null", null, classOf[SmallCar], car) + ) + ) + } + + test("ScalaValidator#not a case class") { + val value = new NotACaseClass(null, "DD-AB-123", 4, "NotAUUID") + val e = intercept[ValidationException] { + validator.validate(value) + } + e.getMessage should equal(s"${classOf[NotACaseClass]} is not a valid case class.") + + // underlying validator doesn't find the Scala annotations (either executable nor inherited) + val violations = validator.underlying.validate(value).asScala.toSet + violations.isEmpty should be(true) + } + + test("ScalaValidator#works with meta annotation 1") { + val value = WithMetaFieldAnnotation("") + + assertViolations( + obj = value, + withViolations = Seq( + WithViolation( + "id", + "must not be empty", + "", + classOf[WithMetaFieldAnnotation], + value, + getValidationAnnotations(classOf[WithMetaFieldAnnotation], "id").headOption + ) + ) + ) + + // also works with underlying validator + val violations = validator.underlying.validate(value).asScala.toSet + violations.size should equal(1) + val violation = violations.head + violation.getMessage should equal("must not be empty") + violation.getPropertyPath.toString should equal("id") + violation.getInvalidValue should equal("") + violation.getRootBeanClass should equal(classOf[WithMetaFieldAnnotation]) + violation.getRootBean should equal(value) + violation.getLeafBean should equal(value) + } + + test("ScalaValidator#works with meta annotation 2") { + val value = WithPartialMetaFieldAnnotation("") + + assertViolations( + obj = value, + withViolations = Seq( + WithViolation( + "id", + "must not be empty", + "", + classOf[WithPartialMetaFieldAnnotation], + value, + getValidationAnnotations(classOf[WithPartialMetaFieldAnnotation], "id").lastOption + ), + WithViolation( + "id", + "size must be between 2 and 10", + "", + classOf[WithPartialMetaFieldAnnotation], + value, + getValidationAnnotations(classOf[WithPartialMetaFieldAnnotation], "id").headOption + ) + ) + ) + } + + test("ScalaValidator#validateValue 1") { + val violations = validator.validateValue(classOf[OneOfSeqExample], "enumValue", Seq("g")) + violations.size should equal(1) + val violation = violations.head + violation.getMessage should equal("g not one of [a, B, c]") + violation.getPropertyPath.toString should equal("enumValue") + violation.getInvalidValue should equal(Seq("g")) + violation.getRootBeanClass should equal(classOf[OneOfSeqExample]) + violation.getRootBean should be(null) + violation.getLeafBean should be(null) + } + + test("ScalaValidator#validateValue 2") { + val descriptor = validator.getConstraintsForClass(classOf[OneOfSeqExample]) + val violations = validator.validateValue(descriptor, "enumValue", Seq("g")) + violations.size should equal(1) + val violation = violations.head + violation.getMessage should equal("g not one of [a, B, c]") + violation.getPropertyPath.toString should equal("enumValue") + violation.getInvalidValue should equal(Seq("g")) + violation.getRootBeanClass should equal(classOf[OneOfSeqExample]) + violation.getRootBean should be(null) + violation.getLeafBean should be(null) + } + + test("ScalaValidator#validateProperty 1") { + val value = CustomerAccount("", None) + val violations = validator.validateProperty(value, "accountName") + violations.size should equal(1) + val violation = violations.head + violation.getMessage should equal("must not be empty") + violation.getPropertyPath.toString should equal("accountName") + violation.getInvalidValue should equal("") + violation.getRootBeanClass should equal(classOf[CustomerAccount]) + violation.getRootBean should equal(value) + violation.getLeafBean should equal(value) + } + + test("ScalaValidator#validateProperty 2") { + validator + .validateProperty( + MinIntExample(numberValue = 2), + "numberValue" + ).isEmpty should be(true) + assertViolations(obj = MinIntExample(numberValue = 2)) + + val invalid = MinIntExample(numberValue = 0) + val violations = validator + .validateProperty( + invalid, + "numberValue" + ) + violations.size should equal(1) + val violation = violations.head + violation.getPropertyPath.toString should equal("numberValue") + violation.getMessage should equal("must be greater than or equal to 1") + violation.getRootBeanClass should equal(classOf[MinIntExample]) + violation.getRootBean.getClass.getName should equal(invalid.getClass.getName) + + assertViolations( + obj = invalid, + withViolations = Seq( + WithViolation( + "numberValue", + "must be greater than or equal to 1", + 0, + classOf[MinIntExample], + invalid, + getValidationAnnotations(classOf[MinIntExample], "numberValue").headOption + ) + ) + ) + } + + test("ScalaValidator#validateFieldValue") { + val constraints: Map[Class[_ <: Annotation], Map[String, Any]] = + Map(classOf[jakarta.validation.constraints.Size] -> Map("min" -> 5, "max" -> 7)) + + // size doesn't work for ints + intercept[UnexpectedTypeException] { + validator.validateFieldValue(constraints, "intField", 4) + } + + // should work fine for collections of the right size + validator + .validateFieldValue(constraints, "javaSetIntField", newJavaSet(6)).isEmpty should be(true) + validator + .validateFieldValue(constraints, "seqIntField", Seq(1, 2, 3, 4, 5, 6)).isEmpty should be(true) + validator + .validateFieldValue(constraints, "setIntField", Set(1, 2, 3, 4, 5, 6)).isEmpty should be(true) + validator + .validateFieldValue(constraints, "arrayIntField", Array(1, 2, 3, 4, 5, 6)).isEmpty should be( + true) + validator + .validateFieldValue(constraints, "stringField", "123456").isEmpty should be(true) + + // will fail for types of the wrong size + val violations = + validator.validateFieldValue(constraints, "seqIntField", Seq(1, 2, 3)) + violations.size should equal(1) + val violation: ConstraintViolation[Any] = violations.head + violation.getMessage should equal("size must be between 5 and 7") + violation.getPropertyPath.toString should equal("seqIntField") + violation.getInvalidValue should equal(Seq(1, 2, 3)) + violation.getRootBeanClass should equal(classOf[AnyRef]) + violation.getRootBean == null should be(true) + violation.getLeafBean should be(null) + + val invalidJavaSet = newJavaSet(3) + val violations2 = validator + .validateFieldValue(constraints, "javaSetIntField", invalidJavaSet) + violations2.size should equal(1) + val violation2: ConstraintViolation[Any] = violations.head + violation2.getMessage should equal("size must be between 5 and 7") + violation2.getPropertyPath.toString should equal("seqIntField") + violation2.getInvalidValue should equal(List(1, 2, 3)) + violation2.getRootBeanClass should equal(classOf[AnyRef]) + violation2.getRootBean == null should be(true) + violation2.getLeafBean should be(null) + + val constraintsWithGroup: Map[Class[_ <: Annotation], Map[String, Any]] = + Map( + classOf[jakarta.validation.constraints.Size] -> Map( + "min" -> 5, + "max" -> 7, + "groups" -> Array(classOf[PersonCheck]) // groups MUST be an array type + ) + ) + + // constraint group is not passed (not activated) + validator + .validateFieldValue(constraintsWithGroup, "seqIntField", Seq(1, 2, 3)).isEmpty should be(true) + // constraint group is passed (activated) + val violationsWithGroup = validator + .validateFieldValue(constraintsWithGroup, "seqIntField", Seq(1, 2, 3), classOf[PersonCheck]) + violationsWithGroup.size should equal(1) + val violationWg = violationsWithGroup.head + violationWg.getMessage should equal("size must be between 5 and 7") + violationWg.getPropertyPath.toString should equal("seqIntField") + violationWg.getInvalidValue should equal(Seq(1, 2, 3)) + violationWg.getRootBeanClass should equal(classOf[AnyRef]) + violationWg.getRootBean == null should be(true) + violationWg.getLeafBean should be(null) + + val violationsSt = + validator.validateFieldValue(constraints, "setIntField", Set(1, 2, 3, 4, 5, 6, 7, 8)) + violationsSt.size should equal(1) + val violationSt = violationsSt.head + violationSt.getMessage should equal("size must be between 5 and 7") + violationSt.getPropertyPath.toString should equal("setIntField") + violationSt.getInvalidValue should equal(Set(1, 2, 3, 4, 5, 6, 7, 8)) + violationSt.getRootBeanClass should equal(classOf[AnyRef]) + violationSt.getRootBean == null should be(true) + violationSt.getLeafBean should be(null) + + validator + .validateFieldValue( + Map[Class[_ <: Annotation], Map[String, AnyRef]](classOf[NotEmpty] + .asInstanceOf[Class[_ <: Annotation]] -> Map[String, AnyRef]()), + "data", + "Hello, world").isEmpty should be(true) + } + + test("ScalaValidator#groups support") { + // see: https://docs.jboss.org/hibernate/stable/scalaValidator/reference/en-US/html_single/#chapter-groups + val toTest = WithPersonCheck("", "Jane Doe") + // PersonCheck group not enabled, no violations + assertViolations(obj = toTest) + // Completely diff group enabled, no violations + validator.validate(obj = toTest, groups = classOf[OtherCheck]) + // PersonCheck group enabled + assertViolations( + obj = toTest, + groups = Seq(classOf[PersonCheck]), + withViolations = Seq( + WithViolation("id", "must not be empty", "", classOf[WithPersonCheck], toTest) + ) + ) + + // multiple groups with PersonCheck group enabled + assertViolations( + obj = toTest, + groups = Seq(classOf[OtherCheck], classOf[PersonCheck]), + withViolations = Seq( + WithViolation("id", "must not be empty", "", classOf[WithPersonCheck], toTest) + ) + ) + } + + test("ScalaValidator#isCascaded method validation - defined fields") { + // nested method validation fields + val owner = Person(id = "", name = "A. Einstein", address = DefaultAddress) + val car = Car( + id = 1234, + make = CarMake.Volkswagen, + model = "Beetle", + year = 1970, + owners = Seq(Person(id = "9999", name = "", address = DefaultAddress)), + licensePlate = "CA123", + numDoors = 2, + manual = true, + ownershipStart = LocalDate.now, + ownershipEnd = LocalDate.now.plusYears(10), + warrantyEnd = Some(LocalDate.now.plusYears(3)), + passengers = Seq( + Person(id = "1001", name = "R. Franklin", address = DefaultAddress) + ) + ) + val driver = Driver( + person = owner, + car = car + ) + + // no PersonCheck -- doesn't fail + assertViolations(obj = driver, groups = Seq(classOf[PersonCheck])) + // with default group - fail + assertViolations( + obj = driver, + withViolations = Seq( + WithViolation("car.owners[0].name", "must not be empty", "", classOf[Driver], driver), + WithViolation("car.validateId", "id may not be even", car, classOf[Driver], driver), + WithViolation( + "car.warrantyTimeValid.warrantyEnd", + "both warrantyStart and warrantyEnd are required for a valid range", + car, + classOf[Driver], + driver), + WithViolation( + "car.warrantyTimeValid.warrantyStart", + "both warrantyStart and warrantyEnd are required for a valid range", + car, + classOf[Driver], + driver), + WithViolation( + "car.year", + "must be greater than or equal to 2000", + 1970, + classOf[Driver], + driver), + WithViolation("person.id", "must not be empty", "", classOf[Driver], driver) + ) + ) + + // Validation with the default group does not return any violations with the standard + // Hibernate Validator because Scala only places annotations the executable fields of the + // case class by default and does copy them to the generated fields with the @field meta annotation. + Validation + .buildDefaultValidatorFactory() + .getValidator + .validate(driver) + .asScala + .isEmpty should be(true) + + val carWithInvalidNumberOfPassengers = Car( + id = 1235, + make = CarMake.Audi, + model = "A4", + year = 2010, + owners = Seq(Person(id = "9999", name = "M Bailey", address = DefaultAddress)), + licensePlate = "CA123", + numDoors = 2, + manual = true, + ownershipStart = LocalDate.now, + ownershipEnd = LocalDate.now.plusYears(10), + warrantyStart = Some(LocalDate.now), + warrantyEnd = Some(LocalDate.now.plusYears(3)), + passengers = Seq.empty + ) + val anotherDriver = Driver( + person = Person(id = "55555", name = "K. Brown", address = DefaultAddress), + car = carWithInvalidNumberOfPassengers + ) + + // ensure the class-level Car validation reports the correct property path of the field name + assertViolations( + obj = anotherDriver, + withViolations = Seq( + WithViolation( + "car", + "number of passenger(s) is not valid", + carWithInvalidNumberOfPassengers, + classOf[Driver], + anotherDriver) + ) + ) + } + + test("ScalaValidator#class-level annotations") { + // The Car case class is annotated with @ValidPassengerCount which checks if the number of + // passengers is greater than zero and less then some max specified in the annotation. + // In this test, no passengers are defined so the validation fails the class-level constraint. + // see: https://docs.jboss.org/hibernate/stable/scalaValidator/reference/en-US/html_single/#scalaValidator-usingscalaValidator-classlevel + val car = Car( + id = 1234, + make = CarMake.Volkswagen, + model = "Beetle", + year = 2004, + owners = Seq(Person(id = "9999", name = "K. Ann", address = DefaultAddress)), + licensePlate = "CA123", + numDoors = 2, + manual = true, + ownershipStart = LocalDate.now, + ownershipEnd = LocalDate.now.plusYears(10), + warrantyEnd = Some(LocalDate.now.plusYears(3)) + ) + + assertViolations( + obj = car, + withViolations = Seq( + WithViolation("", "number of passenger(s) is not valid", car, classOf[Car], car), + WithViolation("validateId", "id may not be even", car, classOf[Car], car), + WithViolation( + "warrantyTimeValid.warrantyEnd", + "both warrantyStart and warrantyEnd are required for a valid range", + car, + classOf[Car], + car), + WithViolation( + "warrantyTimeValid.warrantyStart", + "both warrantyStart and warrantyEnd are required for a valid range", + car, + classOf[Car], + car) + ) + ) + + // compare the class-level violation which should be the same as from the ScalaValidator + val fromUnderlyingViolations = validator.underlying.validate(car).asScala.toSet + fromUnderlyingViolations.size should equal(1) + val violation = fromUnderlyingViolations.head + violation.getPropertyPath.toString should equal("") + violation.getMessage should equal("number of passenger(s) is not valid") + violation.getRootBeanClass should equal(classOf[Car]) + violation.getInvalidValue should equal(car) + violation.getRootBean should equal(car) + + // Validation with the default group does not return any violations with the standard + // Hibernate Validator because Scala only places annotations the executable fields of the + // case class by default and does copy them to the generated fields with the @field meta annotation. + // However, class-level annotations will work. + val configuration = Validation + .byDefaultProvider() + .configure().asInstanceOf[HibernateValidatorConfiguration] + val mapping = configuration.createConstraintMapping() + mapping + .constraintDefinition(classOf[ValidPassengerCount]) + .validatedBy(classOf[ValidPassengerCountConstraintValidator]) + configuration.addMapping(mapping) + + val hibernateViolations = configuration.buildValidatorFactory.getValidator + .validate(car) + .asScala + // only has class-level constraint violation + val classLevelConstraintViolation = hibernateViolations.head + classLevelConstraintViolation.getPropertyPath.toString should be("") + classLevelConstraintViolation.getMessage should equal("number of passenger(s) is not valid") + classLevelConstraintViolation.getRootBeanClass should equal(classOf[Car]) + classLevelConstraintViolation.getRootBean == car should be(true) + + } + + test("ScalaValidator#cascaded validations") { + val validUser: User = User("1234567", "ion", "Other") + val invalidUser: User = User("", "anion", "M") + val nestedValidUser: Users = Users(Seq(validUser)) + val nestedInvalidUser: Users = Users(Seq(invalidUser)) + val nestedDuplicateUser: Users = Users(Seq(validUser, validUser)) + + assertViolations(obj = validUser) + assertViolations( + obj = invalidUser, + withViolations = Seq( + WithViolation("id", "must not be empty", "", classOf[User], invalidUser) + ) + ) + + assertViolations(obj = nestedValidUser) + assertViolations( + obj = nestedInvalidUser, + withViolations = Seq( + WithViolation("users[0].id", "must not be empty", "", classOf[Users], nestedInvalidUser)) + ) + + assertViolations(obj = nestedDuplicateUser) + } + + /* + * Builder withDescriptorCacheSize(...) tests + */ + test("ScalaValidator#withDescriptorCacheSize should override the default cache size") { + val customizedCacheSize: Long = 512 + val scalaValidator = ScalaValidator.builder + .withDescriptorCacheSize(customizedCacheSize) + scalaValidator.descriptorCacheSize should be(customizedCacheSize) + } + + test("ScalaValidator#validate is valid") { + val testUser: User = User(id = "9999", name = "April", gender = "F") + validator.validate(testUser).isEmpty should be(true) + } + + test("ScalaValidator#validate returns valid result even when case class has other fields") { + val testPerson: Person = Person(id = "9999", name = "April", address = DefaultAddress) + validator.validate(testPerson).isEmpty should be(true) + } + + test("ScalaValidator#validate is invalid") { + val testUser: User = User(id = "", name = "April", gender = "F") + val fieldAnnotation = getValidationAnnotations(classOf[User], "id").headOption + + assertViolations( + obj = testUser, + withViolations = Seq( + WithViolation("id", "must not be empty", "", classOf[User], testUser, fieldAnnotation) + ) + ) + } + + test( + "ScalaValidator#validate returns invalid result of both field validations and method validations") { + val testUser: User = User(id = "", name = "", gender = "Female") + + val idAnnotation: Option[Annotation] = + getValidationAnnotations(classOf[User], "id").headOption + val genderAnnotation: Option[Annotation] = + getValidationAnnotations(classOf[User], "gender").headOption + val methodAnnotation: Option[Annotation] = + getValidationAnnotations(classOf[User], "nameCheck").headOption + + assertViolations( + obj = testUser, + withViolations = Seq( + WithViolation( + "gender", + "Female not one of [F, M, Other]", + "Female", + classOf[User], + testUser, + genderAnnotation), + WithViolation("id", "must not be empty", "", classOf[User], testUser, idAnnotation), + WithViolation( + "nameCheck.name", + "cannot be empty", + testUser, + classOf[User], + testUser, + methodAnnotation) + ) + ) + } + + test("ScalaValidator#should register the user defined constraint validator") { + val testState: StateValidationExample = StateValidationExample(state = "NY") + val fieldAnnotation: Option[Annotation] = + getValidationAnnotations(classOf[StateValidationExample], "state").headOption + + assertViolations( + obj = testState, + withViolations = Seq( + WithViolation( + "state", + "Please register with state CA", + "NY", + classOf[StateValidationExample], + testState, + fieldAnnotation) + ) + ) + } + + test("ScalaValidator#secondary case class constructors") { + // the framework does not validate on construction, however, it must use + // the case class executable for finding the field validation annotations + // as Scala carries case class annotations on the executable for fields specified + // and annotated in the executable. Validations only apply to executable parameters that are + // also class fields, e.g., from the primary executable. + assertViolations(obj = TestJsonCreator("42")) + assertViolations(obj = TestJsonCreator(42)) + assertViolations(obj = TestJsonCreator2(scala.collection.immutable.Seq("1", "2", "3"))) + assertViolations( + obj = TestJsonCreator2(scala.collection.immutable.Seq(1, 2, 3), default = "Goodbye, world")) + + assertViolations(obj = TestJsonCreatorWithValidation("42")) + intercept[NumberFormatException] { + // can't validate after the fact -- the instance is already constructed, then we validate + assertViolations(obj = TestJsonCreatorWithValidation("")) + } + assertViolations(obj = TestJsonCreatorWithValidation(42)) + + // annotations are on primary executable + assertViolations( + obj = TestJsonCreatorWithValidations("99"), + withViolations = Seq( + WithViolation( + "int", + "99 not one of [42, 137]", + 99, + classOf[TestJsonCreatorWithValidations], + TestJsonCreatorWithValidations("99") + ) + ) + ) + assertViolations(obj = TestJsonCreatorWithValidations(42)) + + assertViolations(obj = new CaseClassWithMultipleConstructors("10001", "20002", "30003")) + assertViolations(obj = CaseClassWithMultipleConstructors(10001L, 20002L, 30003L)) + + assertViolations(obj = CaseClassWithMultipleConstructorsAnnotated(10001L, 20002L, 30003L)) + assertViolations(obj = + new CaseClassWithMultipleConstructorsAnnotated("10001", "20002", "30003")) + + assertViolations( + obj = CaseClassWithMultipleConstructorsAnnotatedAndValidations( + 10001L, + 20002L, + JUUID.randomUUID().toString)) + + assertViolations( + obj = new CaseClassWithMultipleConstructorsAnnotatedAndValidations( + "10001", + "20002", + JUUID.randomUUID().toString)) + + val invalid = new CaseClassWithMultipleConstructorsPrimaryAnnotatedAndValidations( + "9999", + "10001", + JUUID.randomUUID().toString) + assertViolations( + obj = invalid, + withViolations = Seq( + WithViolation( + "number1", + "must be greater than or equal to 10000", + 9999, + classOf[CaseClassWithMultipleConstructorsPrimaryAnnotatedAndValidations], + invalid + ) + ) + ) + + val d = InvalidDoublePerson("Andrea")("") + assertViolations(d) + val d2 = DoublePerson("Andrea")("") + assertViolations( + obj = d2, + withViolations = + Seq(WithViolation("otherName", "must not be empty", "", classOf[DoublePerson], d2))) + val d3 = ValidDoublePerson("Andrea")("") + val ve = intercept[ValidationException] { + validator.verify(d3) + } + ve.isInstanceOf[ConstraintViolationException] should be(true) + val cve = ve.asInstanceOf[ConstraintViolationException] + cve.getConstraintViolations.size should equal(2) + cve.getMessage should equal( + "\nValidation Errors:\t\tcheckOtherName: otherName must be longer than 3 chars, otherName: must not be empty\n\n") + + val d4 = PossiblyValidDoublePerson("Andrea")("") + assertViolations( + obj = d4, + withViolations = Seq( + WithViolation("otherName", "must not be empty", "", classOf[PossiblyValidDoublePerson], d4)) + ) + + val d5 = WithFinalValField( + JUUID.randomUUID().toString, + Some("Joan"), + Some("Jett"), + Some(Set(1, 2, 3)) + ) + assertViolations(d5) + } + + test("ScalaValidator#cycles") { + validator.validate(A("5678", B("9876", C("444", null)))).isEmpty should be(true) + validator.validate(D("1", E("2", F("3", None)))).isEmpty should be(true) + validator.validate(G("1", H("2", I("3", Seq.empty[G])))).isEmpty should be(true) + } + + test("ScalaValidator#scala BeanProperty") { + val value = AnnotatedBeanProperties(3) + value.setField1("") + assertViolations( + obj = value, + withViolations = Seq( + WithViolation("field1", "must not be empty", "", classOf[AnnotatedBeanProperties], value)) + ) + } + + test("ScalaValidator#case class with no executable params") { + val value = NoConstructorParams() + value.setId("1234") + assertViolations(value) + + assertViolations( + obj = NoConstructorParams(), + withViolations = Seq( + WithViolation( + "id", + "must not be empty", + "", + classOf[NoConstructorParams], + NoConstructorParams())) + ) + } + + test("ScalaValidator#case class annotated non executable field") { + assertViolations( + obj = AnnotatedInternalFields("1234", "thisisabigstring"), + withViolations = Seq( + WithViolation( + "company", + "must not be empty", + "", + classOf[AnnotatedInternalFields], + AnnotatedInternalFields("1234", "thisisabigstring")) + ) + ) + } + + test("ScalaValidator#inherited validation annotations") { + assertViolations( + obj = ImplementsAncestor(""), + withViolations = Seq( + WithViolation( + "field1", + "must not be empty", + "", + classOf[ImplementsAncestor], + ImplementsAncestor("")), + WithViolation( + "validateField1.field1", + "not a double value", + ImplementsAncestor(""), + classOf[ImplementsAncestor], + ImplementsAncestor("")) + ) + ) + + assertViolations( + obj = ImplementsAncestor("blimey"), + withViolations = Seq( + WithViolation( + "validateField1.field1", + "not a double value", + ImplementsAncestor("blimey"), + classOf[ImplementsAncestor], + ImplementsAncestor("blimey") + ) + ) + ) + + assertViolations(obj = ImplementsAncestor("3.141592653589793d")) + } + + test("ScalaValidator#getConstraintsForClass") { + // there is no resolution of nested types in a description + val driverCaseClassDescriptor = validator.getConstraintsForClass(classOf[Driver]) + driverCaseClassDescriptor.members.size should equal(2) + + verifyDriverClassDescriptor(driverCaseClassDescriptor) + } + + test("ScalaValidator#annotations") { + val caseClassDescriptor = validator.descriptorFactory.describe(classOf[TestAnnotations]) + caseClassDescriptor.members.size should equal(1) + val (name, memberDescriptor) = caseClassDescriptor.members.head + name should equal("cars") + memberDescriptor.isInstanceOf[PropertyDescriptor] should be(true) + memberDescriptor.asInstanceOf[PropertyDescriptor].annotations.length should equal(1) + memberDescriptor + .asInstanceOf[PropertyDescriptor].annotations.head.annotationType() should be( + classOf[NotEmpty]) + } + + test("ScalaValidator#generics 1") { + val caseClassDescriptor = + validator.descriptorFactory.describe(classOf[GenericTestCaseClass[_]]) + caseClassDescriptor.members.size should equal(1) + val value = GenericTestCaseClass(data = "Hello, World") + val results = validator.validate(value) + results.isEmpty should be(true) + + intercept[UnexpectedTypeException] { + validator.validate(GenericTestCaseClass(3)) + } + + val genericValue = GenericTestCaseClass[Seq[String]](Seq.empty[String]) + val violations = validator.validate(genericValue) + violations.size should equal(1) + val violation = violations.head + violation.getMessage should equal("must not be empty") + violation.getPropertyPath.toString should equal("data") + violation.getRootBeanClass should equal(classOf[GenericTestCaseClass[Seq[String]]]) + violation.getLeafBean should equal(genericValue) + violation.getInvalidValue should equal(Seq.empty[String]) + + validator.validate(GenericMinTestCaseClass(5)).isEmpty should be(true) + + val caseClassDescriptor1 = + validator.descriptorFactory.describe(classOf[GenericTestCaseClassMultipleTypes[_, _, _]]) + caseClassDescriptor1.members.size should equal(3) + + val value1 = GenericTestCaseClassMultipleTypes( + data = None, + things = Seq(1, 2, 3), + otherThings = Seq( + UUIDExample("1234"), + UUIDExample(JUUID.randomUUID().toString), + UUIDExample(JUUID.randomUUID().toString)) + ) + val results1 = validator.validate(value1) + results1.isEmpty should be(true) + } + + test("ScalaValidator#generics 2") { + val value: Page[Person] = Page[Person]( + List( + Person(id = "9999", name = "April", address = DefaultAddress), + Person(id = "9999", name = "April", address = DefaultAddress) + ), + 0, + None, + None) + + val caseClassDescriptor = + validator.descriptorFactory.describe(classOf[Page[Person]]) + caseClassDescriptor.members.size should equal(4) + + val violations = validator.validate(value) + violations.size should equal(1) + + val violation = violations.head + violation.getPropertyPath.toString should be("pageSize") + violation.getMessage should be("must be greater than or equal to 1") + violation.getRootBeanClass should equal(classOf[Page[Person]]) + violation.getRootBean should equal(value) + violation.getInvalidValue should equal(0) + } + + test("ScalaValidator#test boxed primitives") { + validator + .validateValue( + classOf[CaseClassWithBoxedPrimitives], + "events", + java.lang.Integer.valueOf(42)).isEmpty should be(true) + } + + test("ScalaValidator#find executable 1") { + /* CaseClassWithMultipleConstructorsAnnotatedAndValidations */ + val clazzDescriptor: ClassDescriptor = + Reflector + .describe(classOf[CaseClassWithMultipleConstructorsAnnotatedAndValidations]) + .asInstanceOf[ClassDescriptor] + + val constructorDescriptor: ConstructorDescriptor = + validator.descriptorFactory.findConstructor(clazzDescriptor /*, Seq.empty*/ ) + + // by default this will find Seq(Long, Long, String) + val constructorParameterTypes: Array[Class[_]] = + constructorDescriptor.constructor.getParameterTypes() + constructorParameterTypes.length should equal(3) + constructorParameterTypes(0) should equal(classOf[Long]) + constructorParameterTypes(1) should equal(classOf[Long]) + constructorParameterTypes(2) should equal(classOf[String]) + + val caseClazzDescriptor = validator.getConstraintsForClass( + classOf[CaseClassWithMultipleConstructorsAnnotatedAndValidations]) + caseClazzDescriptor.members.size should equal(3) + caseClazzDescriptor.members.contains("number1") should be(true) + caseClazzDescriptor.members("number1").annotations.isEmpty should be(true) + caseClazzDescriptor.members.contains("number2") should be(true) + caseClazzDescriptor.members("number2").annotations.isEmpty should be(true) + caseClazzDescriptor.members.contains("uuid") should be(true) + caseClazzDescriptor.members("uuid").annotations.isEmpty should be(true) + } + + test("ScalaValidator#find executable 2") { + /* CaseClassWithMultipleConstructorsPrimaryAnnotatedAndValidations */ + val clazzDescriptor = Reflector + .describe(classOf[CaseClassWithMultipleConstructorsPrimaryAnnotatedAndValidations]) + .asInstanceOf[ClassDescriptor] + + val constructorDescriptor = + validator.descriptorFactory.findConstructor(clazzDescriptor) + + // by default this will find Seq(Long, Long, String) + val constructorParameterTypes: Array[Class[_]] = + constructorDescriptor.constructor.getParameterTypes() + constructorParameterTypes.length should equal(3) + constructorParameterTypes(0) should equal(classOf[Long]) + constructorParameterTypes(1) should equal(classOf[Long]) + constructorParameterTypes(2) should equal(classOf[String]) + + val caseClazzDescriptor = validator.getConstraintsForClass( + classOf[CaseClassWithMultipleConstructorsPrimaryAnnotatedAndValidations]) + caseClazzDescriptor.members.size should equal(3) + caseClazzDescriptor.members.contains("number1") should be(true) + caseClazzDescriptor.members("number1").annotations.length should equal(1) + caseClazzDescriptor.members.contains("number2") should be(true) + caseClazzDescriptor.members("number2").annotations.length should equal(1) + caseClazzDescriptor.members.contains("uuid") should be(true) + caseClazzDescriptor.members("uuid").annotations.length should equal(1) + } + + test("ScalaValidator#find executable 3") { + /* CaseClassWithMultipleConstructorsAnnotated */ + val clazzDescriptor = Reflector + .describe(classOf[CaseClassWithMultipleConstructorsAnnotated]) + .asInstanceOf[ClassDescriptor] + + val constructorDescriptor = + validator.descriptorFactory.findConstructor(clazzDescriptor) + + // this should find Seq(Long, Long, Long) + val constructorParameterTypes: Array[Class[_]] = + constructorDescriptor.constructor.getParameterTypes() + constructorParameterTypes.length should equal(3) + constructorParameterTypes(0) should equal(classOf[Long]) + constructorParameterTypes(1) should equal(classOf[Long]) + constructorParameterTypes(2) should equal(classOf[Long]) + + val caseClazzDescriptor = + validator.getConstraintsForClass(classOf[CaseClassWithMultipleConstructorsAnnotated]) + caseClazzDescriptor.members.size should equal(3) + caseClazzDescriptor.members.contains("number1") should be(true) + caseClazzDescriptor.members("number1").annotations.isEmpty should be(true) + caseClazzDescriptor.members.contains("number2") should be(true) + caseClazzDescriptor.members("number2").annotations.isEmpty should be(true) + caseClazzDescriptor.members.contains("number3") should be(true) + caseClazzDescriptor.members("number3").annotations.isEmpty should be(true) + } + + test("ScalaValidator#find executable 4") { + /* CaseClassWithMultipleConstructors */ + val clazzDescriptor = Reflector + .describe(classOf[CaseClassWithMultipleConstructors]) + .asInstanceOf[ClassDescriptor] + + val constructorDescriptor = + validator.descriptorFactory.findConstructor(clazzDescriptor) + + // this should find Seq(Long, Long, Long) + val constructorParameterTypes: Array[Class[_]] = + constructorDescriptor.constructor.getParameterTypes() + constructorParameterTypes.length should equal(3) + constructorParameterTypes(0) should equal(classOf[Long]) + constructorParameterTypes(1) should equal(classOf[Long]) + constructorParameterTypes(2) should equal(classOf[Long]) + + val caseClazzDescriptor = + validator.getConstraintsForClass(classOf[CaseClassWithMultipleConstructors]) + caseClazzDescriptor.members.size should equal(3) + caseClazzDescriptor.members.contains("number1") should be(true) + caseClazzDescriptor.members("number1").annotations.isEmpty should be(true) + caseClazzDescriptor.members.contains("number2") should be(true) + caseClazzDescriptor.members("number2").annotations.isEmpty should be(true) + caseClazzDescriptor.members.contains("number3") should be(true) + caseClazzDescriptor.members("number3").annotations.isEmpty should be(true) + } + + test("ScalaValidator#find executable 5") { + /* TestJsonCreatorWithValidations */ + val clazzDescriptor = Reflector + .describe(classOf[TestJsonCreatorWithValidations]) + .asInstanceOf[ClassDescriptor] + + val constructorDescriptor = + validator.descriptorFactory.findConstructor(clazzDescriptor) + + // should find Seq(int) + val constructorParameterTypes: Array[Class[_]] = + constructorDescriptor.constructor.getParameterTypes() + constructorParameterTypes.length should equal(1) + constructorParameterTypes.head should equal(classOf[Int]) + + val caseClazzDescriptor = + validator.getConstraintsForClass(classOf[TestJsonCreatorWithValidations]) + caseClazzDescriptor.members.size should equal(1) + caseClazzDescriptor.members.contains("int") should be(true) + caseClazzDescriptor.members("int").annotations.length should equal(1) + } + + test("ScalaValidator#with Logging trait") { + val value = PersonWithLogging(42, "Baz Var", None) + validator.validate(value).isEmpty should be(true) + } + + private[this] def newJavaSet(numElements: Int): util.Set[Int] = { + val result: util.Set[Int] = new util.HashSet[Int]() + for (i <- 1 to numElements) { + result.add(i) + } + result + } + + private[this] def verifyDriverClassDescriptor( + driverCaseClassDescriptor: CaseClassDescriptor[Driver] + ): Unit = { + val person = driverCaseClassDescriptor.members("person") + person.scalaType.erasure should equal(classOf[Person]) + // cascaded types will have a contained scala type + person.cascadedScalaType.isDefined should be(true) + person.cascadedScalaType.get.erasure should equal(classOf[Person]) + person.annotations.isEmpty should be(true) + person.isCascaded should be(true) + + // person is cascaded, so let's look it up + val personDescriptor = validator.getConstraintsForClass(classOf[Person]) + personDescriptor.members.size should equal(6) + val city = personDescriptor.members("city") + city.scalaType.erasure should equal(classOf[String]) + city.annotations.isEmpty should be(true) + city.isCascaded should be(false) + + val name = personDescriptor.members("name") + name.scalaType.erasure == classOf[String] should be(true) + name.annotations.length should equal(1) + name.annotations.head.annotationType() should equal(classOf[NotEmpty]) + name.isCascaded should be(false) + + val state = personDescriptor.members("state") + state.scalaType.erasure should equal(classOf[String]) + state.annotations.isEmpty should be(true) + state.isCascaded should be(false) + + val company = personDescriptor.members("company") + company.scalaType.erasure should equal(classOf[String]) + company.annotations.isEmpty should be(true) + company.isCascaded should be(false) + + val id = personDescriptor.members("id") + id.scalaType.erasure == classOf[String] should be(true) + id.annotations.length should equal(1) + id.annotations.head.annotationType() should equal(classOf[NotEmpty]) + id.isCascaded should be(false) + + val address = personDescriptor.members("address") + address.scalaType.erasure should equal(classOf[Address]) + address.annotations.isEmpty should be(true) + address.isCascaded should be(true) + + // address is cascaded, so let's look it up + val addressDescriptor = validator.getConstraintsForClass(classOf[Address]) + addressDescriptor.members.size should equal(6) + + val line1 = addressDescriptor.members("line1") + line1.scalaType.erasure == classOf[String] should be(true) + line1.annotations.isEmpty should be(true) + line1.isCascaded should be(false) + + val line2 = addressDescriptor.members("line2") + line2.scalaType.erasure == classOf[Option[String]] should be(true) + line2.annotations.isEmpty should be(true) + line2.isCascaded should be(false) + + val addressCity = addressDescriptor.members("city") + addressCity.scalaType.erasure == classOf[String] should be(true) + addressCity.annotations.isEmpty should be(true) + addressCity.isCascaded should be(false) + + val addressState = addressDescriptor.members("state") + addressState.scalaType.erasure == classOf[String] should be(true) + addressState.annotations.length should equal(1) + addressState.annotations.head.annotationType() should equal(classOf[StateConstraint]) + addressState.isCascaded should be(false) + + val zipcode = addressDescriptor.members("zipcode") + zipcode.scalaType.erasure == classOf[String] should be(true) + zipcode.annotations.isEmpty should be(true) + zipcode.isCascaded should be(false) + + val additionalPostalCode = addressDescriptor.members("additionalPostalCode") + additionalPostalCode.scalaType.erasure == classOf[Option[String]] should be(true) + // not cascaded so no contained type + additionalPostalCode.cascadedScalaType.isEmpty should be(true) + additionalPostalCode.annotations.isEmpty should be(true) + additionalPostalCode.isCascaded should be(false) + + personDescriptor.methods.length should equal(1) + personDescriptor.methods.head.method.getName should equal("validateAddress") + personDescriptor.methods.head.annotations.length should equal(1) + personDescriptor.methods.head.annotations.head.annotationType() should equal( + classOf[MethodValidation]) + personDescriptor.methods.head.members.size should equal(0) + + val car = driverCaseClassDescriptor.members("car") + car.scalaType.erasure should equal(classOf[Car]) + car.cascadedScalaType.isDefined should be(true) + car.cascadedScalaType.get.erasure should equal(classOf[Car]) + car.annotations.isEmpty should be(true) + car.isCascaded should be(true) + + // car is cascaded, so let's look it up + val carDescriptor = validator.getConstraintsForClass(classOf[Car]) + carDescriptor.members.size should equal(13) + + // lets just look at annotated and cascaded fields in car + val year = carDescriptor.members("year") + year.scalaType.erasure == classOf[Int] should be(true) + year.annotations.length should equal(1) + year.annotations.head.annotationType() should equal(classOf[Min]) + year.isCascaded should be(false) + + val owners = carDescriptor.members("owners") + owners.scalaType.erasure == classOf[Seq[Person]] should be(true) + owners.cascadedScalaType.isDefined should be(true) + owners.cascadedScalaType.get.erasure should equal(classOf[Person]) + owners.annotations.isEmpty should be(true) + owners.isCascaded should be(true) + + val licensePlate = carDescriptor.members("licensePlate") + licensePlate.scalaType.erasure == classOf[String] should be(true) + licensePlate.annotations.length should equal(2) + licensePlate.annotations.head.annotationType() should equal(classOf[NotEmpty]) + licensePlate.annotations.last.annotationType() should equal(classOf[Size]) + licensePlate.isCascaded should be(false) + + val numDoors = carDescriptor.members("numDoors") + numDoors.scalaType.erasure == classOf[Int] should be(true) + numDoors.annotations.length should equal(1) + numDoors.annotations.head.annotationType() should equal(classOf[Min]) + numDoors.isCascaded should be(false) + + val passengers = carDescriptor.members("passengers") + passengers.scalaType.erasure == classOf[Seq[Person]] should be(true) + passengers.cascadedScalaType.isDefined should be(true) + passengers.cascadedScalaType.get.erasure should equal(classOf[Person]) + passengers.annotations.isEmpty should be(true) + passengers.isCascaded should be(true) + + carDescriptor.methods.length should equal(4) + val mappedCarMethods = carDescriptor.methods.map { methodDescriptor => + methodDescriptor.method.getName -> methodDescriptor + }.toMap + + val validateId = mappedCarMethods("validateId") + validateId.members.isEmpty should be(true) + validateId.annotations.length should equal(1) + validateId.annotations.head.annotationType() should equal(classOf[MethodValidation]) + validateId.members.size should equal(0) + + val validateYearBeforeNow = mappedCarMethods("validateYearBeforeNow") + validateYearBeforeNow.members.isEmpty should be(true) + validateYearBeforeNow.annotations.length should equal(1) + validateYearBeforeNow.annotations.head.annotationType() should equal(classOf[MethodValidation]) + validateYearBeforeNow.members.size should equal(0) + + val ownershipTimesValid = mappedCarMethods("ownershipTimesValid") + ownershipTimesValid.members.isEmpty should be(true) + ownershipTimesValid.annotations.length should equal(1) + ownershipTimesValid.annotations.head.annotationType() should equal(classOf[MethodValidation]) + ownershipTimesValid.members.size should equal(0) + + val warrantyTimeValid = mappedCarMethods("warrantyTimeValid") + warrantyTimeValid.members.isEmpty should be(true) + warrantyTimeValid.annotations.length should equal(1) + warrantyTimeValid.annotations.head.annotationType() should equal(classOf[MethodValidation]) + warrantyTimeValid.members.size should equal(0) + } + + private[this] def getValidationAnnotations( + clazz: Class[_], + name: String + ): Array[Annotation] = { + val caseClassDescriptor = validator.descriptorFactory.describe(clazz) + caseClassDescriptor.members.get(name) match { + case Some(property: PropertyDescriptor) => + property.annotations + case _ => Array.empty[Annotation] + } + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/WrongMessageInterpolator.scala b/util-validator/src/test/scala/com/twitter/util/validation/WrongMessageInterpolator.scala new file mode 100644 index 0000000000..6691819240 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/WrongMessageInterpolator.scala @@ -0,0 +1,17 @@ +package com.twitter.util.validation + +import jakarta.validation.MessageInterpolator +import java.util.Locale + +class WrongMessageInterpolator extends MessageInterpolator { + override def interpolate( + s: String, + context: MessageInterpolator.Context + ): String = "Whatever you entered was wrong" + + override def interpolate( + s: String, + context: MessageInterpolator.Context, + locale: Locale + ): String = "Whatever you entered was wrong" +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/caseclasses.scala b/util-validator/src/test/scala/com/twitter/util/validation/caseclasses.scala new file mode 100644 index 0000000000..a5f41ffddc --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/caseclasses.scala @@ -0,0 +1,523 @@ +package com.twitter.util.validation + +import com.fasterxml.jackson.annotation.JsonCreator +import com.twitter.util.Try +import com.twitter.util.logging.Logging +import com.twitter.util.validation.constraints._ +import com.twitter.util.validation.engine.MethodValidationResult +import jakarta.validation.{Payload, Valid, ValidationException} +import jakarta.validation.constraints._ +import java.time.LocalDate +import scala.annotation.meta.field +import scala.beans.BeanProperty + +object OuterObject { + + case class NestedCaseClass(id: Long) + + object InnerObject { + + case class SuperNestedCaseClass(@Min(5) id: Long) + } +} + +object caseclasses { + + object Path { + val FieldSeparator = "." + + /** An empty path (no names) */ + val Empty: Path = Path(Seq.empty[String]) + + /** Creates a new path from the given name */ + def apply(name: String): Path = new Path(Seq(name)) + } + + /** + * Represents the navigation path from an object to another in an object graph. + */ + case class Path(names: Seq[String]) + + case class Page[T]( + data: List[T], + @Min(1) pageSize: Int, + next: Option[Long], + previous: Option[Long]) + + case class UnicodeNameCaseClass(@Max(3) `winning-id`: Int, @NotEmpty name: String) + + class EvaluateConstraints + + trait NotACaseClassTrait { + @UUID def uuid: String + } + + class NotACaseClass( + @NotNull manufacturer: String, + @NotNull @Size(min = 2, max = 14) licensePlate: String, + @Min(2) seatCount: Int, + uuid: String) + + final case class WithFinalValField( + @UUID uuid: String, + @NotEmpty first: Option[String], + last: Option[String], + private val permissions: Option[Set[Int]]) + + case class WithBogusCascadedField( + @NotNull @Size(min = 2, max = 14) licensePlate: String, + @Valid ints: Seq[Int]) + + case class WithMetaFieldAnnotation(@(NotEmpty @field) id: String) + + // @NotEmpty is copied to generated field, but @Size is not. We should find both. + case class WithPartialMetaFieldAnnotation(@(NotEmpty @field) @Size(min = 2, max = 10) id: String) + + case class PathNotEmpty(@NotEmpty path: Path, @NotEmpty id: String) + + case class Customer(@NotEmpty first: String, last: String) + + case class AlwaysFails(@InvalidConstraint name: String) + + case class AlwaysFailsMethodValidation(@NotEmpty id: String) { + @MethodValidation(fields = Array("id")) + def checkId: MethodValidationResult = { + throw new ValidationException("oh noes!") + } + } + + case class SmallCar( + @NotNull manufacturer: String, + @NotNull @Size(min = 2, max = 14) licensePlate: String, + @Min(2) seatCount: Int) + + trait RentalStationMixin { + @Min(10) def name: String + } + + // doesn't define any field + trait RandoMixin + + case class RentalStation(@NotNull name: String) { + def rentCar( + @NotNull customer: Customer, + @NotNull @Future start: LocalDate, + @Min(1) duration: Int + ): Unit = ??? + + def updateCustomerRecords( + @NotEmpty @Size(min = 1) @Valid customers: Seq[Customer] + ): Unit = ??? + + @NotEmpty + @Size(min = 1) + def getCustomers: Seq[Customer] = ??? + + def listCars(@NotEmpty `airport-code`: String): Seq[String] = ??? + + @ConsistentDateParameters + def reserve( + start: LocalDate, + end: LocalDate + ): Boolean = ??? + } + + trait OtherCheck + trait PersonCheck + + case class MinIntExample(@Min(1) numberValue: Int) + + // CountryCode + case class CountryCodeExample(@CountryCode countryCode: String) + case class CountryCodeOptionExample(@CountryCode countryCode: Option[String]) + case class CountryCodeSeqExample(@CountryCode countryCode: Seq[String]) + case class CountryCodeOptionSeqExample(@CountryCode countryCode: Option[Seq[String]]) + case class CountryCodeArrayExample(@CountryCode countryCode: Array[String]) + case class CountryCodeOptionArrayExample(@CountryCode countryCode: Option[Array[String]]) + case class CountryCodeInvalidTypeExample(@CountryCode countryCode: Long) + case class CountryCodeOptionInvalidTypeExample(@CountryCode countryCode: Option[Long]) + // OneOf + case class OneOfExample(@OneOf(value = Array("a", "B", "c")) enumValue: String) + case class OneOfOptionExample(@OneOf(value = Array("a", "B", "c")) enumValue: Option[String]) + case class OneOfSeqExample(@OneOf(Array("a", "B", "c")) enumValue: Seq[String]) + case class OneOfOptionSeqExample(@OneOf(Array("a", "B", "c")) enumValue: Option[Seq[String]]) + case class OneOfInvalidTypeExample(@OneOf(Array("a", "B", "c")) enumValue: Long) + case class OneOfOptionInvalidTypeExample(@OneOf(Array("a", "B", "c")) enumValue: Option[Long]) + // UUID + case class UUIDExample(@UUID uuid: String) + case class UUIDOptionExample(@UUID uuid: Option[String]) + + case class NestedOptionExample( + @NotEmpty id: String, + name: String, + @Valid user: Option[User]) + + // multiple annotations + case class User( + @NotEmpty id: String, + name: String, + @OneOf(Array("F", "M", "Other")) gender: String) { + @MethodValidation(fields = Array("name")) + def nameCheck: MethodValidationResult = { + MethodValidationResult.validIfTrue(name.length > 0, "cannot be empty") + } + } + // nested NotEmpty on a Scala collection of a case class type + case class Users(@NotEmpty @Valid users: Seq[User]) + + // Other fields not in executable + case class Person(@NotEmpty id: String, @NotEmpty name: String, @Valid address: Address) { + val company: String = "Twitter" + val city: String = "San Francisco" + val state: String = "California" + + @MethodValidation(fields = Array("address")) + def validateAddress: MethodValidationResult = + MethodValidationResult.validIfTrue(address.state.nonEmpty, "state must be specified") + } + + case class WithPersonCheck( + @NotEmpty(groups = Array(classOf[PersonCheck])) id: String, + @NotEmpty name: String) + + case class NoConstructorParams() { + @BeanProperty @NotEmpty var id: String = "" + } + + case class AnnotatedInternalFields( + @NotEmpty id: String, + @Size(min = 1, max = 100) bigString: String) { + @NotEmpty val company: String = "" // this will be validated + } + + case class AnnotatedBeanProperties(@Min(1) numbers: Int) { + @BeanProperty @NotEmpty var field1: String = "default" + } + + case class Address( + line1: String, + line2: Option[String] = None, + city: String, + @StateConstraint(payload = Array(classOf[StateConstraintPayload])) state: String, + zipcode: String, + additionalPostalCode: Option[String] = None) + + // Customer defined validations + case class StateValidationExample(@StateConstraint state: String) + // Integration test + case class ValidateUserRequest( + @NotEmpty @Pattern(regexp = "[a-z]+") userName: String, + @Max(value = 9999) id: Long, + title: String) + + case class NestedUserPayload(id: String, job: String) extends Payload + case class NestedUser( + @NotEmpty id: String, + @Valid person: Person, + @OneOf(Array("F", "M", "Other")) gender: String, + job: String) { + @MethodValidation(fields = Array("job"), payload = Array(classOf[NestedUserPayload])) + def jobCheck: MethodValidationResult = { + MethodValidationResult.validIfTrue( + job.length > 0, + "cannot be empty", + payload = Some(NestedUserPayload(id, job)) + ) + } + } + + case class Division( + name: String, + @Valid team: Seq[Group]) + + case class Persons( + @NotEmpty id: String, + @Min(1) @Valid people: Seq[Person]) + + case class Group( + @NotEmpty id: String, + @Min(1) @Valid people: Seq[Person]) + + case class CustomerAccount( + @NotEmpty accountName: String, + @Valid customer: Option[User]) + + case class CaseClassWithBoxedPrimitives(events: Integer, errors: Integer) + + case class CollectionOfCollection( + @NotEmpty id: String, + @Valid @Min(1) people: Seq[Persons]) + + case class CollectionWithArray( + @NotEmpty @Size(min = 1, max = 30) names: Array[String], + @NotEmpty @Max(2) users: Array[User]) + + // cycle -- impossible to instantiate without using null to terminate somewhere + case class A(@NotEmpty id: String, @Valid b: B) + case class B(@NotEmpty id: String, @Valid c: C) + case class C(@NotEmpty id: String, @Valid a: A) + // another cycle -- fails as we detect the D type in F + case class D(@NotEmpty id: String, @Valid e: E) + case class E(@NotEmpty id: String, @Valid f: F) + case class F(@NotEmpty id: String, @Valid d: Option[D]) + // last one -- fails as we detect the G type in I + case class G(@NotEmpty id: String, @Valid h: H) + case class H(@NotEmpty id: String, @Valid i: I) + case class I(@NotEmpty id: String, @Valid g: Seq[G]) + + // no validation -- no annotations + object TestJsonCreator { + @JsonCreator + def apply(s: String): TestJsonCreator = TestJsonCreator(s.toInt) + } + case class TestJsonCreator(int: Int) + + // no validation -- no annotations + object TestJsonCreator2 { + @JsonCreator + def apply(strings: Seq[String]): TestJsonCreator2 = TestJsonCreator2(strings.map(_.toInt)) + } + case class TestJsonCreator2(ints: Seq[Int], default: String = "Hello, World") + + // no validation -- annotation is on a factory method "executable" + object TestJsonCreatorWithValidation { + @JsonCreator + def apply(@NotEmpty s: String): TestJsonCreatorWithValidation = + TestJsonCreatorWithValidation(s.toInt) + } + case class TestJsonCreatorWithValidation(int: Int) + + // this should validate per the primary executable annotation + object TestJsonCreatorWithValidations { + @JsonCreator + def apply(@NotEmpty s: String): TestJsonCreatorWithValidations = + TestJsonCreatorWithValidations(s.toInt) + } + case class TestJsonCreatorWithValidations(@OneOf(Array("42", "137")) int: Int) + + // no validation -- no annotations + case class CaseClassWithMultipleConstructors(number1: Long, number2: Long, number3: Long) { + def this(numberAsString1: String, numberAsString2: String, numberAsString3: String) { + this(numberAsString1.toLong, numberAsString2.toLong, numberAsString3.toLong) + } + } + + // no validation -- no annotations + case class CaseClassWithMultipleConstructorsAnnotated( + number1: Long, + number2: Long, + number3: Long) { + @JsonCreator + def this(numberAsString1: String, numberAsString2: String, numberAsString3: String) { + this(numberAsString1.toLong, numberAsString2.toLong, numberAsString3.toLong) + } + } + + // no validation -- annotations are on secondary executable + case class CaseClassWithMultipleConstructorsAnnotatedAndValidations( + number1: Long, + number2: Long, + uuid: String) { + @JsonCreator + def this( + @NotEmpty numberAsString1: String, + @OneOf(Array("10001", "20002", "30003")) numberAsString2: String, + @UUID thirdArgument: String + ) { + this(numberAsString1.toLong, numberAsString2.toLong, thirdArgument) + } + } + + // this should validate -- annotations are on the primary executable + case class CaseClassWithMultipleConstructorsPrimaryAnnotatedAndValidations( + @Min(10000) number1: Long, + @OneOf(Array("10001", "20002", "30003")) number2: Long, + @UUID uuid: String) { + def this( + numberAsString1: String, + numberAsString2: String, + thirdArgument: String + ) { + this(numberAsString1.toLong, numberAsString2.toLong, thirdArgument) + } + } + + trait AncestorWithValidation { + @NotEmpty def field1: String + + @MethodValidation(fields = Array("field1")) + def validateField1: MethodValidationResult = + MethodValidationResult.validIfTrue(Try(field1.toDouble).isReturn, "not a double value") + } + + case class ImplementsAncestor(override val field1: String) extends AncestorWithValidation + + case class InvalidDoublePerson(@NotEmpty name: String)(@NotEmpty otherName: String) + case class DoublePerson(@NotEmpty name: String)(@NotEmpty val otherName: String) + case class ValidDoublePerson(@NotEmpty name: String)(@NotEmpty otherName: String) { + @MethodValidation + def checkOtherName: MethodValidationResult = + MethodValidationResult.validIfTrue( + otherName.length >= 3, + "otherName must be longer than 3 chars") + } + case class PossiblyValidDoublePerson(@NotEmpty name: String)(@NotEmpty otherName: String) { + def referenceSecondArgInMethod: String = + s"The otherName value is $otherName" + } + + case class CaseClassWithIntAndLocalDate( + @NotEmpty name: String, + age: Int, + age2: Int, + age3: Int, + dateTime: LocalDate, + dateTime2: LocalDate, + dateTime3: LocalDate, + dateTime4: LocalDate, + @NotEmpty dateTime5: Option[LocalDate]) + + case class SimpleCar( + id: Long, + make: CarMake, + model: String, + @Min(2000) year: Int, + @NotEmpty @Size(min = 2, max = 14) licensePlate: String, //multiple annotations + @Min(0) numDoors: Int = 4, + manual: Boolean = false) + + // executable annotation -- note the validator must specify a `@SupportedValidationTarget` of annotated element + case class CarWithPassengerCount @ValidPassengerCountReturnValue(max = 1) ( + passengers: Seq[Person]) + + // class-level annotation + @ValidPassengerCount(max = 4) + case class Car( + id: Long, + make: CarMake, + model: String, + @Min(2000) year: Int, + @Valid owners: Seq[Person], + @NotEmpty @Size(min = 2, max = 14) licensePlate: String, //multiple annotations + @Min(0) numDoors: Int = 4, + manual: Boolean = false, + ownershipStart: LocalDate = LocalDate.now, + ownershipEnd: LocalDate = LocalDate.now.plusYears(1), + warrantyStart: Option[LocalDate] = None, + warrantyEnd: Option[LocalDate] = None, + @Valid passengers: Seq[Person] = Seq.empty[Person]) { + + @MethodValidation + def validateId: MethodValidationResult = { + MethodValidationResult.validIfTrue(id % 2 == 1, "id may not be even") + } + + @MethodValidation + def validateYearBeforeNow: MethodValidationResult = { + val thisYear = LocalDate.now().getYear + val yearMoreThanOneYearInFuture: Boolean = + if (year > thisYear) { (year - thisYear) > 1 } + else false + MethodValidationResult.validIfFalse( + yearMoreThanOneYearInFuture, + "Model year can be at most one year newer." + ) + } + + @MethodValidation(fields = Array("ownershipEnd")) + def ownershipTimesValid: MethodValidationResult = { + validateTimeRange( + ownershipStart, + ownershipEnd, + "ownershipStart", + "ownershipEnd" + ) + } + + @MethodValidation(fields = Array("warrantyStart", "warrantyEnd")) + def warrantyTimeValid: MethodValidationResult = { + validateTimeRange( + warrantyStart, + warrantyEnd, + "warrantyStart", + "warrantyEnd" + ) + } + } + + case class Driver( + @Valid person: Person, + @Valid car: Car) + + case class MultipleConstraints( + @NotEmpty @Max(100) ints: Seq[Int], + @NotEmpty @Pattern(regexp = "\\d") digits: String, + @NotEmpty @Size(min = 3, max = 3) @OneOf(Array("how", "now", "brown", + "cow")) strings: Seq[String]) + + // maps not supported + case class MapsAndMaps( + @NotEmpty id: String, + @AssertTrue bool: Boolean, + @Valid carAndDriver: Map[Person, SimpleCar]) + + // @Valid only visible on field not on type arg + case class TestAnnotations( + @NotEmpty cars: Seq[SimpleCar @Valid]) + + case class GenericTestCaseClass[T](@NotEmpty @Valid data: T) + case class GenericMinTestCaseClass[T](@Min(4) data: T) + case class GenericTestCaseClassMultipleTypes[T, U, V]( + @NotEmpty @Valid data: T, + @Size(max = 100) things: Seq[U], + @Size(min = 3, max = 3) @Valid otherThings: Seq[V]) + case class GenericTestCaseClassWithMultipleArgs[T]( + @NotEmpty data: T, + @Min(5) number: Int) + + def validateTimeRange( + startTime: Option[LocalDate], + endTime: Option[LocalDate], + startTimeProperty: String, + endTimeProperty: String + ): MethodValidationResult = { + + val rangeDefined = startTime.isDefined && endTime.isDefined + val partialRange = !rangeDefined && (startTime.isDefined || endTime.isDefined) + + if (rangeDefined) + validateTimeRange(startTime.get, endTime.get, startTimeProperty, endTimeProperty) + else if (partialRange) + MethodValidationResult.Invalid( + "both %s and %s are required for a valid range".format(startTimeProperty, endTimeProperty) + ) + else MethodValidationResult.Valid + } + + def validateTimeRange( + startTime: LocalDate, + endTime: LocalDate, + startTimeProperty: String, + endTimeProperty: String + ): MethodValidationResult = { + + MethodValidationResult.validIfTrue( + startTime.isBefore(endTime), + "%s [%s] must be after %s [%s]" + .format(endTimeProperty, endTime.toEpochDay, startTimeProperty, startTime.toEpochDay) + ) + } + + case class PersonWithLogging( + id: Int, + name: String, + age: Option[Int], + age_with_default: Option[Int] = None, + nickname: String = "unknown") + extends Logging + + case class UsersRequest( + @Max(100) @DoesNothing max: Int, + @Past @DoesNothing startDate: Option[LocalDate], + @DoesNothing verbose: Boolean = false) +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/constraints/ConsistentDateParameters.java b/util-validator/src/test/scala/com/twitter/util/validation/constraints/ConsistentDateParameters.java new file mode 100644 index 0000000000..98a14c3865 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/constraints/ConsistentDateParameters.java @@ -0,0 +1,30 @@ +package com.twitter.util.validation.constraints; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import com.twitter.util.validation.internal.validators.ConsistentDateParametersValidator; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Target({ METHOD, ANNOTATION_TYPE }) +@Retention(RUNTIME) +@Constraint(validatedBy = { ConsistentDateParametersValidator.class }) +@Documented +public @interface ConsistentDateParameters { + + /** message */ + String message() default "start is not before end"; + + /** groups */ + Class[] groups() default { }; + + /** payload */ + Class[] payload() default { }; +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/constraints/StateConstraint.java b/util-validator/src/test/scala/com/twitter/util/validation/constraints/StateConstraint.java new file mode 100644 index 0000000000..946e4aa893 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/constraints/StateConstraint.java @@ -0,0 +1,32 @@ +package com.twitter.util.validation.constraints; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import com.twitter.util.validation.internal.validators.StateConstraintValidator; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +@Target(ElementType.PARAMETER) +@Retention(RetentionPolicy.RUNTIME) +@Constraint(validatedBy = { StateConstraintValidator.class }) +public @interface StateConstraint { + + /** Every constraint annotation must define a message element of type String. */ + String message() default "Please register with state CA"; + + /** + * Every constraint annotation must define a groups element that specifies the processing groups + * with which the constraint declaration is associated. + */ + Class[] groups() default {}; + + /** + * Constraint annotations must define a payload element that specifies the payload with which + * the constraint declaration is associated. * + */ + Class[] payload() default {}; +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/constraints/StateConstraintPayload.scala b/util-validator/src/test/scala/com/twitter/util/validation/constraints/StateConstraintPayload.scala new file mode 100644 index 0000000000..85d1a5d03d --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/constraints/StateConstraintPayload.scala @@ -0,0 +1,8 @@ +package com.twitter.util.validation.constraints + +import jakarta.validation.Payload + +case class StateConstraintPayload( + invalid: String, + expected: String) + extends Payload diff --git a/util-validator/src/test/scala/com/twitter/util/validation/conversions/PathOpsTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/conversions/PathOpsTest.scala new file mode 100644 index 0000000000..15a949d5eb --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/conversions/PathOpsTest.scala @@ -0,0 +1,58 @@ +package com.twitter.util.validation.conversions + +import com.twitter.util.validation.conversions.PathOps._ +import jakarta.validation.Path +import org.hibernate.validator.internal.engine.path.PathImpl +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class PathOpsTest extends AnyFunSuite with Matchers { + + test("PathOps#root") { + val path: Path = PathImpl.createRootPath() + + val leafNode = path.getLeafNode + leafNode should not be (null) + leafNode.toString should equal("") + } + + test("PathOps#without leaf node") { + val path = PathImpl.createRootPath() + + // only the root exists, so this fails + intercept[IndexOutOfBoundsException] { + PathImpl.createCopyWithoutLeafNode(path) + } + + // add nodes + path.addBeanNode() + path.addPropertyNode("foo") + + val pathWithLeaf: Path = PathImpl.createCopy(path) + val leafNode = pathWithLeaf.getLeafNode + leafNode should not be (null) + leafNode.toString should equal("foo") + } + + test("PathOps#propertyNode") { + val path = PathImpl.createRootPath() + path.addPropertyNode("leaf") + + val propertyPath: Path = PathImpl.createCopy(path) + + val leafNode = propertyPath.getLeafNode + leafNode should not be (null) + leafNode.toString should equal("leaf") + } + + test("PathOps#fromString") { + val path: Path = PathImpl.createPathFromString("foo.bar.baz") + + val leafNode = path.getLeafNode + leafNode should not be (null) + leafNode.toString should equal("baz") + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/executable/ScalaExecutableValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/executable/ScalaExecutableValidatorTest.scala new file mode 100644 index 0000000000..2f441606f9 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/executable/ScalaExecutableValidatorTest.scala @@ -0,0 +1,820 @@ +package com.twitter.util.validation.executable + +import com.twitter.util.validation.AssertViolationTest.WithViolation +import com.twitter.util.validation.ScalaValidatorTest.DefaultAddress +import com.twitter.util.validation.caseclasses.{ + Car, + CarWithPassengerCount, + Customer, + Page, + Path, + PathNotEmpty, + Person, + RandoMixin, + RentalStation, + RentalStationMixin, + UnicodeNameCaseClass, + UsersRequest +} +import com.twitter.util.validation.cfg.ConstraintMapping +import com.twitter.util.validation.constraints.ValidPassengerCountReturnValue +import com.twitter.util.validation.engine.ConstraintViolationHelper +import com.twitter.util.validation.internal.validators.ValidPassengerCountReturnValueConstraintValidator +import com.twitter.util.validation.{AssertViolationTest, CarMake, ScalaValidator} +import jakarta.validation.{ConstraintViolation, UnexpectedTypeException} +import java.time.LocalDate + +class ScalaExecutableValidatorTest extends AssertViolationTest { + + override protected val validator: ScalaValidator = + ScalaValidator.builder + .withConstraintMappings( + Set( + ConstraintMapping( + classOf[ValidPassengerCountReturnValue], + classOf[ValidPassengerCountReturnValueConstraintValidator] + ) + ) + ) + .validator + + private[this] val executableValidator: ScalaExecutableValidator = validator.forExecutables + + test("ScalaExecutableValidator#validateMethods 1") { + intercept[IllegalArgumentException] { + executableValidator.validateMethods[Car](null) + } + } + + test("ScalaExecutableValidator#validateMethods 2") { + val owner = Person(id = "", name = "A. Einstein", address = DefaultAddress) + val car = Car( + id = 1234, + make = CarMake.Volkswagen, + model = "Beetle", + year = 1970, + owners = Seq(owner), + licensePlate = "CA123", + numDoors = 2, + manual = true, + ownershipStart = LocalDate.now, + ownershipEnd = LocalDate.now.plusYears(10), + warrantyEnd = Some(LocalDate.now.plusYears(3)), + passengers = Seq( + Person(id = "1001", name = "R. Franklin", address = DefaultAddress) + ) + ) + + val violations = executableValidator.validateMethods(car) + assertViolations( + violations, + Seq( + WithViolation("validateId", "id may not be even", car, classOf[Car], car), + WithViolation( + "warrantyTimeValid.warrantyEnd", + "both warrantyStart and warrantyEnd are required for a valid range", + car, + classOf[Car], + car), + WithViolation( + "warrantyTimeValid.warrantyStart", + "both warrantyStart and warrantyEnd are required for a valid range", + car, + classOf[Car], + car) + ) + ) + } + + test("ScalaExecutableValidator#validateMethod") { + val owner = Person(id = "", name = "A. Einstein", address = DefaultAddress) + val car = Car( + id = 1234, + make = CarMake.Volkswagen, + model = "Beetle", + year = 1970, + owners = Seq(owner), + licensePlate = "CA123", + numDoors = 2, + manual = true, + ownershipStart = LocalDate.now, + ownershipEnd = LocalDate.now.plusYears(10), + warrantyEnd = Some(LocalDate.now.plusYears(3)), + passengers = Seq( + Person(id = "1001", name = "R. Franklin", address = DefaultAddress) + ) + ) + + val method = classOf[Car].getDeclaredMethod("validateId") + + val violations = executableValidator.validateMethod(car, method) + assertViolations( + violations, + Seq( + WithViolation("validateId", "id may not be even", car, classOf[Car], car) + ) + ) + } + + test("ScalaExecutableValidator#validateParameters 1") { + val customer = Customer("Jane", "Smith") + val rentalStation = RentalStation("Hertz") + + val method = + classOf[RentalStation].getMethod( + "rentCar", + classOf[Customer], + classOf[LocalDate], + classOf[Int]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateParameters( + rentalStation, + method, + Array(customer, LocalDate.now().minusDays(1), Integer.valueOf(5))) + violations.size should equal(1) + val invalidRentalStartDateViolation = violations.head + invalidRentalStartDateViolation.getMessage should equal("must be a future date") + invalidRentalStartDateViolation.getPropertyPath.toString should equal("rentCar.start") + invalidRentalStartDateViolation.getRootBeanClass should equal(classOf[RentalStation]) + invalidRentalStartDateViolation.getRootBean should equal(rentalStation) + invalidRentalStartDateViolation.getLeafBean should equal(rentalStation) + + val invalidRentalStartDateAndNullCustomerViolations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateParameters( + rentalStation, + method, + Array(null, LocalDate.now(), Integer.valueOf(5))) + invalidRentalStartDateAndNullCustomerViolations.size should equal(2) + val sortedInvalidRentalStartDateAndNullCustomerViolationsViolations = + ConstraintViolationHelper.sortViolations[RentalStation]( + invalidRentalStartDateAndNullCustomerViolations) + val nullCustomerViolation = sortedInvalidRentalStartDateAndNullCustomerViolationsViolations.head + nullCustomerViolation.getMessage should equal("must not be null") + nullCustomerViolation.getPropertyPath.toString should equal("rentCar.customer") + nullCustomerViolation.getRootBeanClass should equal(classOf[RentalStation]) + nullCustomerViolation.getRootBean should equal(rentalStation) + nullCustomerViolation.getLeafBean should equal(rentalStation) + + val invalidRentalStartDateViolation2 = + sortedInvalidRentalStartDateAndNullCustomerViolationsViolations.last + invalidRentalStartDateViolation2.getMessage should equal("must be a future date") + invalidRentalStartDateViolation2.getPropertyPath.toString should equal("rentCar.start") + invalidRentalStartDateViolation2.getRootBeanClass should equal(classOf[RentalStation]) + invalidRentalStartDateViolation2.getRootBean should equal(rentalStation) + invalidRentalStartDateViolation2.getLeafBean should equal(rentalStation) + } + + test("ScalaExecutableValidator#validateParameters 2") { + val rentalStation = RentalStation("Hertz") + val method = + classOf[RentalStation].getMethod("updateCustomerRecords", classOf[Seq[Customer]]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateParameters( + rentalStation, + method, + Array(Seq.empty[Customer]) + ) + violations.size should equal(2) + val sortedViolations: Seq[ConstraintViolation[RentalStation]] = + ConstraintViolationHelper.sortViolations(violations) + val violation1 = sortedViolations.head + violation1.getMessage should equal("must not be empty") + violation1.getPropertyPath.toString should equal("updateCustomerRecords.customers") + violation1.getRootBeanClass should equal(classOf[RentalStation]) + violation1.getRootBean should equal(rentalStation) + violation1.getInvalidValue should equal(Seq.empty[Customer]) + + val violation2 = sortedViolations.last + violation2.getMessage should equal("size must be between 1 and 2147483647") + violation2.getPropertyPath.toString should equal("updateCustomerRecords.customers") + violation2.getRootBeanClass should equal(classOf[RentalStation]) + violation2.getRootBean should equal(rentalStation) + violation2.getInvalidValue should equal(Seq.empty[Customer]) + } + + test("ScalaExecutableValidator#validateParameters 3") { + // validation should cascade to customer parameter which is missing first name + val rentalStation = RentalStation("Hertz") + val method = + classOf[RentalStation].getMethod("updateCustomerRecords", classOf[Seq[Customer]]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateParameters( + rentalStation, + method, + Array( + Seq( + Customer("", "Ride") + )) + ) + violations.size should equal(1) + val sortedViolations: Seq[ConstraintViolation[RentalStation]] = + ConstraintViolationHelper.sortViolations(violations) + val violation1 = sortedViolations.head + violation1.getMessage should equal("must not be empty") + violation1.getPropertyPath.toString should equal("updateCustomerRecords.customers[0].first") + violation1.getRootBeanClass should equal(classOf[RentalStation]) + violation1.getRootBean should equal(rentalStation) + violation1.getInvalidValue should equal("") + } + + test("ScalaExecutableValidator#validateParameters 4") { + // validation should handle encoded parameter name + val rentalStation = RentalStation("Hertz") + val method = + classOf[RentalStation].getMethod("listCars", classOf[String]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateParameters( + rentalStation, + method, + Array("") + ) + violations.size should equal(1) + val violation = violations.head + violation.getMessage should equal("must not be empty") + violation.getPropertyPath.toString should equal("listCars.airport-code") + violation.getRootBeanClass should equal(classOf[RentalStation]) + violation.getRootBean should equal(rentalStation) + violation.getInvalidValue should equal("") + } + + test("ScalaExecutableValidator#validateReturnValue") { + val rentalStation = RentalStation("Hertz") + + val method = + classOf[RentalStation].getMethod("getCustomers") + val returnValue = Seq.empty[Customer] + + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateReturnValue( + rentalStation, + method, + returnValue + ) + violations.size should equal(2) + val sortedViolations: Seq[ConstraintViolation[RentalStation]] = + ConstraintViolationHelper.sortViolations(violations) + + val violation1 = sortedViolations.head + violation1.getMessage should equal("must not be empty") + violation1.getPropertyPath.toString should equal("getCustomers.") + violation1.getRootBeanClass should equal(classOf[RentalStation]) + violation1.getRootBean should equal(rentalStation) + violation1.getInvalidValue should equal(Seq.empty[Customer]) + + val violation2 = sortedViolations.last + violation2.getMessage should equal("size must be between 1 and 2147483647") + violation2.getPropertyPath.toString should equal("getCustomers.") + violation2.getRootBeanClass should equal(classOf[RentalStation]) + violation2.getRootBean should equal(rentalStation) + violation2.getInvalidValue should equal(Seq.empty[Customer]) + } + + test("ScalaExecutableValidator#validateConstructorParameters 1") { + val constructor = classOf[RentalStation].getConstructor(classOf[String]) + + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateConstructorParameters( + constructor, + Array(null) + ) + violations.size should equal(1) + + val violation = violations.head + violation.getMessage should equal("must not be null") + violation.getPropertyPath.toString should equal("RentalStation.name") + violation.getRootBeanClass should equal(classOf[RentalStation]) + violation.getRootBean should be(null) + violation.getInvalidValue should be(null) + } + + test("ScalaExecutableValidator#validateConstructorParameters 2") { + val constructor = classOf[UnicodeNameCaseClass].getConstructor(classOf[Int], classOf[String]) + + val violations: Set[ConstraintViolation[UnicodeNameCaseClass]] = + executableValidator.validateConstructorParameters( + constructor, + Array(5, "") + ) + violations.size should equal(2) + + val sortedViolations: Seq[ConstraintViolation[UnicodeNameCaseClass]] = + ConstraintViolationHelper.sortViolations(violations) + + val violation1 = sortedViolations.head + violation1.getMessage should equal("must not be empty") + violation1.getPropertyPath.toString should equal("UnicodeNameCaseClass.name") + violation1.getRootBeanClass should equal(classOf[UnicodeNameCaseClass]) + violation1.getRootBean should be(null) + violation1.getInvalidValue should equal("") + + val violation2 = sortedViolations(1) + violation2.getMessage should equal("must be less than or equal to 3") + violation2.getPropertyPath.toString should equal("UnicodeNameCaseClass.winning-id") + violation2.getRootBeanClass should equal(classOf[UnicodeNameCaseClass]) + violation2.getRootBean should be(null) + violation2.getInvalidValue should equal(5) + } + + test("ScalaExecutableValidator#validateConstructorParameters 3") { + val constructor = classOf[Page[String]].getConstructor( + classOf[List[String]], + classOf[Int], + classOf[Option[Long]], + classOf[Option[Long]]) + + val violations: Set[ConstraintViolation[Page[String]]] = + executableValidator.validateConstructorParameters( + constructor, + Array( + List("foo", "bar"), + 0.asInstanceOf[AnyRef], + Some(1), + Some(2) + ) + ) + violations.size should equal(1) + val violation = violations.head + violation.getPropertyPath.toString should be("Page.pageSize") + violation.getMessage should be("must be greater than or equal to 1") + violation.getRootBeanClass should equal(classOf[Page[Person]]) + violation.getRootBean should be(null) + violation.getInvalidValue should equal(0) + } + + test("ScalaExecutableValidator#validateMethodParameters 1") { + // validation should handle encoded parameter name + val method = + classOf[RentalStation].getMethod("listCars", classOf[String]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateMethodParameters( + method, + Array("") + ) + violations.size should equal(1) + val violation = violations.head + violation.getMessage should equal("must not be empty") + violation.getPropertyPath.toString should equal("listCars.airport-code") + violation.getRootBeanClass should equal(classOf[RentalStation]) + violation.getRootBean should be(null) + violation.getInvalidValue should equal("") + } + + test("ScalaExecutableValidator#validateMethodParameters 2") { + // validation should cascade to customer parameter which is missing first name + val method = + classOf[RentalStation].getMethod("updateCustomerRecords", classOf[Seq[Customer]]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateMethodParameters( + method, + Array( + Seq( + Customer("", "Ride") + )) + ) + violations.size should equal(1) + val sortedViolations: Seq[ConstraintViolation[RentalStation]] = + ConstraintViolationHelper.sortViolations(violations) + val violation1 = sortedViolations.head + violation1.getMessage should equal("must not be empty") + violation1.getPropertyPath.toString should equal("updateCustomerRecords.customers[0].first") + violation1.getRootBeanClass should equal(classOf[RentalStation]) + violation1.getRootBean should be(null) + violation1.getInvalidValue should equal("") + } + + test("ScalaExecutableValidator#validateMethodParameters 3") { + val method = + classOf[RentalStation].getMethod("updateCustomerRecords", classOf[Seq[Customer]]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateMethodParameters( + method, + Array(Seq.empty[Customer]) + ) + violations.size should equal(2) + val sortedViolations: Seq[ConstraintViolation[RentalStation]] = + ConstraintViolationHelper.sortViolations(violations) + val violation1 = sortedViolations.head + violation1.getMessage should equal("must not be empty") + violation1.getPropertyPath.toString should equal("updateCustomerRecords.customers") + violation1.getRootBeanClass should equal(classOf[RentalStation]) + violation1.getRootBean should be(null) + violation1.getInvalidValue should equal(Seq.empty[Customer]) + + val violation2 = sortedViolations.last + violation2.getMessage should equal("size must be between 1 and 2147483647") + violation2.getPropertyPath.toString should equal("updateCustomerRecords.customers") + violation2.getRootBeanClass should equal(classOf[RentalStation]) + violation2.getRootBean should be(null) + violation2.getInvalidValue should equal(Seq.empty[Customer]) + } + + test("ScalaExecutableValidator#validateMethodParameters 4") { + val customer = Customer("Jane", "Smith") + + val method = + classOf[RentalStation].getMethod( + "rentCar", + classOf[Customer], + classOf[LocalDate], + classOf[Int]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateMethodParameters( + method, + Array(customer, LocalDate.now().minusDays(1), Integer.valueOf(5))) + violations.size should equal(1) + val invalidRentalStartDateViolation = violations.head + invalidRentalStartDateViolation.getMessage should equal("must be a future date") + invalidRentalStartDateViolation.getPropertyPath.toString should equal("rentCar.start") + invalidRentalStartDateViolation.getRootBeanClass should equal(classOf[RentalStation]) + invalidRentalStartDateViolation.getRootBean should be(null) + invalidRentalStartDateViolation.getLeafBean should be(null) + + val invalidRentalStartDateAndNullCustomerViolations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateMethodParameters( + method, + Array(null, LocalDate.now(), Integer.valueOf(5))) + invalidRentalStartDateAndNullCustomerViolations.size should equal(2) + val sortedInvalidRentalStartDateAndNullCustomerViolationsViolations: Seq[ + ConstraintViolation[RentalStation] + ] = ConstraintViolationHelper.sortViolations(invalidRentalStartDateAndNullCustomerViolations) + + val nullCustomerViolation = sortedInvalidRentalStartDateAndNullCustomerViolationsViolations.head + nullCustomerViolation.getMessage should equal("must not be null") + nullCustomerViolation.getPropertyPath.toString should equal("rentCar.customer") + nullCustomerViolation.getRootBeanClass should equal(classOf[RentalStation]) + nullCustomerViolation.getRootBean should be(null) + nullCustomerViolation.getLeafBean should be(null) + + val invalidRentalStartDateViolation2 = + sortedInvalidRentalStartDateAndNullCustomerViolationsViolations.last + invalidRentalStartDateViolation2.getMessage should equal("must be a future date") + invalidRentalStartDateViolation2.getPropertyPath.toString should equal("rentCar.start") + invalidRentalStartDateViolation2.getRootBeanClass should equal(classOf[RentalStation]) + invalidRentalStartDateViolation2.getRootBean should be(null) + invalidRentalStartDateViolation2.getLeafBean should be(null) + } + + test("ScalaExecutableValidator#validateExecutableParameters 1") { + val constructor = classOf[RentalStation].getConstructor(classOf[String]) + + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateExecutableParameters( + constructor, + Array(null), + constructor.getParameters.map(_.getName), + None + ) + violations.size should equal(1) + + val violation = violations.head + violation.getMessage should equal("must not be null") + violation.getPropertyPath.toString should equal("RentalStation.name") + violation.getRootBeanClass should equal(classOf[RentalStation]) + violation.getRootBean should be(null) + violation.getInvalidValue should be(null) + } + + test("ScalaExecutableValidator#validateExecutableParameters 2") { + val constructor = classOf[UnicodeNameCaseClass].getConstructor(classOf[Int], classOf[String]) + + val violations: Set[ConstraintViolation[UnicodeNameCaseClass]] = + executableValidator.validateExecutableParameters( + constructor, + Array(5, ""), + constructor.getParameters.map(_.getName), + None + ) + violations.size should equal(2) + + val sortedViolations: Seq[ConstraintViolation[UnicodeNameCaseClass]] = + ConstraintViolationHelper.sortViolations(violations) + + val violation1 = sortedViolations.head + violation1.getMessage should equal("must not be empty") + violation1.getPropertyPath.toString should equal("UnicodeNameCaseClass.name") + violation1.getRootBeanClass should equal(classOf[UnicodeNameCaseClass]) + violation1.getRootBean should be(null) + violation1.getInvalidValue should equal("") + + val violation2 = sortedViolations(1) + violation2.getMessage should equal("must be less than or equal to 3") + violation2.getPropertyPath.toString should equal("UnicodeNameCaseClass.winning-id") + violation2.getRootBeanClass should equal(classOf[UnicodeNameCaseClass]) + violation2.getRootBean should be(null) + violation2.getInvalidValue should equal(5) + } + + test("ScalaExecutableValidator#validateExecutableParameters 3") { + val constructor = classOf[Page[String]].getConstructor( + classOf[List[String]], + classOf[Int], + classOf[Option[Long]], + classOf[Option[Long]]) + + val violations: Set[ConstraintViolation[Page[String]]] = + executableValidator.validateExecutableParameters( + constructor, + Array( + List("foo", "bar"), + 0.asInstanceOf[AnyRef], + Some(1), + Some(2) + ), + constructor.getParameters.map(_.getName), + None + ) + violations.size should equal(1) + val violation = violations.head + violation.getPropertyPath.toString should be("Page.pageSize") + violation.getMessage should be("must be greater than or equal to 1") + violation.getRootBeanClass should equal(classOf[Page[Person]]) + violation.getRootBean should be(null) + violation.getInvalidValue should equal(0) + } + + test("ScalaExecutableValidator#validateExecutableParameters 4") { + // validation should handle encoded parameter name + val method = + classOf[RentalStation].getMethod("listCars", classOf[String]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateExecutableParameters( + method, + Array(""), + method.getParameters.map(_.getName), + None + ) + violations.size should equal(1) + val violation = violations.head + violation.getMessage should equal("must not be empty") + violation.getPropertyPath.toString should equal("listCars.airport-code") + violation.getRootBeanClass should equal(classOf[RentalStation]) + violation.getRootBean should be(null) + violation.getInvalidValue should equal("") + } + + test("ScalaExecutableValidator#validateExecutableParameters 5") { + // validation should cascade to customer parameter which is missing first name + val method = + classOf[RentalStation].getMethod("updateCustomerRecords", classOf[Seq[Customer]]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateExecutableParameters( + method, + Array( + Seq( + Customer("", "Ride") + ) + ), + method.getParameters.map(_.getName), + None + ) + violations.size should equal(1) + val sortedViolations: Seq[ConstraintViolation[RentalStation]] = + ConstraintViolationHelper.sortViolations(violations) + val violation1 = sortedViolations.head + violation1.getMessage should equal("must not be empty") + violation1.getPropertyPath.toString should equal("updateCustomerRecords.customers[0].first") + violation1.getRootBeanClass should equal(classOf[RentalStation]) + violation1.getRootBean should be(null) + violation1.getInvalidValue should equal("") + } + + test("ScalaExecutableValidator#validateExecutableParameters 6") { + val method = + classOf[RentalStation].getMethod("updateCustomerRecords", classOf[Seq[Customer]]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateExecutableParameters( + method, + Array(Seq.empty[Customer]), + method.getParameters.map(_.getName), + None + ) + violations.size should equal(2) + val sortedViolations: Seq[ConstraintViolation[RentalStation]] = + ConstraintViolationHelper.sortViolations(violations) + val violation1 = sortedViolations.head + violation1.getMessage should equal("must not be empty") + violation1.getPropertyPath.toString should equal("updateCustomerRecords.customers") + violation1.getRootBeanClass should equal(classOf[RentalStation]) + violation1.getRootBean should be(null) + violation1.getInvalidValue should equal(Seq.empty[Customer]) + + val violation2 = sortedViolations.last + violation2.getMessage should equal("size must be between 1 and 2147483647") + violation2.getPropertyPath.toString should equal("updateCustomerRecords.customers") + violation2.getRootBeanClass should equal(classOf[RentalStation]) + violation2.getRootBean should be(null) + violation2.getInvalidValue should equal(Seq.empty[Customer]) + } + + test("ScalaExecutableValidator#validateExecutableParameters 7") { + val customer = Customer("Jane", "Smith") + + val method = + classOf[RentalStation].getMethod( + "rentCar", + classOf[Customer], + classOf[LocalDate], + classOf[Int]) + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateExecutableParameters( + method, + Array(customer, LocalDate.now().minusDays(1), Integer.valueOf(5)), + method.getParameters.map(_.getName), + None + ) + violations.size should equal(1) + val invalidRentalStartDateViolation = violations.head + invalidRentalStartDateViolation.getMessage should equal("must be a future date") + invalidRentalStartDateViolation.getPropertyPath.toString should equal("rentCar.start") + invalidRentalStartDateViolation.getRootBeanClass should equal(classOf[RentalStation]) + invalidRentalStartDateViolation.getRootBean should be(null) + invalidRentalStartDateViolation.getLeafBean should be(null) + + val invalidRentalStartDateAndNullCustomerViolations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateMethodParameters( + method, + Array(null, LocalDate.now(), Integer.valueOf(5))) + invalidRentalStartDateAndNullCustomerViolations.size should equal(2) + val sortedInvalidRentalStartDateAndNullCustomerViolationsViolations: Seq[ + ConstraintViolation[RentalStation] + ] = ConstraintViolationHelper.sortViolations(invalidRentalStartDateAndNullCustomerViolations) + + val nullCustomerViolation = sortedInvalidRentalStartDateAndNullCustomerViolationsViolations.head + nullCustomerViolation.getMessage should equal("must not be null") + nullCustomerViolation.getPropertyPath.toString should equal("rentCar.customer") + nullCustomerViolation.getRootBeanClass should equal(classOf[RentalStation]) + nullCustomerViolation.getRootBean should be(null) + nullCustomerViolation.getLeafBean should be(null) + + val invalidRentalStartDateViolation2 = + sortedInvalidRentalStartDateAndNullCustomerViolationsViolations.last + invalidRentalStartDateViolation2.getMessage should equal("must be a future date") + invalidRentalStartDateViolation2.getPropertyPath.toString should equal("rentCar.start") + invalidRentalStartDateViolation2.getRootBeanClass should equal(classOf[RentalStation]) + invalidRentalStartDateViolation2.getRootBean should be(null) + invalidRentalStartDateViolation2.getLeafBean should be(null) + } + + test("ScalaExecutableValidator#validateExecutableParameters 8") { + // test mixin class support + val constructor = classOf[RentalStation].getConstructor(classOf[String]) + + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateExecutableParameters( + constructor, + Array("Hertz"), + constructor.getParameters.map(_.getName), + Some(classOf[RentalStationMixin]) + ) + violations.size should equal(1) + + // this violates the @Min(10) constraint from RentalStationMixin + val violation = violations.head + violation.getMessage should equal("must be greater than or equal to 10") + violation.getPropertyPath.toString should equal("RentalStation.name") + violation.getRootBeanClass should equal(classOf[RentalStation]) + violation.getRootBean should be(null) + violation.getInvalidValue should be("Hertz") + } + + test("ScalaExecutableValidator#validateExecutableParameters 9") { + // test mixin class support and replacement of parameter names + val constructor = classOf[RentalStation].getConstructor(classOf[String]) + + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateExecutableParameters( + constructor, + Array("Hertz"), + Array("id"), + Some(classOf[RentalStationMixin]) + ) + violations.size should equal(1) + + // this violates the @Min(10) constraint from RentalStationMixin + val violation = violations.head + violation.getMessage should equal("must be greater than or equal to 10") + violation.getPropertyPath.toString should equal("RentalStation.id") + violation.getRootBeanClass should equal(classOf[RentalStation]) + violation.getRootBean should be(null) + violation.getInvalidValue should be("Hertz") + } + + test("ScalaExecutableValidator#validateExecutableParameters 10") { + // test mixin class support with mixin that doesn't define the field + val constructor = classOf[RentalStation].getConstructor(classOf[String]) + + val violations: Set[ConstraintViolation[RentalStation]] = + executableValidator.validateExecutableParameters( + constructor, + Array("Hertz"), + Array("id"), + Some(classOf[RandoMixin]) // doesn't define any fields so no additional constraints + ) + violations.isEmpty should be(true) + } + + test("ScalaExecutableValidator#validateExecutableParameters 11") { + val pathValue = Path.apply("foo") + val constructor = classOf[PathNotEmpty].getConstructor(classOf[Path], classOf[String]) + + // no validator registered `@NotEmpty` for Path type, should be returned as a violation not an exception + val e = intercept[UnexpectedTypeException] { + executableValidator.validateExecutableParameters( + constructor, + Array(pathValue, "12345"), + constructor.getParameters.map(_.getName), + None + ) + } + e.getMessage should equal( + s"No validator could be found for constraint 'interface jakarta.validation.constraints.NotEmpty' validating type 'com.twitter.util.validation.caseclasses$$Path'. Check configuration for 'PathNotEmpty.path'") + } + + test("ScalaExecutableValidator#validateExecutableParameters 12") { + val constructor = classOf[UsersRequest].getConstructor( + classOf[Int], + classOf[Option[LocalDate]], + classOf[Boolean]) + + executableValidator + .validateExecutableParameters( + constructor, + Array( + 10, + Some(LocalDate.parse("2013-01-01")), + true + ), + constructor.getParameters.map(_.getName), + None + ).isEmpty should be(true) + + val violations: Set[ConstraintViolation[UsersRequest]] = + executableValidator + .validateExecutableParameters( + constructor, + Array( + 200, + Some(LocalDate.parse("2013-01-01")), + true + ), + constructor.getParameters.map(_.getName), + None + ) + violations.size should equal(1) + + val violation = violations.head + violation.getMessage should equal("must be less than or equal to 100") + violation.getPropertyPath.toString should equal("UsersRequest.max") + violation.getRootBeanClass should equal(classOf[UsersRequest]) + violation.getRootBean should be(null) + violation.getInvalidValue should be(200) + } + + test("ScalaExecutableValidator#validateConstructorReturnValue") { + val car = CarWithPassengerCount( + Seq( + Person("abcd1234", "J Doe", DefaultAddress), + Person("abcd1235", "K Doe", DefaultAddress), + Person("abcd1236", "L Doe", DefaultAddress), + Person("abcd1237", "M Doe", DefaultAddress), + Person("abcd1238", "N Doe", DefaultAddress) + ) + ) + val constructor = classOf[CarWithPassengerCount].getConstructor(classOf[Seq[Person]]) + + val violations: Set[ConstraintViolation[CarWithPassengerCount]] = + executableValidator.validateConstructorReturnValue( + constructor, + car + ) + violations.size should equal(1) + + val violation = violations.head + violation.getMessage should equal("number of passenger(s) is not valid") + violation.getPropertyPath.toString should equal("CarWithPassengerCount.") + violation.getRootBeanClass should equal(classOf[CarWithPassengerCount]) + violation.getRootBean should equal(null) + violation.getInvalidValue should be(car) + } + + test("ScalaExecutableValidator#cross-parameter") { + val rentalStation = RentalStation("Hertz") + val method = + classOf[RentalStation].getMethod("reserve", classOf[LocalDate], classOf[LocalDate]) + + val start = LocalDate.now.plusDays(1) + val end = LocalDate.now + val violations = executableValidator.validateParameters( + rentalStation, + method, + Array(start, end) + ) + violations.size should equal(1) + val violation = violations.head + violation.getMessage should equal("start is not before end") + violation.getPropertyPath.toString should equal("reserve.") + violation.getRootBeanClass should equal(classOf[RentalStation]) + violation.getRootBean should equal(rentalStation) + violation.getLeafBean should equal(rentalStation) + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ConsistentDateParametersValidator.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ConsistentDateParametersValidator.scala new file mode 100644 index 0000000000..51a6b8fcba --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ConsistentDateParametersValidator.scala @@ -0,0 +1,21 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.validation.constraints.ConsistentDateParameters +import jakarta.validation.constraintvalidation.{SupportedValidationTarget, ValidationTarget} +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} +import java.time.LocalDate + +@SupportedValidationTarget(Array(ValidationTarget.PARAMETERS)) +class ConsistentDateParametersValidator + extends ConstraintValidator[ConsistentDateParameters, Array[AnyRef]] { + override def isValid( + value: Array[AnyRef], + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = { + if (value.length != 2) throw new IllegalArgumentException("Unexpected method signature") + else if (value(0) == null || value(1) == null) true + else if (!value(0).isInstanceOf[LocalDate] || !value(1).isInstanceOf[LocalDate]) + throw new IllegalArgumentException("Unexpected method signature") + else value(0).asInstanceOf[LocalDate].isBefore(value(1).asInstanceOf[LocalDate]) + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ConstraintValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ConstraintValidatorTest.scala new file mode 100644 index 0000000000..1d7fbc3b09 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ConstraintValidatorTest.scala @@ -0,0 +1,29 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.validation.ScalaValidator +import jakarta.validation.ConstraintViolation +import java.lang.annotation.Annotation +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import scala.reflect.runtime.universe._ + +@RunWith(classOf[JUnitRunner]) +abstract class ConstraintValidatorTest + extends AnyFunSuite + with Matchers + with ScalaCheckDrivenPropertyChecks { + + protected def validator: ScalaValidator + + protected def testFieldName: String + + protected def validate[A <: Annotation, T: TypeTag]( + clazz: Class[_], + paramName: String, + value: Any + ): Set[ConstraintViolation[T]] = + validator.validateValue(clazz.asInstanceOf[Class[T]], paramName, value) +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidatorTest.scala new file mode 100644 index 0000000000..889c5d25ed --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidatorTest.scala @@ -0,0 +1,112 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.reflect.{Types => ReflectTypes} +import com.twitter.util.validation.ScalaValidator +import com.twitter.util.validation.caseclasses.{ + CountryCodeArrayExample, + CountryCodeExample, + CountryCodeInvalidTypeExample, + CountryCodeSeqExample +} +import com.twitter.util.validation.constraints.CountryCode +import jakarta.validation.ConstraintViolation +import java.util.Locale +import org.scalacheck.Gen +import scala.reflect.runtime.universe._ + +class ISO3166CountryCodeConstraintValidatorTest extends ConstraintValidatorTest { + protected val validator: ScalaValidator = ScalaValidator() + protected val testFieldName: String = "countryCode" + + private val countryCodes = Locale.getISOCountries.filter(_.nonEmpty).toSeq + + test("pass validation for valid country code") { + countryCodes.foreach { value => + validate[CountryCodeExample](value).isEmpty should be(true) + } + } + + test("fail validation for invalid country code") { + forAll(genFakeCountryCode) { value => + val violations = validate[CountryCodeExample](value) + violations.size should equal(1) + violations.head.getMessage should equal(s"$value not a valid country code") + violations.head.getPropertyPath.toString should equal(testFieldName) + } + } + + test("pass validation for valid country codes in seq") { + val passValue = Gen.nonEmptyContainerOf[Seq, String](Gen.oneOf(countryCodes)) + + forAll(passValue) { value => + // shrinks end up with collection of empty string + if (value != null && !value.exists(_.nonEmpty) && value.nonEmpty) { + validate[CountryCodeSeqExample](value).isEmpty should be(true) + } + } + } + + test("fail validation for empty seq") { + val emptyValue = Seq.empty + val violations = validate[CountryCodeSeqExample](emptyValue) + violations.size should equal(1) + violations.head.getMessage should equal(" not a valid country code") + violations.head.getPropertyPath.toString should equal(testFieldName) + } + + test("fail validation for invalid country codes in seq") { + val failValue = Gen.nonEmptyContainerOf[Seq, String](genFakeCountryCode) + + forAll(failValue) { value => + val violations = validate[CountryCodeSeqExample](value) + violations.size should equal(1) + violations.head.getMessage should equal(s"${mkString(value)} not a valid country code") + violations.head.getPropertyPath.toString should equal(testFieldName) + } + } + + test("pass validation for valid country codes in array") { + val passValue = Gen.containerOf[Array, String](Gen.oneOf(countryCodes)) + + forAll(passValue) { value => + // shrinks end up with collection of empty string + if (value != null && !value.exists(_.nonEmpty) && value.nonEmpty) { + validate[CountryCodeArrayExample](value).isEmpty should be(true) + } + } + } + + test("fail validation for invalid country codes in array") { + val failValue = Gen.nonEmptyContainerOf[Array, String](genFakeCountryCode) + + forAll(failValue) { value => + val violations = validate[CountryCodeArrayExample](value) + violations.size should equal(1) + violations.head.getMessage should equal(s"${mkString(value)} not a valid country code") + violations.head.getPropertyPath.toString should equal(testFieldName) + } + } + + test("fail validation for invalid country code type") { + val failValue = Gen.choose[Int](0, 100) + + forAll(failValue) { value => + val violations = validate[CountryCodeInvalidTypeExample](value) + violations.size should equal(1) + violations.head.getMessage should equal(s"${value.toString} not a valid country code") + violations.head.getPropertyPath.toString should equal(testFieldName) + } + } + + //generate random uppercase string for fake country code + private def genFakeCountryCode: Gen[String] = { + Gen + .nonEmptyContainerOf[Seq, Char](Gen.alphaUpperChar) + .map(_.mkString) + .filter(!countryCodes.contains(_)) + } + + private def validate[T: TypeTag](value: Any): Set[ConstraintViolation[T]] = { + super.validate[CountryCode, T](ReflectTypes.runtimeClass[T], testFieldName, value) + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/InvalidConstraintValidator.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/InvalidConstraintValidator.scala new file mode 100644 index 0000000000..2975dba8c3 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/InvalidConstraintValidator.scala @@ -0,0 +1,12 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.validation.constraints.InvalidConstraint +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} +import scala.util.control.NoStackTrace + +class InvalidConstraintValidator extends ConstraintValidator[InvalidConstraint, Any] { + override def isValid( + obj: Any, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = throw new RuntimeException("validator foo error") with NoStackTrace +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/NotEmptyPathConstraintValidator.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/NotEmptyPathConstraintValidator.scala new file mode 100644 index 0000000000..fc9ad037ec --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/NotEmptyPathConstraintValidator.scala @@ -0,0 +1,25 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.validation.caseclasses.Path +import jakarta.validation.constraints.NotEmpty +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} + +class NotEmptyPathConstraintValidator extends ConstraintValidator[NotEmpty, Path] { + override def isValid( + obj: Path, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = obj match { + case Path.Empty => false + case _ => true + } +} + +class NotEmptyAnyConstraintValidator extends ConstraintValidator[NotEmpty, Any] { + override def isValid( + obj: Any, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = obj match { + case Path.Empty => false + case _ => true + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidatorTest.scala new file mode 100644 index 0000000000..e9c433ec88 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidatorTest.scala @@ -0,0 +1,80 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.reflect.{Types => ReflectTypes} +import com.twitter.util.validation.ScalaValidator +import com.twitter.util.validation.caseclasses._ +import com.twitter.util.validation.constraints.OneOf +import jakarta.validation.ConstraintViolation +import org.scalacheck.Gen +import scala.reflect.runtime.universe._ + +class OneOfConstraintValidatorTest extends ConstraintValidatorTest { + + protected val validator: ScalaValidator = ScalaValidator() + protected val testFieldName: String = "enumValue" + + val oneOfValues: Set[String] = Set("a", "B", "c") + + test("pass validation for single value") { + oneOfValues.foreach { value => + validate[OneOfExample](value).isEmpty should be(true) + } + } + + test("fail validation for single value") { + val failValue = Gen.alphaStr.filter(!oneOfValues.contains(_)) + + forAll(failValue) { value => + val violations = validate[OneOfExample](value) + violations.size should equal(1) + violations.head.getMessage.endsWith("not one of [a, B, c]") should be(true) + violations.head.getPropertyPath.toString should equal(testFieldName) + } + } + + test("pass validation for seq") { + val passValue = Gen.nonEmptyContainerOf[Seq, String](Gen.oneOf(oneOfValues.toSeq)) + + forAll(passValue) { value => + // shrinks end up with collection of empty string + if (value != null && !value.exists(_.nonEmpty) && value.nonEmpty) { + validate[OneOfSeqExample](value) + } + } + } + + test("fail validation for empty seq") { + val emptySeq = Seq.empty + + val violations = validate[OneOfSeqExample](emptySeq) + violations.size should equal(1) + violations.head.getMessage.endsWith("not one of [a, B, c]") should be(true) + violations.head.getPropertyPath.toString should equal(testFieldName) + } + + test("fail validation for invalid value in seq") { + val failValue = + Gen.nonEmptyContainerOf[Seq, String](Gen.alphaStr.filter(!oneOfValues.contains(_))) + + forAll(failValue) { value => + val violations = validate[OneOfSeqExample](value) + violations.size should equal(1) + violations.head.getMessage.endsWith("not one of [a, B, c]") should be(true) + violations.head.getPropertyPath.toString should equal(testFieldName) + } + } + + test("fail validation for invalid type") { + val failValue = Gen.choose(0, 100) + + forAll(failValue) { value => + val violations = validate[OneOfInvalidTypeExample](value) + violations.size should equal(1) + violations.head.getMessage.endsWith("not one of [a, B, c]") should be(true) + violations.head.getPropertyPath.toString should equal(testFieldName) + } + } + + private def validate[T: TypeTag](value: Any): Set[ConstraintViolation[T]] = + super.validate[OneOf, T](ReflectTypes.runtimeClass[T], testFieldName, value) +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/PackageObjectTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/PackageObjectTest.scala new file mode 100644 index 0000000000..2bd27c1f07 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/PackageObjectTest.scala @@ -0,0 +1,69 @@ +package com.twitter.util.validation.internal.validators + +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class PackageObjectTest extends AnyFunSuite with Matchers { + + test("mkString with more than 1 entry - Seq") { + mkString(Seq(1, 2)) should equal("[1, 2]") + } + + test("mkString with map entries") { + val m: Map[String, String] = Map("foo" -> "1", "bar" -> "2", "baz" -> "3") + mkString(m) should equal("[(foo,1), (bar,2), (baz,3)]") + } + + test("mkString with option") { + val o = Some("foo") + mkString(o) should equal("foo") + } + + test("mkString with more than 1 entry - Seq some empty values") { + mkString(Seq(" ", " 33")) should equal("33") + mkString(Seq(" ", " 33", " ")) should equal("33") + mkString(Seq(" ", " ", " ", " 33", " ")) should equal("33") + } + + test("mkString with more than 1 entry - Array") { + mkString(Array(1, 2)) should equal("[1, 2]") + } + + test("mkString with 1 entry - Seq") { + mkString(Seq(1)) should equal("1") + mkString(Seq(-1)) should equal("-1") + mkString(Seq("!!")) should equal("!!") + } + + test("mkString with 1 entry - Array") { + mkString(Array(1)) should equal("1") + } + + test("mkString with no entries - Seq") { + mkString(Seq(" ", " ")) should equal("") + mkString(Seq.empty[Int]) should equal("") + mkString(Seq.empty[String]) should equal("") + } + + test("mkString with no entries - Array") { + mkString(Array(" ", " ")) should equal("") + mkString(Array.empty[Int]) should equal("") + mkString(Array.empty[String]) should equal("") + } + + test("mkString with no entries - Map") { + mkString(Map("" -> "")) should equal("(,)") + mkString(Map.empty[String, Int]) should equal("") + mkString(Map.empty) should equal("") + } + + test("mkString with no entries - Option") { + mkString(None) should equal("") + mkString(Option(null)) should equal("") + mkString(Option("")) should equal("") + mkString(Some("")) should equal("") + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/StateConstraintValidator.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/StateConstraintValidator.scala new file mode 100644 index 0000000000..91665b67b6 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/StateConstraintValidator.scala @@ -0,0 +1,23 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.validation.constraints.{StateConstraint, StateConstraintPayload} +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} +import org.hibernate.validator.constraintvalidation.HibernateConstraintValidatorContext + +class StateConstraintValidator extends ConstraintValidator[StateConstraint, String] { + + override def isValid( + obj: String, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = { + val valid = obj.equalsIgnoreCase("CA") + + if (!valid) { + val hibernateContext: HibernateConstraintValidatorContext = + constraintValidatorContext.unwrap(classOf[HibernateConstraintValidatorContext]) + hibernateContext.withDynamicPayload(StateConstraintPayload(obj, "CA")) + } + + valid + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/UUIDConstraintValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/UUIDConstraintValidatorTest.scala new file mode 100644 index 0000000000..343358a50e --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/UUIDConstraintValidatorTest.scala @@ -0,0 +1,38 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.reflect.{Types => ReflectTypes} +import com.twitter.util.validation.ScalaValidator +import com.twitter.util.validation.caseclasses._ +import com.twitter.util.validation.constraints.UUID +import jakarta.validation.ConstraintViolation +import org.scalacheck.Gen +import scala.reflect.runtime.universe._ + +class UUIDConstraintValidatorTest extends ConstraintValidatorTest { + + protected val validator: ScalaValidator = ScalaValidator() + protected val testFieldName: String = "uuid" + + test("pass validation for valid uuid") { + val passValue = Gen.uuid + + forAll(passValue) { value => + validate[UUIDExample](value.toString).isEmpty should be(true) + } + } + + test("fail validation for invalid uuid") { + val passValue = Gen.alphaStr + + forAll(passValue) { value => + val violations = validate[UUIDExample](value) + violations.size should equal(1) + violations.head.getMessage should equal("must be a valid UUID") + violations.head.getPropertyPath.toString should equal(testFieldName) + } + } + + private def validate[T: TypeTag](value: String): Set[ConstraintViolation[T]] = { + super.validate[UUID, T](ReflectTypes.runtimeClass[T], testFieldName, value) + } +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ValidPassengerCountConstraintValidator.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ValidPassengerCountConstraintValidator.scala new file mode 100644 index 0000000000..d3fcc55ae9 --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ValidPassengerCountConstraintValidator.scala @@ -0,0 +1,21 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.validation.caseclasses.Car +import com.twitter.util.validation.constraints.ValidPassengerCount +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} + +class ValidPassengerCountConstraintValidator extends ConstraintValidator[ValidPassengerCount, Car] { + + @volatile private[this] var maxPassengers: Long = _ + + override def initialize(constraintAnnotation: ValidPassengerCount): Unit = { + this.maxPassengers = constraintAnnotation.max() + } + + override def isValid( + car: Car, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = + if (car == null) true + else car.passengers.nonEmpty && car.passengers.size <= this.maxPassengers +} diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ValidPassengerCountReturnValueConstraintValidator.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ValidPassengerCountReturnValueConstraintValidator.scala new file mode 100644 index 0000000000..99bbd77d0a --- /dev/null +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ValidPassengerCountReturnValueConstraintValidator.scala @@ -0,0 +1,22 @@ +package com.twitter.util.validation.internal.validators + +import com.twitter.util.validation.caseclasses.CarWithPassengerCount +import com.twitter.util.validation.constraints.ValidPassengerCountReturnValue +import jakarta.validation.constraintvalidation.{SupportedValidationTarget, ValidationTarget} +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} + +@SupportedValidationTarget(Array(ValidationTarget.ANNOTATED_ELEMENT)) +class ValidPassengerCountReturnValueConstraintValidator + extends ConstraintValidator[ValidPassengerCountReturnValue, CarWithPassengerCount] { + + @volatile private[this] var maxPassengers: Long = _ + + override def initialize(constraintAnnotation: ValidPassengerCountReturnValue): Unit = { + this.maxPassengers = constraintAnnotation.max() + } + + override def isValid( + obj: CarWithPassengerCount, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = obj.passengers.nonEmpty && obj.passengers.size <= this.maxPassengers +} From 9c07c22e8ccc3f8184089772bdca96f3779835bd Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Tue, 27 Apr 2021 19:48:34 +0000 Subject: [PATCH 475/796] util/util-validator: Update documentation Problem/Solution Fix line emphasis and add some notes about best practices when using the library. Differential Revision: https://phabricator.twitter.biz/D659553 --- doc/src/sphinx/util-validator/index.rst | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/doc/src/sphinx/util-validator/index.rst b/doc/src/sphinx/util-validator/index.rst index e58ddbeb2c..71fa37179c 100644 --- a/doc/src/sphinx/util-validator/index.rst +++ b/doc/src/sphinx/util-validator/index.rst @@ -1753,7 +1753,7 @@ with `@MethodValidation`, optionally specifying validated fields of the case cla error-reporting. .. code:: scala - :emphasize-lines: 13,14 + :emphasize-lines: 11,12 import com.twitter.util.validation.MethodValidation import com.twitter.util.validation.engine.MethodValidationResult @@ -1789,6 +1789,27 @@ This library provides built-in value extraction for Scala `Iterable[_] <#iterabl `Hibernate documentation `__ for more details on value extraction including how to define and register a new value extractor. +Best Practices +-------------- + +- Case classes used for validation should be side-effect free under property access. That is, accessing a + case class field should be able to be done eagerly and without side-effects. +- Case class methods for `@MethodValidation` validation should be side-effect free under method access. That is, accessing a + `@MethodValidation`-annotated method should be able to be done eagerly and without side-effects. +- Case classes with generic type params should be considered **not supported** for validation. E.g., + + .. code:: scala + + case class GenericCaseClass[T](@NotEmpty @Valid data: T) + + This may appear to work for some category of data but in practice the way the library caches reflection data does not + discriminate on the type binding and thus validation of genericized case classes is not fully guaranteed to be successful for + differing type bindings of a given genericized class. +- While `Iterable` collections are supported for cascaded validation when annotated with `@Valid`, this does + not include `Map` types. Annotated `Map` types will be ignored during validation. +- More generally, types with multiple type params, e.g. `Either[T, U]`, are not supported for validation + of contents when annotated with `@Valid`. Annotated unsupported types will be **ignored** during validation. + More resources -------------- From 1756eaa62ce16ebc899fd99c0fc64ec2ce33afed Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 28 Apr 2021 01:04:40 +0000 Subject: [PATCH 476/796] metric-expressions: add the field namespaces for multitenancy support Problem Multitenancy system would like to instrument expressions pre-tenant, so each tenant can get its own monitoring and alerting. Expression APIs currently don't have this supported. Solution Add a field in expressions, multitenancy systems can define expressions in their layers for each tenant. JIRA Issues: CSL-10810 Differential Revision: https://phabricator.twitter.biz/D657317 --- .../finagle/stats/InMemoryStatsReceiver.scala | 4 +- .../finagle/stats/exp/ExpressionLabels.scala | 2 +- .../finagle/stats/exp/ExpressionSchema.scala | 13 ++ .../stats/exp/ExpressionSchemaTest.scala | 142 +++++++++++------- 4 files changed, 102 insertions(+), 59 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index a144c8ff0d..6dab7158cc 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -285,10 +285,10 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { * Designed to match the behavior of Metrics::registerExpression(). */ override protected[finagle] def registerExpression(schema: ExpressionSchema): Unit = { - val expressionId = schema.labels.serviceName match { + val expressionId = (schema.labels.serviceName match { case Some(serviceName) => schema.name + "_" + serviceName case None => schema.name - } + }) + schema.namespaces.mkString("_") preloadExpressions.put(expressionId, schema) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala index 3cb6779e02..5185a093a3 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala @@ -11,7 +11,7 @@ private[exp] object ExpressionLabels { * * @param processPath a universal coordinate for the resource * @param serviceName a client label or a server label depends on the `role` - * @param role whether the service is playing the part of client or server regarding this metric + * @param role whether the service is playing the part of client or server regarding this metric */ private[exp] case class ExpressionLabels( processPath: Option[String], diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index 8c90ad9904..aa23727b72 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -7,6 +7,9 @@ import com.twitter.finagle.stats.{MetricUnit, SourceRole, StatsReceiver, Unspeci * * @param name this is going to be an important query key when fetching expressions * @param labels service related information, see [[ExpressionLabels]] + * @param namespaces a list of namespaces the expression belongs to, this is usually + * used to indicate a tenant in a multi-tenancy systems or similar concepts. + * For standalone services, this should be empty. * @param expr class representation of the expression, see [[Expression]] * @param bounds thresholds for this expression * @param description human-readable description of an expression's significance @@ -17,6 +20,7 @@ case class ExpressionSchema private ( name: String, labels: ExpressionLabels, expr: Expression, + namespaces: Seq[String], bounds: Bounds, description: String, unit: MetricUnit, @@ -27,6 +31,14 @@ case class ExpressionSchema private ( def withUnit(unit: MetricUnit): ExpressionSchema = copy(unit = unit) + /** + * Configure the expression with the given namespaces. The path can be composed of segments or + * a single string. This is for multi-tenancy system tenants or similar concepts and should + * remain unset for standalone services. + */ + def withNamespaces(name: String*): ExpressionSchema = + copy(namespaces = this.namespaces ++ name) + private[finagle] def withRole(role: SourceRole): ExpressionSchema = copy(labels = labels.copy(role = role)) @@ -48,6 +60,7 @@ private[twitter] object ExpressionSchema { ExpressionSchema( name = name, labels = ExpressionLabels.empty, + namespaces = Seq.empty, expr = expr, bounds = Unbounded.get, description = "Unspecified", diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index be04d3d311..198a13042b 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -7,76 +7,106 @@ import org.scalatest.FunSuite import scala.util.control.NonFatal class ExpressionSchemaTest extends FunSuite { + trait Ctx { + val sr = new InMemoryStatsReceiver + val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) - val sr = new InMemoryStatsReceiver - val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) - - val successMb = - CounterSchema(MetricBuilder(name = Seq("success"), statsReceiver = clientSR).withKernel) - val failuresMb = - CounterSchema(MetricBuilder(name = Seq("failures"), statsReceiver = clientSR).withKernel) - val latencyMb = - HistogramSchema(MetricBuilder(name = Seq("latency"), statsReceiver = clientSR).withKernel) - val sum = Expression(successMb).plus(Expression(failuresMb)) - - val successRate = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) - .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) - .withUnit(Percentage) - .withDescription("The success rate of the slow query") - - val latency = ExpressionSchema("latency", Expression(latencyMb, Right(0.99))) - .withUnit(Milliseconds) - .withDescription("The latency of the slow query") - - def slowQuery(ctl: TimeControl, succeed: Boolean): Unit = { - ctl.advance(50.milliseconds) - if (!succeed) { - throw new Exception("boom!") + val successMb = + CounterSchema(MetricBuilder(name = Seq("success"), statsReceiver = clientSR).withKernel) + val failuresMb = + CounterSchema(MetricBuilder(name = Seq("failures"), statsReceiver = clientSR).withKernel) + val latencyMb = + HistogramSchema(MetricBuilder(name = Seq("latency"), statsReceiver = clientSR).withKernel) + val sum = Expression(successMb).plus(Expression(failuresMb)) + + def slowQuery(ctl: TimeControl, succeed: Boolean): Unit = { + ctl.advance(50.milliseconds) + if (!succeed) { + throw new Exception("boom!") + } } } test("Demonstrate an end to end example of using metric expressions") { + new Ctx { + val successCounter = clientSR.counter(successMb) + val failuresCounter = clientSR.counter(failuresMb) + val latencyStat = clientSR.stat(latencyMb) + + val successRate = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query") + + val latency = ExpressionSchema("latency", Expression(latencyMb, Right(0.99))) + .withUnit(Milliseconds) + .withDescription("The latency of the slow query") - val successCounter = clientSR.counter(successMb) - val failuresCounter = clientSR.counter(failuresMb) - val latencyStat = clientSR.stat(latencyMb) - - successRate.register() - latency.register() - - def runTheQuery(succeed: Boolean): Unit = { - Time.withCurrentTimeFrozen { ctl => - val elapsed = Stopwatch.start() - try { - slowQuery(ctl, succeed) - successCounter.incr() - } catch { - case NonFatal(exn) => - failuresCounter.incr() - } finally { - latencyStat.add(elapsed().inMilliseconds) + successRate.register() + latency.register() + + def runTheQuery(succeed: Boolean): Unit = { + Time.withCurrentTimeFrozen { ctl => + val elapsed = Stopwatch.start() + try { + slowQuery(ctl, succeed) + successCounter.incr() + } catch { + case NonFatal(exn) => + failuresCounter.incr() + } finally { + latencyStat.add(elapsed().inMilliseconds) + } } } - } - runTheQuery(true) - runTheQuery(false) + runTheQuery(true) + runTheQuery(false) - assert(sr.expressions("success_rate_downstream").name == successRate.name) - assert(sr.expressions("success_rate_downstream").expr != successRate.expr) - assert(sr.expressions("latency_downstream").name == latency.name) - assert(sr.expressions("latency_downstream").expr != latency.expr) + assert(sr.expressions("success_rate_downstream").name == successRate.name) + assert(sr.expressions("success_rate_downstream").expr != successRate.expr) + assert(sr.expressions("latency_downstream").name == latency.name) + assert(sr.expressions("latency_downstream").expr != latency.expr) - assert(sr.counters(Seq("success")) == 1) - assert(sr.counters(Seq("failures")) == 1) - assert(sr.stats(Seq("latency")) == Seq(50, 50)) + assert(sr.counters(Seq("success")) == 1) + assert(sr.counters(Seq("failures")) == 1) + assert(sr.stats(Seq("latency")) == Seq(50, 50)) - assert(sr.expressions("success_rate_downstream").labels.role == Client) + assert(sr.expressions("success_rate_downstream").labels.role == Client) + } } test("create a histogram expression requires a component") { - val e = intercept[IllegalArgumentException] { - ExpressionSchema("latency", Expression(latencyMb)) + new Ctx { + val e = intercept[IllegalArgumentException] { + ExpressionSchema("latency", Expression(latencyMb)) + } + + assert(e.getMessage.contains("provide a component for histogram")) + } + } + + test("expressions with different namespaces are all stored") { + new Ctx { + val successRate1 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) + .withNamespaces("section1") + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query") + .register() + + val successRate2 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) + .withNamespaces("section2", "a") + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query") + .register() + + assert(sr.expressions.values.size == 2) + val namespaces = sr.expressions.values.toSeq.map { exprSchema => + exprSchema.namespaces + } + assert(namespaces.contains(Seq("section1"))) + assert(namespaces.contains(Seq("section2", "a"))) } - assert(e.getMessage.contains("provide a component for histogram")) } } From 78714aa117fa767a54dd520a81c29d1c2794b5bc Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 28 Apr 2021 15:30:04 +0000 Subject: [PATCH 477/796] [util-validator] fix dependencies JIRA Issues: DPB-13760 Differential Revision: https://phabricator.twitter.biz/D659954 --- .../main/scala/com/twitter/util/validation/internal/engine/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD index b6fb9b349f..d92c75574c 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD @@ -10,6 +10,7 @@ scala_library( dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", + "3rdparty/jvm/org/scala-lang/modules:scala-collection-compat", "util/util-validator/src/main/java/com/twitter/util/validation", "util/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor", "util/util-validator/src/main/scala/org/hibernate/validator/internal/engine", From 2e9a6b452c41c7e3f32afad88fce9ba7cc7994db Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 29 Apr 2021 00:34:29 +0000 Subject: [PATCH 478/796] util/util-validator: Add specific PROJECT file Problem/Solution Add a more specific PROJECT file to the util-validator sub-directory. Differential Revision: https://phabricator.twitter.biz/D660898 --- util-validator/PROJECT | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 util-validator/PROJECT diff --git a/util-validator/PROJECT b/util-validator/PROJECT new file mode 100644 index 0000000000..2a8a1534c3 --- /dev/null +++ b/util-validator/PROJECT @@ -0,0 +1,8 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - ccoco + - ibennett + - jyan + - mnakamura + - roanta From 504bdf37a6c060ea4e3e8fc165693f10000153e2 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 29 Apr 2021 21:42:54 +0000 Subject: [PATCH 479/796] util-validator: Use org.scalacheck.Shrink.shrinkAny in tests Problem When using ScalaCheck in tests to generate failing values, shrinking the failing values may generate invalid values as the constraints of the generator being used are not respected. Solution In the relevant tests, import `org.scalacheck.Shrink.shrinkAny` to change the behavior of `forAll` to disable shrinking. Result Less brittle code to deal with the invalid values being produced by the generator when testing failure values. Differential Revision: https://phabricator.twitter.biz/D661479 --- .../ISO3166CountryCodeConstraintValidatorTest.scala | 13 ++++--------- .../validators/OneOfConstraintValidatorTest.scala | 6 ++---- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidatorTest.scala index 889c5d25ed..a5b16ce029 100644 --- a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidatorTest.scala +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/ISO3166CountryCodeConstraintValidatorTest.scala @@ -12,6 +12,7 @@ import com.twitter.util.validation.constraints.CountryCode import jakarta.validation.ConstraintViolation import java.util.Locale import org.scalacheck.Gen +import org.scalacheck.Shrink.shrinkAny import scala.reflect.runtime.universe._ class ISO3166CountryCodeConstraintValidatorTest extends ConstraintValidatorTest { @@ -39,10 +40,7 @@ class ISO3166CountryCodeConstraintValidatorTest extends ConstraintValidatorTest val passValue = Gen.nonEmptyContainerOf[Seq, String](Gen.oneOf(countryCodes)) forAll(passValue) { value => - // shrinks end up with collection of empty string - if (value != null && !value.exists(_.nonEmpty) && value.nonEmpty) { - validate[CountryCodeSeqExample](value).isEmpty should be(true) - } + validate[CountryCodeSeqExample](value).isEmpty should be(true) } } @@ -66,13 +64,10 @@ class ISO3166CountryCodeConstraintValidatorTest extends ConstraintValidatorTest } test("pass validation for valid country codes in array") { - val passValue = Gen.containerOf[Array, String](Gen.oneOf(countryCodes)) + val passValue = Gen.nonEmptyContainerOf[Array, String](Gen.oneOf(countryCodes)) forAll(passValue) { value => - // shrinks end up with collection of empty string - if (value != null && !value.exists(_.nonEmpty) && value.nonEmpty) { - validate[CountryCodeArrayExample](value).isEmpty should be(true) - } + validate[CountryCodeArrayExample](value).isEmpty should be(true) } } diff --git a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidatorTest.scala index e9c433ec88..141891b7af 100644 --- a/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidatorTest.scala +++ b/util-validator/src/test/scala/com/twitter/util/validation/internal/validators/OneOfConstraintValidatorTest.scala @@ -6,6 +6,7 @@ import com.twitter.util.validation.caseclasses._ import com.twitter.util.validation.constraints.OneOf import jakarta.validation.ConstraintViolation import org.scalacheck.Gen +import org.scalacheck.Shrink.shrinkAny import scala.reflect.runtime.universe._ class OneOfConstraintValidatorTest extends ConstraintValidatorTest { @@ -36,10 +37,7 @@ class OneOfConstraintValidatorTest extends ConstraintValidatorTest { val passValue = Gen.nonEmptyContainerOf[Seq, String](Gen.oneOf(oneOfValues.toSeq)) forAll(passValue) { value => - // shrinks end up with collection of empty string - if (value != null && !value.exists(_.nonEmpty) && value.nonEmpty) { - validate[OneOfSeqExample](value) - } + validate[OneOfSeqExample](value) } } From c6ab0cf07b078667003634f8a2f91bb5a58183ab Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 30 Apr 2021 01:04:35 +0000 Subject: [PATCH 480/796] util-validator: Fix doc typos Problem/Solution Fix documentation typos. Differential Revision: https://phabricator.twitter.biz/D661781 --- doc/src/sphinx/util-validator/index.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/src/sphinx/util-validator/index.rst b/doc/src/sphinx/util-validator/index.rst index 71fa37179c..a00d8e1595 100644 --- a/doc/src/sphinx/util-validator/index.rst +++ b/doc/src/sphinx/util-validator/index.rst @@ -748,7 +748,7 @@ Use `validate` to perform validation of all constraints of a given case class. :emphasize-lines: 8 import jakarta.validation.ConstraintViolation - import jakarta.validation.constraints.{AssertTure, NotEmpty} + import jakarta.validation.constraints.{AssertTrue, NotEmpty} case class Car(@NotEmpty manufacturer: String, @AssertTrue isRegistered: Boolean) @@ -774,7 +774,7 @@ the method throws a `ConstraintViolationException Date: Sat, 8 May 2021 01:43:31 +0000 Subject: [PATCH 481/796] util-validator|finatra-jackson: Better optimize Jackson deserialization Problem Finatra's Jackson integration caches the `CaseClassDeserializer` but we're not taking advantage of that for validation as we're regenerating expensive objects via reflection when perfoming field and method validations. Solution Move the validation descriptors needed to perform case class validations to be member variables of the `CaseClassDeserializer` for a given class. Jackson will cache these members meaning they will not need to be regenerated when performing validation. Update the validation logic to use these memeber variables against the current data for validating the case class. Result Much less CPU overhead and less object allocation which should lead to better resource utilization. Differential Revision: https://phabricator.twitter.biz/D666476 --- .../util/validation/DescriptorFactory.scala | 210 ++++++++++----- .../util/validation/ScalaValidator.scala | 253 ++++++++++-------- .../twitter/util/validation/executable/BUILD | 4 +- .../executable/ScalaExecutableValidator.scala | 34 ++- .../util/validation/ScalaValidatorTest.scala | 21 ++ .../twitter/util/validation/caseclasses.scala | 14 + .../ScalaExecutableValidatorTest.scala | 149 ++++++++--- 7 files changed, 464 insertions(+), 221 deletions(-) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/DescriptorFactory.scala b/util-validator/src/main/scala/com/twitter/util/validation/DescriptorFactory.scala index 84aa137672..4d4818a0dc 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/DescriptorFactory.scala +++ b/util-validator/src/main/scala/com/twitter/util/validation/DescriptorFactory.scala @@ -7,18 +7,19 @@ import com.twitter.util.validation.internal.Types import com.twitter.util.validation.metadata.{ CaseClassDescriptor, ConstructorDescriptor, + ExecutableDescriptor, MethodDescriptor, PropertyDescriptor } import com.twitter.util.{Return, Try} import jakarta.validation.{Constraint, ConstraintDeclarationException, Valid, ValidationException} import java.lang.annotation.Annotation -import java.lang.reflect.{Constructor, Method, Parameter} -import java.util.function.Function +import java.lang.reflect.{Constructor, Executable, Method, Parameter} import org.hibernate.validator.internal.metadata.core.ConstraintHelper import org.json4s.reflect.{ClassDescriptor, ConstructorParamDescriptor, Reflector, ScalaType} import org.json4s.{reflect => json4s} import scala.collection.mutable +import scala.util.control.NonFatal private[validation] object DescriptorFactory { @@ -69,56 +70,142 @@ private[validation] class DescriptorFactory( caseClassDescriptors.cleanUp() } - /** Describe a [[Class]] */ + /** + * Describe a [[Class]]. + * + * @note the returned [[CaseClassDescriptor]] is cached for repeated lookup attempts keyed by + * the given Class[T] type. + */ def describe[T]( clazz: Class[T] ): CaseClassDescriptor[T] = { caseClassDescriptors .get( clazz, - new Function[Class[_], CaseClassDescriptor[_]] { - def apply(key: Class[_]): CaseClassDescriptor[_] = { - buildDescriptor[T](clazz) - } - } + (key: Class[_]) => buildDescriptor[T](clazz) ).asInstanceOf[CaseClassDescriptor[T]] } /** - * Describe a [[Method]]. + * Describe a [[Constructor]]. + * + * @note the returned [[ConstructorDescriptor]] is NOT cached. It is up to the caller of this + * method to optimize any calls to this method. + */ + def describe[T]( + constructor: Constructor[T] + ): ConstructorDescriptor = { + buildConstructorDescriptor( + constructor.getDeclaringClass, + getJson4sConstructorDescriptor(constructor), + None + ) + } + + /** + * Describe a `@MethodValidation`-annotated or otherwise constrained [[Method]]. * * As we do not want to describe every possible case class method, this potentially * returns a None in the case where the method has no constraint annotation and no * constrained parameters. + * + * @note the returned [[MethodDescriptor]] is NOT cached. It is up to the caller of this + * method to optimize any calls to this method. */ def describe(method: Method): Option[MethodDescriptor] = buildMethodDescriptor(method) - /** Describe a [[Constructor]]. */ - def describe[T]( - constructor: Constructor[T] - ): ConstructorDescriptor = describe(constructor, None) - - /** Describe a [[Constructor]] given an optional "mix-in" Class. */ - def describe[T]( - constructor: Constructor[T], + /** + * Describe an [[Executable]] given an optional "mix-in" Class. + * + * @note the returned [[ExecutableDescriptor]] is NOT cached. It is up to the caller of this + * method to optimize any calls to this method. + */ + def describeExecutable[T]( + executable: Executable, mixinClazz: Option[Class[_]] - ): ConstructorDescriptor = { - val parameters: Array[Parameter] = constructor.getParameters - val desc = org.json4s.reflect.ConstructorDescriptor( - params = Seq.tabulate(parameters.length) { index => - val parameter = parameters(index) - org.json4s.reflect.ConstructorParamDescriptor( - name = unmangleName(parameter.getName), - mangledName = parameter.getName, - argIndex = index, - argType = Reflector.scalaTypeOf(parameter.getParameterizedType), - defaultValue = None + ): ExecutableDescriptor = { + executable match { + case constructor: Constructor[_] => + val desc = getJson4sConstructorDescriptor(constructor) + val clazz = constructor.getDeclaringClass.asInstanceOf[Class[T]] + val parameterAnnotations = mixinClazz match { + case Some(mixin) => + val constructorAnnotationMap = getConstructorParams(clazz, desc) + // augment with mixin class field annotations + constructorAnnotationMap.map { + case (name, params) => + try { + val method = mixin.getDeclaredMethod(name) + ( + name, + params.copy(annotations = + params.annotations ++ + method.getAnnotations.filter(isConstraintAnnotation)) + ) + } catch { + case NonFatal(_) => // do nothing + (name, params) + } + } + case _ => + getConstructorParams(clazz, desc) + } + buildConstructorDescriptor( + constructor.getDeclaringClass, + desc, + Some(parameterAnnotations) ) - }, - constructor = new org.json4s.reflect.Executable(constructor, true), - isPrimary = true - ) - buildConstructorDescriptor[T](constructor.getDeclaringClass, mixinClazz, desc) + case method: Method => + MethodDescriptor( + method = method, + annotations = method.getDeclaredAnnotations.filter(isConstraintAnnotation), + members = method.getParameters.map { parameter => + val name = parameter.getName + // augment with mixin class field annotations + val parameterAnnotations = + mixinClazz match { + case Some(mixin) => + try { + val method = mixin.getDeclaredMethod(name) + parameter.getAnnotations ++ + method.getAnnotations.filter(isConstraintAnnotation) + } catch { + case NonFatal(_) => // do nothing + parameter.getAnnotations + } + case _ => // do nothing + parameter.getAnnotations + } + parameter.getName -> buildPropertyDescriptor( + Reflector.scalaTypeOf(parameter.getParameterizedType), + parameterAnnotations + ) + }.toMap + ) + case _ => // should not get here + throw new IllegalArgumentException + } + } + + /** + * Describe `@MethodValidation`-annotated or otherwise constrained methods of a given `Class[_]`. + * + * @note the returned [[MethodDescriptor]] instances are NOT cached. It is up to the caller of this + * method to optimize any calls to this method. + */ + def describeMethods(clazz: Class[_]): Array[MethodDescriptor] = { + val clazzMethods = clazz.getMethods + if (clazzMethods.nonEmpty) { + val methods = new mutable.ArrayBuffer[MethodDescriptor](clazzMethods.length) + var index = 0 + val length = clazzMethods.length + while (index < length) { + val method = clazzMethods(index) + buildMethodDescriptor(method).foreach(methods.append(_)) + index += 1 + } + methods.toArray + } else Array.empty[MethodDescriptor] } /* Private */ @@ -162,7 +249,7 @@ private[validation] class DescriptorFactory( val jsize = clazzDescriptor.constructors.size while (jindex < jsize) { constructors.append( - buildConstructorDescriptor(clazz, None, clazzDescriptor.constructors(jindex)) + buildConstructorDescriptor(clazz, clazzDescriptor.constructors(jindex), None) ) jindex += 1 } @@ -309,34 +396,35 @@ private[validation] class DescriptorFactory( ) } + private[this] def getJson4sConstructorDescriptor( + constructor: Constructor[_] + ): org.json4s.reflect.ConstructorDescriptor = { + val parameters: Array[Parameter] = constructor.getParameters + org.json4s.reflect.ConstructorDescriptor( + params = Seq.tabulate(parameters.length) { index => + val parameter = parameters(index) + org.json4s.reflect.ConstructorParamDescriptor( + name = unmangleName(parameter.getName), + mangledName = parameter.getName, + argIndex = index, + argType = Reflector.scalaTypeOf(parameter.getParameterizedType), + defaultValue = None + ) + }, + constructor = new org.json4s.reflect.Executable(constructor, true), + isPrimary = true + ) + } + private[this] def buildConstructorDescriptor[T]( clazz: Class[T], - mixinClazz: Option[Class[_]], - constructorDescriptor: org.json4s.reflect.ConstructorDescriptor + constructorDescriptor: org.json4s.reflect.ConstructorDescriptor, + parameterAnnotationsMap: Option[scala.collection.Map[String, ConstructorParams]] ): ConstructorDescriptor = { - val parameterAnnotations = - mixinClazz match { - case Some(mixin) => - val constructorAnnotationMap = getConstructorParams(clazz, constructorDescriptor) - // augment with mixin class field annotations - constructorAnnotationMap.map { - case (name, params) => - Try(mixin.getDeclaredMethod(name)).toOption match { - case Some(mixinField) => - ( - name, - params.copy(annotations = - params.annotations ++ - mixinField.getAnnotations.filter(isConstraintAnnotation)) - ) - case _ => // do nothing - (name, params) - } - } - case _ => - getConstructorParams(clazz, constructorDescriptor) - } - + val parameterAnnotations = parameterAnnotationsMap match { + case Some(m) => m + case _ => getConstructorParams(clazz, constructorDescriptor) + } val (executable, annotations) = Option(constructorDescriptor.constructor.constructor) match { case Some(constructor) => (constructor, constructor.getAnnotations) @@ -407,8 +495,8 @@ private[validation] class DescriptorFactory( throw new ConstraintDeclarationException( s"Methods annotated with @${classOf[MethodValidation].getSimpleName} must not declare any arguments") if (method.getReturnType != classOf[MethodValidationResult]) - throw new ConstraintDeclarationException( - s"Methods annotated with @${classOf[MethodValidation].getSimpleName} must return a ${classOf[MethodValidationResult]}") + throw new ConstraintDeclarationException(s"Methods annotated with @${classOf[ + MethodValidation].getSimpleName} must return a ${classOf[MethodValidationResult].getName}") true } diff --git a/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala b/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala index bac1195732..bca0626154 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala +++ b/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala @@ -1,5 +1,6 @@ package com.twitter.util.validation +import com.twitter.util.Memoize import com.twitter.util.logging.Logging import com.twitter.util.reflect.{Classes, Annotations => ReflectAnnotations, Types => ReflectTypes} import com.twitter.util.validation.cfg.ConstraintMapping @@ -189,8 +190,8 @@ class ScalaValidator private[validation] ( } /** - * Returns the descriptorFactory object describing case class constraints. Descriptors describe constraint - * constraints on a given class and any isCascaded types. + * Returns a [[CaseClassDescriptor]] object describing the given case class type constraints. + * Descriptors describe constraints on a given class and any cascaded case class types. * * The returned object (and associated objects including CaseClassDescriptors) are immutable. * @@ -204,26 +205,68 @@ class ScalaValidator private[validation] ( def getConstraintsForClass[T](clazz: Class[T]): CaseClassDescriptor[T] = descriptorFactory.describe[T](clazz) + /** + * Returns a [[ExecutableDescriptor]] object describing a given [[Executable]]. + * + * The returned object (and associated objects including ExecutableDescriptors) are immutable. + * + * @param executable the [[Executable]] to describe. + * + * @return the [[ExecutableDescriptor]] for the specified [[Executable]] + * + * @note the returned [[ExecutableDescriptor]] is NOT cached. It is up to the caller of this + * method to optimize any calls to this method. + */ + private[twitter] def describeExecutable( + executable: Executable, + mixinClazz: Option[Class[_]] + ): ExecutableDescriptor = descriptorFactory.describeExecutable(executable, mixinClazz) + + /** + * Returns an array of [[MethodDescriptor]] instances describing the `@MethodValidation`-annotated + * or otherwise constrained methods of the given case class type. + * + * @param clazz class or interface type evaluated + * + * @return an array of [[MethodDescriptor]] for `@MethodValidation`-annotated or otherwise + * constrained methods of the class + * + * @note the returned [[MethodDescriptor]] instances are NOT cached. It is up to the caller of this + * method to optimize any calls to this method. + */ + private[twitter] def describeMethods( + clazz: Class[_] + ): Array[MethodDescriptor] = descriptorFactory.describeMethods(clazz) + /** * Returns the set of Constraints which validate the given [[Annotation]] class. * * For instance if the constraint type is `Class[com.twitter.util.validation.constraints.CountryCode]`, * the returned set would contain an instance of the `ISO3166CountryCodeConstraintValidator`. * - * @param annotationType the Class[_] of the [[Annotation]] for which to find all supporting constraint validators. - * * @return the set of supporting constraint validators for a given [[Annotation]]. + * @note this method is memoized as it should only ever need to be calculated once for a given [[Class]]. */ - def findConstraintValidators[A <: Annotation]( - annotationType: Class[A] - ): Set[ConstraintValidatorType] = { - // compute from constraint, otherwise compute from registry - val validatedBy = findValidatedBy(annotationType) - if (validatedBy.isEmpty) { - validatorFactory.constraintHelper - .getAllValidatorDescriptors(annotationType).asScala - .map(_.newInstance(validatorFactory.constraintValidatorFactory)).toSet - } else validatedBy + val findConstraintValidators: Class[_ <: Annotation] => Set[ConstraintValidatorType] = Memoize { + annotationType: Class[_ <: Annotation] => + // compute from constraint, otherwise compute from registry + val validatedBy: Set[ConstraintValidatorType] = + if (annotationType.isAnnotationPresent(classOf[Constraint])) { + val constraintAnnotation = annotationType.getAnnotation(classOf[Constraint]) + constraintAnnotation + .validatedBy().map { clazz => + val instance = validatorFactory.constraintValidatorFactory.getInstance(clazz) + if (instance == null) + throw new ValidationException( + s"Constraint factory returned null when trying to create instance of $clazz.") + instance.asInstanceOf[ConstraintValidatorType] + }.toSet + } else Set.empty + if (validatedBy.isEmpty) { + validatorFactory.constraintHelper + .getAllValidatorDescriptors(annotationType).asScala + .map(_.newInstance(validatorFactory.constraintValidatorFactory)).toSet + } else validatedBy } /** @@ -251,12 +294,13 @@ class ScalaValidator private[validation] ( * - Define a `groups` parameter * - Define a `payload` parameter * - * @param annotationType The [[Annotation]] class to test. - * * @return true if the constraint fulfills the above conditions, false otherwise. + * @note this method is memoized as it should only ever need to be calculated once for a given [[Class]]. */ - def isConstraintAnnotation(annotationType: Class[_ <: Annotation]): Boolean = - validatorFactory.constraintHelper.isConstraintAnnotation(annotationType) + val isConstraintAnnotation: Class[_ <: Annotation] => Boolean = Memoize { + annotationType: Class[_ <: Annotation] => + validatorFactory.constraintHelper.isConstraintAnnotation(annotationType) + } /** * Validates all constraint constraints on an object. @@ -625,10 +669,9 @@ class ScalaValidator private[validation] ( /** @inheritdoc */ private[twitter] def validateExecutableParameters[T]( - executable: Executable, + executable: ExecutableDescriptor, parameterValues: Array[Any], parameterNames: Array[String], - mixinClazz: Option[Class[_]], groups: Class[_]* ): Set[ConstraintViolation[T]] = { if (executable == null) @@ -636,37 +679,35 @@ class ScalaValidator private[validation] ( if (parameterValues == null) throw new IllegalArgumentException("The constructor parameter array cannot not be null.") - val parameterCount = executable.getParameterCount + val parameterCount = executable.executable.getParameterCount val parameterValuesLength = parameterValues.length if (parameterCount != parameterValuesLength) throw new ValidationException( s"Wrong number of parameters. Method or constructor $executable expects $parameterCount parameters, but got $parameterValuesLength.") - executable match { - case constructor: Constructor[_] => - validateExecutableParameters[T]( - executableDescriptor = - descriptorFactory.describe[T](constructor.asInstanceOf[Constructor[T]], mixinClazz), - root = None, - leaf = None, - parameterValues = parameterValues, - parameterNames = Some(parameterNames), - groups = groups - ) - case _ => - descriptorFactory.describe(executable.asInstanceOf[Method]) match { - case Some(descriptor: ExecutableDescriptor) => - validateExecutableParameters[T]( - executableDescriptor = descriptor, - root = None, - leaf = None, - parameterValues = parameterValues, - parameterNames = Some(parameterNames), - groups = groups - ) - case _ => Set.empty[ConstraintViolation[T]] - } - } + validateExecutableParameters[T]( + executableDescriptor = executable, + root = None, + leaf = None, + parameterValues = parameterValues, + parameterNames = Some(parameterNames), + groups = groups + ) + } + + /** @inheritdoc */ + private[twitter] def validateMethods[T]( + methods: Array[MethodDescriptor], + obj: T, + groups: Class[_]* + ): Set[ConstraintViolation[T]] = { + validateMethods[T]( + rootClazz = Some(obj.getClass.asInstanceOf[Class[T]]), + root = Some(obj), + methods = methods, + obj = obj, + groups = groups + ) } // note: Prefer while loop over Scala for loop for better performance. The scala for loop @@ -785,24 +826,6 @@ class ScalaValidator private[validation] ( results.toSet } else Set.empty[ConstraintViolation[Any]] - private[this] def findValidatedBy[A <: Annotation]( - annotationType: Class[A] - ): Set[ConstraintValidatorType] = { - if (annotationType.isAnnotationPresent(classOf[Constraint])) { - val constraintAnnotation = annotationType.getAnnotation(classOf[Constraint]) - val clazzes: Array[Class[ConstraintValidatorType]] = - constraintAnnotation.validatedBy().map(_.asInstanceOf[Class[ConstraintValidatorType]]) - - clazzes.map { clazz => - val instance = validatorFactory.constraintValidatorFactory.getInstance(clazz) - if (instance == null) - throw new ValidationException( - s"Constraint factory returned null when trying to create instance of $clazz.") - instance - }.toSet - } else Set.empty - } - // https://beanvalidation.org/2.0/spec/#constraintsdefinitionimplementation-validationimplementation // From the documentation: // @@ -841,12 +864,7 @@ class ScalaValidator private[validation] ( case Some(names) => names case _ => - val parameterNameProviderNames: java.util.List[String] = - validatorFactory.validatorFactoryScopedContext.getParameterNameProvider - .getParameterNames(executableDescriptor.executable) - Array.tabulate(parameterNameProviderNames.size()) { index => - parameterNameProviderNames.get(index) - } + getExecutableParameterNames(executableDescriptor.executable) } // executable parameter constraints @@ -1366,49 +1384,64 @@ class ScalaValidator private[validation] ( else groups.exists(groupsFromAnnotation.contains) } - private[this] def parametersValidationTargetFilter( - constraintValidator: ConstraintValidator[_, _] - ): Boolean = - ReflectAnnotations - .findAnnotation( - classOf[SupportedValidationTarget], - constraintValidator.getClass.getAnnotations) match { - case Some(annotation: SupportedValidationTarget) => - annotation.value().contains(ValidationTarget.PARAMETERS) - case _ => false - } + /** @note this method is memoized as it should only ever need to be calculated once for a given [[Executable]] */ + private[this] val getExecutableParameterNames: Executable => Array[String] = Memoize { + executable: Executable => + val parameterNameProviderNames: java.util.List[String] = + validatorFactory.validatorFactoryScopedContext.getParameterNameProvider + .getParameterNames(executable) + // depending on the underlying list this may be linear and not constant lookup + Array.tabulate(parameterNameProviderNames.size()) { index => + parameterNameProviderNames.get(index) + } + } - private[this] def getExecutableMetaData[T](executable: Executable): ExecutableMetaData = { - val callable: ExecutableCallable[_] = executable match { - case constructor: Constructor[_] => - new ConstructorCallable(constructor) - case method: Method => - new MethodCallable(method) + /** @note this method is memoized as it should only ever need to be calculated once for a given [[ConstraintValidator]] */ + private[this] val parametersValidationTargetFilter: ConstraintValidator[_, _] => Boolean = + Memoize { constraintValidator: ConstraintValidator[_, _] => + ReflectAnnotations + .findAnnotation( + classOf[SupportedValidationTarget], + constraintValidator.getClass.getAnnotations) match { + case Some(annotation: SupportedValidationTarget) => + annotation.value().contains(ValidationTarget.PARAMETERS) + case _ => false + } } - val builder = new ExecutableMetaData.Builder( - executable.getDeclaringClass, - new ConstrainedExecutable( - ConfigurationSource.ANNOTATION, - callable, - callable.getParameters - .map(p => - new ConstrainedParameter( - ConfigurationSource.ANNOTATION, - callable, - p.getType, - p.getIndex)).asJava, - Collections.emptySet(), - Collections.emptySet(), - Collections.emptySet(), - CascadingMetaDataBuilder.nonCascading - ), - validatorFactory.getConstraintCreationContext, - validatorFactory.getExecutableHelper, - validatorFactory.getExecutableParameterNameProvider, - validatorFactory.getMethodValidationConfiguration - ) - builder.build() + /** @note this method is memoized as it should only ever need to be calculated once for a given [[Executable]] */ + private[this] val getExecutableMetaData: Executable => ExecutableMetaData = Memoize { + executable: Executable => + val callable: ExecutableCallable[_] = executable match { + case constructor: Constructor[_] => + new ConstructorCallable(constructor) + case method: Method => + new MethodCallable(method) + } + val builder = new ExecutableMetaData.Builder( + executable.getDeclaringClass, + new ConstrainedExecutable( + ConfigurationSource.ANNOTATION, + callable, + callable.getParameters + .map(p => + new ConstrainedParameter( + ConfigurationSource.ANNOTATION, + callable, + p.getType, + p.getIndex)).asJava, + Collections.emptySet(), + Collections.emptySet(), + Collections.emptySet(), + CascadingMetaDataBuilder.nonCascading + ), + validatorFactory.getConstraintCreationContext, + validatorFactory.getExecutableHelper, + validatorFactory.getExecutableParameterNameProvider, + validatorFactory.getMethodValidationConfiguration + ) + + builder.build() } private[validation] def parseMethodValidationFailures[T]( diff --git a/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD index 0930df7bae..e267af8e30 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD @@ -9,10 +9,10 @@ scala_library( strict_deps = True, dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", - "util/util-core:scala", - "util/util-slf4j-api/src/main/scala", + "util/util-validator/src/main/scala/com/twitter/util/validation/metadata", ], exports = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "util/util-validator/src/main/scala/com/twitter/util/validation/metadata", ], ) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/executable/ScalaExecutableValidator.scala b/util-validator/src/main/scala/com/twitter/util/validation/executable/ScalaExecutableValidator.scala index 5fd113983c..62f77809fc 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/executable/ScalaExecutableValidator.scala +++ b/util-validator/src/main/scala/com/twitter/util/validation/executable/ScalaExecutableValidator.scala @@ -1,7 +1,8 @@ package com.twitter.util.validation.executable +import com.twitter.util.validation.metadata.{ExecutableDescriptor, MethodDescriptor} import jakarta.validation.{ConstraintViolation, ValidationException} -import java.lang.reflect.{Constructor, Executable, Method} +import java.lang.reflect.{Constructor, Method} /** * Scala version of the Bean Specification [[jakarta.validation.executable.ExecutableValidator]]. @@ -155,24 +156,39 @@ trait ScalaExecutableValidator { /** * Validates all constraints placed on the parameters of the given executable. * - * @param executable the executable for which the parameter constraints is validated. + * @param executable the [[ExecutableDescriptor]] for which the parameter constraints is validated. * @param parameterValues the values provided by the caller for the given executable's parameters. * @param parameterNames the parameter names to use for error reporting. - * @param mixinClazz the optional mix-in class to also consider for annotation processing. * @param groups the list of groups targeted for validation (defaults to Default). * @tparam T the type defining the executable to validate. * - * @return a set with the constraint violations caused by this validation; Will be empty if no error occurs, but never null. + * @return a set with the constraint violations caused by this validation; Will be empty if no error occurs, but never null. * - * @throws IllegalArgumentException - if null is passed for any of the parameters or if parameters don't match with each other. - * @throws ValidationException - if a non recoverable error happens during the validation process. + * @throws IllegalArgumentException - if null is passed for any of the parameters or if parameters don't match with each other. + * @throws ValidationException - if a non-recoverable error happens during the validation process. */ - @throws[ValidationException] private[twitter] def validateExecutableParameters[T]( - executable: Executable, + executable: ExecutableDescriptor, parameterValues: Array[Any], parameterNames: Array[String], - mixinClazz: Option[Class[_]], + groups: Class[_]* + ): Set[ConstraintViolation[T]] + + /** + * Validates the `@MethodValidation` placed constraints on the given array of descriptors. + * @param methods the array of [[MethodDescriptor]] instances to evaluate. + * @param obj the object on which the method to validate is invoked. + * @param groups the list of groups targeted for validation (defaults to Default). + * @tparam T the type defining the methods to validate. + * + * @return a set with the constraint violations caused by this validation; Will be empty if no error occurs, but never null. + * + * @throws IllegalArgumentException - if null is passed for any of the parameters or if parameters don't match with each other. + * @throws ValidationException - if a non-recoverable error happens during the validation process. + */ + private[twitter] def validateMethods[T]( + methods: Array[MethodDescriptor], + obj: T, groups: Class[_]* ): Set[ConstraintViolation[T]] } diff --git a/util-validator/src/test/scala/com/twitter/util/validation/ScalaValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/ScalaValidatorTest.scala index fafeb3abb5..71609b96c1 100644 --- a/util-validator/src/test/scala/com/twitter/util/validation/ScalaValidatorTest.scala +++ b/util-validator/src/test/scala/com/twitter/util/validation/ScalaValidatorTest.scala @@ -22,6 +22,7 @@ import com.twitter.util.validation.internal.validators.{ } import jakarta.validation.constraints.{Min, NotEmpty, Pattern, Size} import jakarta.validation.{ + ConstraintDeclarationException, ConstraintValidator, ConstraintViolation, ConstraintViolationException, @@ -1493,6 +1494,26 @@ class ScalaValidatorTest extends AssertViolationTest with BeforeAndAfterAll { warrantyTimeValid.members.size should equal(0) } + test("ScalaValidator#class with incorrect method validation definition 1") { + val value = WithIncorrectlyDefinedMethodValidation("abcd1234") + + val e = intercept[ConstraintDeclarationException] { + validator.validate(value) + } + e.getMessage should equal( + "Methods annotated with @MethodValidation must not declare any arguments") + } + + test("ScalaValidator#class with incorrect method validation definition 2") { + val value = AnotherIncorrectlyDefinedMethodValidation("abcd1234") + + val e = intercept[ConstraintDeclarationException] { + validator.validate(value) + } + e.getMessage should equal( + "Methods annotated with @MethodValidation must return a com.twitter.util.validation.engine.MethodValidationResult") + } + private[this] def getValidationAnnotations( clazz: Class[_], name: String diff --git a/util-validator/src/test/scala/com/twitter/util/validation/caseclasses.scala b/util-validator/src/test/scala/com/twitter/util/validation/caseclasses.scala index a5f41ffddc..d35de93d9d 100644 --- a/util-validator/src/test/scala/com/twitter/util/validation/caseclasses.scala +++ b/util-validator/src/test/scala/com/twitter/util/validation/caseclasses.scala @@ -86,6 +86,20 @@ object caseclasses { } } + // throws a ConstraintDefinitionException as `@ethodValidation` must not define args + case class WithIncorrectlyDefinedMethodValidation(@NotEmpty id: String) { + @MethodValidation(fields = Array("id")) + def checkId(a: String): MethodValidationResult = { + MethodValidationResult.validIfTrue(a == id, "id not equal") + } + } + + // throws a ConstraintDefinitionException as `@ethodValidation` must return a MethodValidationResult + case class AnotherIncorrectlyDefinedMethodValidation(@NotEmpty id: String) { + @MethodValidation(fields = Array("id")) + def checkId: Boolean = id.nonEmpty + } + case class SmallCar( @NotNull manufacturer: String, @NotNull @Size(min = 2, max = 14) licensePlate: String, diff --git a/util-validator/src/test/scala/com/twitter/util/validation/executable/ScalaExecutableValidatorTest.scala b/util-validator/src/test/scala/com/twitter/util/validation/executable/ScalaExecutableValidatorTest.scala index 2f441606f9..137c0b78f5 100644 --- a/util-validator/src/test/scala/com/twitter/util/validation/executable/ScalaExecutableValidatorTest.scala +++ b/util-validator/src/test/scala/com/twitter/util/validation/executable/ScalaExecutableValidatorTest.scala @@ -6,6 +6,7 @@ import com.twitter.util.validation.caseclasses.{ Car, CarWithPassengerCount, Customer, + NestedUser, Page, Path, PathNotEmpty, @@ -86,6 +87,69 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { ) } + test("ScalaExecutableValidator#validateMethods 3") { + val owner = Person(id = "", name = "A. Einstein", address = DefaultAddress) + val car = Car( + id = 1234, + make = CarMake.Volkswagen, + model = "Beetle", + year = 1970, + owners = Seq(owner), + licensePlate = "CA123", + numDoors = 2, + manual = true, + ownershipStart = LocalDate.now, + ownershipEnd = LocalDate.now.plusYears(10), + warrantyEnd = Some(LocalDate.now.plusYears(3)), + passengers = Seq( + Person(id = "1001", name = "R. Franklin", address = DefaultAddress) + ) + ) + + val methods = validator.describeMethods(classOf[Car]) + val violations = executableValidator.validateMethods(methods, car) + assertViolations( + violations, + Seq( + WithViolation("validateId", "id may not be even", car, classOf[Car], car), + WithViolation( + "warrantyTimeValid.warrantyEnd", + "both warrantyStart and warrantyEnd are required for a valid range", + car, + classOf[Car], + car), + WithViolation( + "warrantyTimeValid.warrantyStart", + "both warrantyStart and warrantyEnd are required for a valid range", + car, + classOf[Car], + car) + ) + ) + } + + test("ScalaExecutableValidator#validateMethods 4") { + val nestedUser = NestedUser( + "abcd1234", + Person(id = "1234abcd", name = "A. Einstein", address = DefaultAddress), + "Other", + "" + ) + val methods = validator.describeMethods(classOf[NestedUser]) + val violations = executableValidator.validateMethods(methods, nestedUser) + assertViolations( + violations, + Seq( + WithViolation( + "jobCheck.job", + "cannot be empty", + nestedUser, + classOf[NestedUser], + nestedUser) + ) + ) + } + test("ScalaExecutableValidator#validateMethod") { val owner = Person(id = "", name = "A. Einstein", address = DefaultAddress) val car = Car( @@ -456,13 +520,13 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { test("ScalaExecutableValidator#validateExecutableParameters 1") { val constructor = classOf[RentalStation].getConstructor(classOf[String]) + val descriptor = validator.describeExecutable(constructor, None) val violations: Set[ConstraintViolation[RentalStation]] = executableValidator.validateExecutableParameters( - constructor, + descriptor, Array(null), - constructor.getParameters.map(_.getName), - None + constructor.getParameters.map(_.getName) ) violations.size should equal(1) @@ -476,13 +540,13 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { test("ScalaExecutableValidator#validateExecutableParameters 2") { val constructor = classOf[UnicodeNameCaseClass].getConstructor(classOf[Int], classOf[String]) + val descriptor = validator.describeExecutable(constructor, None) val violations: Set[ConstraintViolation[UnicodeNameCaseClass]] = executableValidator.validateExecutableParameters( - constructor, + descriptor, Array(5, ""), - constructor.getParameters.map(_.getName), - None + constructor.getParameters.map(_.getName) ) violations.size should equal(2) @@ -510,18 +574,18 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { classOf[Int], classOf[Option[Long]], classOf[Option[Long]]) + val descriptor = validator.describeExecutable(constructor, None) val violations: Set[ConstraintViolation[Page[String]]] = executableValidator.validateExecutableParameters( - constructor, + descriptor, Array( List("foo", "bar"), 0.asInstanceOf[AnyRef], Some(1), Some(2) ), - constructor.getParameters.map(_.getName), - None + constructor.getParameters.map(_.getName) ) violations.size should equal(1) val violation = violations.head @@ -536,12 +600,13 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { // validation should handle encoded parameter name val method = classOf[RentalStation].getMethod("listCars", classOf[String]) + val descriptor = validator.describeExecutable(method, None) + val violations: Set[ConstraintViolation[RentalStation]] = executableValidator.validateExecutableParameters( - method, + descriptor, Array(""), - method.getParameters.map(_.getName), - None + method.getParameters.map(_.getName) ) violations.size should equal(1) val violation = violations.head @@ -556,16 +621,17 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { // validation should cascade to customer parameter which is missing first name val method = classOf[RentalStation].getMethod("updateCustomerRecords", classOf[Seq[Customer]]) + val descriptor = validator.describeExecutable(method, None) + val violations: Set[ConstraintViolation[RentalStation]] = executableValidator.validateExecutableParameters( - method, + descriptor, Array( Seq( Customer("", "Ride") ) ), - method.getParameters.map(_.getName), - None + method.getParameters.map(_.getName) ) violations.size should equal(1) val sortedViolations: Seq[ConstraintViolation[RentalStation]] = @@ -581,12 +647,13 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { test("ScalaExecutableValidator#validateExecutableParameters 6") { val method = classOf[RentalStation].getMethod("updateCustomerRecords", classOf[Seq[Customer]]) + val descriptor = validator.describeExecutable(method, None) + val violations: Set[ConstraintViolation[RentalStation]] = executableValidator.validateExecutableParameters( - method, + descriptor, Array(Seq.empty[Customer]), - method.getParameters.map(_.getName), - None + method.getParameters.map(_.getName) ) violations.size should equal(2) val sortedViolations: Seq[ConstraintViolation[RentalStation]] = @@ -615,12 +682,13 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { classOf[Customer], classOf[LocalDate], classOf[Int]) + val descriptor = validator.describeExecutable(method, None) + val violations: Set[ConstraintViolation[RentalStation]] = executableValidator.validateExecutableParameters( - method, + descriptor, Array(customer, LocalDate.now().minusDays(1), Integer.valueOf(5)), - method.getParameters.map(_.getName), - None + method.getParameters.map(_.getName) ) violations.size should equal(1) val invalidRentalStartDateViolation = violations.head @@ -658,13 +726,13 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { test("ScalaExecutableValidator#validateExecutableParameters 8") { // test mixin class support val constructor = classOf[RentalStation].getConstructor(classOf[String]) + val descriptor = validator.describeExecutable(constructor, Some(classOf[RentalStationMixin])) val violations: Set[ConstraintViolation[RentalStation]] = executableValidator.validateExecutableParameters( - constructor, + descriptor, Array("Hertz"), - constructor.getParameters.map(_.getName), - Some(classOf[RentalStationMixin]) + constructor.getParameters.map(_.getName) ) violations.size should equal(1) @@ -680,13 +748,13 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { test("ScalaExecutableValidator#validateExecutableParameters 9") { // test mixin class support and replacement of parameter names val constructor = classOf[RentalStation].getConstructor(classOf[String]) + val descriptor = validator.describeExecutable(constructor, Some(classOf[RentalStationMixin])) val violations: Set[ConstraintViolation[RentalStation]] = executableValidator.validateExecutableParameters( - constructor, + descriptor, Array("Hertz"), - Array("id"), - Some(classOf[RentalStationMixin]) + Array("id") ) violations.size should equal(1) @@ -702,13 +770,17 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { test("ScalaExecutableValidator#validateExecutableParameters 10") { // test mixin class support with mixin that doesn't define the field val constructor = classOf[RentalStation].getConstructor(classOf[String]) + val descriptor = + validator.describeExecutable( + constructor, + Some(classOf[RandoMixin]) + ) // doesn't define any fields so no additional constraints val violations: Set[ConstraintViolation[RentalStation]] = executableValidator.validateExecutableParameters( - constructor, + descriptor, Array("Hertz"), - Array("id"), - Some(classOf[RandoMixin]) // doesn't define any fields so no additional constraints + Array("id") ) violations.isEmpty should be(true) } @@ -716,14 +788,14 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { test("ScalaExecutableValidator#validateExecutableParameters 11") { val pathValue = Path.apply("foo") val constructor = classOf[PathNotEmpty].getConstructor(classOf[Path], classOf[String]) + val descriptor = validator.describeExecutable(constructor, None) // no validator registered `@NotEmpty` for Path type, should be returned as a violation not an exception val e = intercept[UnexpectedTypeException] { executableValidator.validateExecutableParameters( - constructor, + descriptor, Array(pathValue, "12345"), - constructor.getParameters.map(_.getName), - None + constructor.getParameters.map(_.getName) ) } e.getMessage should equal( @@ -735,30 +807,29 @@ class ScalaExecutableValidatorTest extends AssertViolationTest { classOf[Int], classOf[Option[LocalDate]], classOf[Boolean]) + val descriptor = validator.describeExecutable(constructor, None) executableValidator .validateExecutableParameters( - constructor, + descriptor, Array( 10, Some(LocalDate.parse("2013-01-01")), true ), - constructor.getParameters.map(_.getName), - None + constructor.getParameters.map(_.getName) ).isEmpty should be(true) val violations: Set[ConstraintViolation[UsersRequest]] = executableValidator .validateExecutableParameters( - constructor, + descriptor, Array( 200, Some(LocalDate.parse("2013-01-01")), true ), - constructor.getParameters.map(_.getName), - None + constructor.getParameters.map(_.getName) ) violations.size should equal(1) From 07d525bd285a8c8b95a3cbb9d25507937b19cb54 Mon Sep 17 00:00:00 2001 From: Flavio Brasil Date: Mon, 10 May 2021 19:56:38 +0000 Subject: [PATCH 482/796] util-core: lock-free async semaphore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem The current implementation of `AsyncSemaphore` is lock-based and has proved to be too costly under contention. I've also observed that while testing the fiber scheduler implementation. Solution Migrate to a lock-free implementation similar to how semaphores are implemented in Linux by using a single atomic counter to indicate the number of available permits or the number of waiters. The happy path is uses a single CAS and if there aren't available permits, a concurrent linked queue is used. Since the queue incurs into additional allocations, I've also optimized the code to avoid allocations in a few places to compensate. Result The new implementation has better performance and lower allocation rate: baseline ``` [info] Benchmark Mode Cnt Score Error Units [info] AsyncSemaphoreBenchmark.mixed avgt 10 1265.338 ± 97.302 ns/op [info] AsyncSemaphoreBenchmark.mixed:·gc.alloc.rate avgt 10 258.451 ± 14.962 MB/sec [info] AsyncSemaphoreBenchmark.mixed:·gc.alloc.rate.norm avgt 10 50.551 ± 0.590 B/op [info] AsyncSemaphoreBenchmark.mixed:·gc.churn.Par_Eden_Space avgt 10 260.352 ± 138.484 MB/sec [info] AsyncSemaphoreBenchmark.mixed:·gc.churn.Par_Eden_Space.norm avgt 10 51.108 ± 28.172 B/op [info] AsyncSemaphoreBenchmark.mixed:·gc.count avgt 10 12.000 counts [info] AsyncSemaphoreBenchmark.mixed:·gc.time avgt 10 6.000 ms [info] AsyncSemaphoreBenchmark.noWaiters avgt 10 806.794 ± 208.589 ns/op [info] AsyncSemaphoreBenchmark.noWaiters:·gc.alloc.rate avgt 10 131.521 ± 35.256 MB/sec [info] AsyncSemaphoreBenchmark.noWaiters:·gc.alloc.rate.norm avgt 10 16.000 ± 0.001 B/op [info] AsyncSemaphoreBenchmark.noWaiters:·gc.churn.Par_Eden_Space avgt 10 130.410 ± 169.691 MB/sec [info] AsyncSemaphoreBenchmark.noWaiters:·gc.churn.Par_Eden_Space.norm avgt 10 16.496 ± 22.444 B/op [info] AsyncSemaphoreBenchmark.noWaiters:·gc.count avgt 10 6.000 counts [info] AsyncSemaphoreBenchmark.noWaiters:·gc.time avgt 10 5.000 ms [info] AsyncSemaphoreBenchmark.waiters avgt 10 3671.510 ± 773.611 ns/op [info] AsyncSemaphoreBenchmark.waiters:·gc.alloc.rate avgt 10 213.112 ± 22.384 MB/sec [info] AsyncSemaphoreBenchmark.waiters:·gc.alloc.rate.norm avgt 10 120.443 ± 11.879 B/op [info] AsyncSemaphoreBenchmark.waiters:·gc.churn.Par_Eden_Space avgt 10 215.990 ± 0.763 MB/sec [info] AsyncSemaphoreBenchmark.waiters:·gc.churn.Par_Eden_Space.norm avgt 10 123.114 ± 26.471 B/op [info] AsyncSemaphoreBenchmark.waiters:·gc.count avgt 10 10.000 counts [info] AsyncSemaphoreBenchmark.waiters:·gc.time avgt 10 7.000 ms ``` optimized ``` [info] Benchmark Mode Cnt Score Error Units [info] AsyncSemaphoreBenchmark.mixed avgt 10 826.948 ± 33.943 ns/op [info] AsyncSemaphoreBenchmark.mixed:·gc.alloc.rate avgt 10 383.787 ± 14.822 MB/sec [info] AsyncSemaphoreBenchmark.mixed:·gc.alloc.rate.norm avgt 10 48.244 ± 0.177 B/op [info] AsyncSemaphoreBenchmark.mixed:·gc.churn.Par_Eden_Space avgt 10 391.142 ± 99.118 MB/sec [info] AsyncSemaphoreBenchmark.mixed:·gc.churn.Par_Eden_Space.norm avgt 10 49.164 ± 12.446 B/op [info] AsyncSemaphoreBenchmark.mixed:·gc.count avgt 10 19.000 counts [info] AsyncSemaphoreBenchmark.mixed:·gc.time avgt 10 12.000 ms [info] AsyncSemaphoreBenchmark.noWaiters avgt 10 735.404 ± 24.172 ns/op [info] AsyncSemaphoreBenchmark.noWaiters:·gc.alloc.rate avgt 10 145.514 ± 4.828 MB/sec [info] AsyncSemaphoreBenchmark.noWaiters:·gc.alloc.rate.norm avgt 10 16.000 ± 0.001 B/op [info] AsyncSemaphoreBenchmark.noWaiters:·gc.churn.Par_Eden_Space avgt 10 143.113 ± 149.378 MB/sec [info] AsyncSemaphoreBenchmark.noWaiters:·gc.churn.Par_Eden_Space.norm avgt 10 15.715 ± 16.429 B/op [info] AsyncSemaphoreBenchmark.noWaiters:·gc.count avgt 10 7.000 counts [info] AsyncSemaphoreBenchmark.noWaiters:·gc.time avgt 10 5.000 ms [info] AsyncSemaphoreBenchmark.waiters avgt 10 2518.887 ± 781.773 ns/op [info] AsyncSemaphoreBenchmark.waiters:·gc.alloc.rate avgt 10 278.906 ± 49.568 MB/sec [info] AsyncSemaphoreBenchmark.waiters:·gc.alloc.rate.norm avgt 10 104.685 ± 17.330 B/op [info] AsyncSemaphoreBenchmark.waiters:·gc.churn.Par_Eden_Space avgt 10 281.048 ± 157.356 MB/sec [info] AsyncSemaphoreBenchmark.waiters:·gc.churn.Par_Eden_Space.norm avgt 10 104.627 ± 55.264 B/op [info] AsyncSemaphoreBenchmark.waiters:·gc.count avgt 10 13.000 counts [info] AsyncSemaphoreBenchmark.waiters:·gc.time avgt 10 9.000 ms ``` JIRA Issues: CSL-8426, VM-4536 Differential Revision: https://phabricator.twitter.biz/D663434 --- .../concurrent/AsyncSemaphoreBenchmark.scala | 18 ++- .../twitter/concurrent/AsyncSemaphore.scala | 145 ++++++++++-------- .../concurrent/AsyncSemaphoreTest.scala | 38 ++++- 3 files changed, 137 insertions(+), 64 deletions(-) diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncSemaphoreBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncSemaphoreBenchmark.scala index c3cb7a0b8b..7a234eb9b5 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncSemaphoreBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/AsyncSemaphoreBenchmark.scala @@ -8,10 +8,22 @@ import org.openjdk.jmh.annotations._ @Threads(10) class AsyncSemaphoreBenchmark extends StdBenchAnnotations { - private[this] val semaphore: AsyncSemaphore = new AsyncSemaphore(1) + private[this] val noWaitersSemaphore: AsyncSemaphore = new AsyncSemaphore(10) + private[this] val waitersSemaphore: AsyncSemaphore = new AsyncSemaphore(1) + private[this] val mixedSemaphore: AsyncSemaphore = new AsyncSemaphore(5) @Benchmark - def acquireAndRelease(): Unit = { - Await.result(semaphore.acquire().map(_.release())) + def noWaiters(): Unit = { + Await.result(noWaitersSemaphore.acquire().map(_.release())) + } + + @Benchmark + def waiters(): Unit = { + Await.result(waitersSemaphore.acquire().map(_.release())) + } + + @Benchmark + def mixed(): Unit = { + Await.result(mixedSemaphore.acquire().map(_.release())) } } diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala index dd341de3dc..51bce79d36 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncSemaphore.scala @@ -1,10 +1,9 @@ package com.twitter.concurrent import com.twitter.util._ -import java.util.ArrayDeque -import java.util.concurrent.RejectedExecutionException +import java.util.concurrent.{ConcurrentLinkedQueue, RejectedExecutionException} +import java.util.concurrent.atomic.AtomicInteger import scala.annotation.tailrec -import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal /** @@ -50,42 +49,48 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { require(maxWaiters.getOrElse(0) >= 0, s"maxWaiters must be non-negative: $maxWaiters") require(initialPermits > 0, s"initialPermits must be positive: $initialPermits") - // access to `closed`, `waitq`, and `availablePermits` is synchronized - // by locking on `lock` - private[this] var closed: Option[Throwable] = None - private[this] val waitq = new ArrayDeque[Promise[Permit]] - private[this] var availablePermits = initialPermits + // if failedState (Int.MinValue), failed + // else if >= 0, # of available permits + // else if < 0, # of waiters + private[this] final val state = new AtomicInteger(initialPermits) + @volatile private[this] final var failure: Future[Nothing] = null - // Serves as our intrinsic lock. - private[this] final def lock: Object = waitq + private[this] final val failedState = Int.MinValue + private[this] final val waiters = new ConcurrentLinkedQueue[Waiter] + private[this] final val waitersLimit = maxWaiters.map(v => -v).getOrElse(failedState + 1) - private[this] val semaphorePermit = new Permit { - private[this] val ReturnThis = Return(this) - - @tailrec override def release(): Unit = { - val waiter = lock.synchronized { - val next = waitq.pollFirst() - if (next == null) { - availablePermits += 1 - } - next - } - - if (waiter != null) { + private[this] final val permit = new Permit { + override final def release(): Unit = AsyncSemaphore.this.release() + } + private[this] final val permitFuture = Future.value(permit) + private[this] final val permitReturn = Return(permit) + private[this] final val releasePermit: Any => Unit = _ => permit.release() + + @tailrec private[this] final def release(): Unit = { + val s = state.get() + if (s != failedState) { + if (!state.compareAndSet(s, s + 1) || // If the waiter is already satisfied, it must // have been interrupted, so we can simply move on. - if (!waiter.updateIfEmpty(ReturnThis)) { - release() - } + (s < 0 && !pollWaiter().updateIfEmpty(permitReturn))) { + release() } } } - private[this] val futurePermit = Future.value(semaphorePermit) + def numWaiters: Int = { + val s = state.get() + if (s < 0 && s != failedState) -s + else 0 + } - def numWaiters: Int = lock.synchronized(waitq.size) def numInitialPermits: Int = initialPermits - def numPermitsAvailable: Int = lock.synchronized(availablePermits) + + def numPermitsAvailable: Int = { + val s = state.get() + if (s > 0) s + else 0 + } /** * Fail the semaphore and stop it from distributing further permits. Subsequent @@ -93,12 +98,29 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { * are also failed with `exc`. */ def fail(exc: Throwable): Unit = { - val drained = lock.synchronized { - closed = Some(exc) - waitq.asScala.toList + failure = Future.exception(exc) + var s = state.getAndSet(failedState) + // drain the wait queue + while (s < 0 && s != failedState) { + pollWaiter().raise(exc) + s += 1 + } + } + + /** + * Polls from the wait queue until it returns an item. Must be called + * after a state modification that indicates that a new waiter will be available + * shortly. The retry accounts for the race when `acquire` updates the state to + * indicate that a waiter is available but it hasn't been added to the queue yet. + * @return + */ + @tailrec private[this] final def pollWaiter(): Waiter = { + val w = waiters.poll() + if (w != null) { + w + } else { + pollWaiter() } - // delegate dequeuing to the interrupt handler defined in #acquire - drained.foreach(_.raise(exc)) } /** @@ -116,29 +138,26 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { * or a Future.Exception[RejectedExecutionException]` if the configured maximum * number of waiters would be exceeded. */ - def acquire(): Future[Permit] = lock.synchronized { - if (closed.isDefined) - return Future.exception(closed.get) - - if (availablePermits > 0) { - availablePermits -= 1 - futurePermit - } else { - maxWaiters match { - case Some(max) if waitq.size >= max => - MaxWaitersExceededException - case _ => - val promise = new Promise[Permit] - promise.setInterruptHandler { - case t: Throwable => - if (promise.updateIfEmpty(Throw(t))) lock.synchronized { - waitq.remove(promise) - } - } - waitq.addLast(promise) - promise + def acquire(): Future[Permit] = { + @tailrec def loop(): Future[Permit] = { + val s = state.get() + if (s == failedState) { + failure + } else if (s == waitersLimit) { + AsyncSemaphore.MaxWaitersExceededException + } else if (state.compareAndSet(s, s - 1)) { + if (s > 0) { + permitFuture + } else { + val w = new Waiter + waiters.add(w) + w + } + } else { + loop() } } + loop() } /** @@ -162,9 +181,7 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { permit.release() throw e } - f.ensure { - permit.release() - } + f.respond(releasePermit) } /** @@ -178,14 +195,22 @@ class AsyncSemaphore protected (initialPermits: Int, maxWaiters: Option[Int]) { * returned. */ def acquireAndRunSync[T](func: => T): Future[T] = - acquire().flatMap { permit => - Future(func).ensure { + acquire().map { _ => + try { + func + } finally { permit.release() } } } object AsyncSemaphore { + + private final class Waiter extends Promise[Permit] with Promise.InterruptHandler { + // the waiter will be removed from the queue later when a permit is released + override protected def onInterrupt(t: Throwable): Unit = updateIfEmpty(Throw(t)) + } + private val MaxWaitersExceededException = Future.exception(new RejectedExecutionException("Max waiters exceeded")) } diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala index d7d30ee8cc..8ee13d57d7 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncSemaphoreTest.scala @@ -2,7 +2,14 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ import com.twitter.util._ -import java.util.concurrent.{ConcurrentLinkedQueue, RejectedExecutionException, CountDownLatch} +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.{ + ConcurrentLinkedQueue, + CountDownLatch, + Executors, + RejectedExecutionException, + ThreadLocalRandom +} import scala.collection.mutable import org.scalatest.Outcome import org.scalatest.funspec.FixtureAnyFunSpec @@ -387,5 +394,34 @@ class AsyncSemaphoreTest extends FixtureAnyFunSpec { val msgs = results.collect { case Some(Throw(e)) => e.getMessage } assert(msgs.forall(_ == "woop")) } + + it("behaves properly under contention") { () => + val as = new AsyncSemaphore(5) + val exec = Executors.newCachedThreadPool() + val pool = FuturePool(exec) + val active = new AtomicInteger(0) + try { + val done = + Future.collect { + for (i <- 0 until 10) yield { + pool { + for (j <- 0 until 100) { + val res = + as.acquireAndRunSync { + assert(active.incrementAndGet() <= as.numInitialPermits) + Thread.sleep(ThreadLocalRandom.current().nextInt(10)) + assert(active.decrementAndGet() <= as.numInitialPermits) + i * j + } + assert(await(res) == i * j) + } + } + } + } + await(done) + } finally { + exec.shutdown() + } + } } } From c386ece8fb7c04e7d3526ff10d8c33f86787bb33 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Sat, 15 May 2021 06:30:35 +0000 Subject: [PATCH 483/796] util-app: Java friendly API for `onExit` Problem There is no Java-friendly `onExit` and `onExitLast` endpoint in util app. Java users need to return a `BoxedUnit` when calling `onExit(f: => Unit)`. Solution Introduce a Java-friendly `runOnExit(Runnable)` and `runOnExistLast(Runnable)`API. JIRA Issues: CSL-9789 Differential Revision: https://phabricator.twitter.biz/D511536 --- CHANGELOG.rst | 7 +++ .../src/main/scala/com/twitter/app/App.scala | 50 +++++++++++++++++++ .../test/scala/com/twitter/app/AppTest.scala | 14 ++++++ 3 files changed, 71 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6e2dd6303f..15b3a12276 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,11 +7,18 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + * util-validator: Introduce new library for case class validations (akin to Java bean validation) which follows the Jakarta Bean Validation specification (https://beanvalidation.org/) by wrapping the Hibernate Validator library and thus supports `jakarta.validation.Constraint` annotations and validators for annotating and validating fields of Scala case classes. ``PHAB_ID=D638603`` +* util-app: Introduce a Java-friendly API `c.t.app.App#runOnExit(Runnable)` and +`c.t.app.App#runOnExitLast(Runnable)` to allow Java 8 users to call `c.t.app.App#runOnExit` +and `c.t.app.App#runOnExitLast` with lambda expressions. ``PHAB_ID=D511536`` + 21.4.0 ------ diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index 51a8bd9345..32cfb2b38d 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -258,6 +258,8 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { * Invoke `f` when shutdown is requested. Exit hooks run in parallel and are * executed after all postmains complete. The thread resumes when all exit * hooks complete or `closeDeadline` expires. + * + * @see [[runOnExit(Runnable)]] for a variant that is more suitable for Java. */ protected final def onExit(f: => Unit): Unit = { closeOnExit { @@ -267,10 +269,35 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { } } + /** + * Invoke `runnable.run()` when shutdown is requested. Exit hooks run in parallel + * and are executed after all postmains complete. The thread resumes when all exit + * hooks complete or `closeDeadline` expires. + * + * This is a Java friendly API to allow Java 8 users to invoke `onExit` with lambda + * expressions. + * + * For example (in Java): + * + * {{{ + * runOnExit(() -> { + * clientA.close(); + * clientB.close(); + * }); + * }}} + * + * @see [[onExit(f: => Unit)]] for a variant that is more suitable for Scala. + */ + protected final def runOnExit(runnable: Runnable): Unit = { + onExit(runnable.run()) + } + /** * Invoke `f` when shutdown is requested. Exit hooks run in parallel and are * executed after all closeOnExit functions complete. The thread resumes when all exit * hooks complete or `closeDeadline` expires. + * + * @see [[runOnExitLast(Runnable)]] for a variant that is more suitable for Java. */ protected final def onExitLast(f: => Unit): Unit = { closeOnExitLast { @@ -280,6 +307,29 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { } } + /** + * Invoke `runnable.run()` when shutdown is requested. Exit hooks run in parallel + * and are executed after all closeOnExit functions complete. The thread resumes + * when all exit hooks complete or `closeDeadline` expires. + * + * This is a Java friendly API to allow Java 8 users to invoke `onExitLast` with lambda + * expressions. + * + * For example (in Java): + * + * {{{ + * runOnExitLast(() -> { + * clientA.close(); + * clientB.close(); + * }); + * }}} + * + * @see [[onExitLast(f: => Unit)]] for a variant that is more suitable for Scala. + */ + protected final def runOnExitLast(runnable: Runnable): Unit = { + onExitLast(runnable.run()) + } + /** * Invoke `f` after the user's main has exited. */ diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 85bf6690ff..ea66b2fd1b 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -114,6 +114,20 @@ class AppTest extends AnyFunSuite { assert(q.asScala.toSeq == Seq(0, 1, 2, 3, 4, 5)) } + test("App: order of hooks with runOnExit and runOnExitLast") { + val q = new ConcurrentLinkedQueue[Int] + class Test1 extends App { + runOnExitLast(() => q.add(5)) + runOnExit(() => q.add(4)) + postmain(q.add(3)) + def main(): Unit = q.add(2) + premain(q.add(1)) + init(q.add(0)) + } + new Test1().main(Array.empty) + assert(q.asScala.toSeq == Seq(0, 1, 2, 3, 4, 5)) + } + test("App: order of hooks via observer") { val enterQueue = new ConcurrentLinkedQueue[Event] val completeQueue = new ConcurrentLinkedQueue[Event] From 7cbd14a7db2487bc3b3ed8ab88f4fe245c63715b Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Wed, 19 May 2021 17:52:03 +0000 Subject: [PATCH 484/796] util-stats: store expressions indexed via a new caseclass instead of string Problem Expressions are stored with a brittle String key. Leading to cases where expressions may clobber one another unexpectedly. Solution Swap the String key (which contains the name and serviceName of the expression) for a caseclass (composed of the name, serviceName, and namespaces of the expression). JIRA Issues: CSL-10620 Differential Revision: https://phabricator.twitter.biz/D666502 --- .../twitter/finagle/stats/SchemaRegistry.scala | 4 ++-- .../finagle/stats/exp/ExpressionSchema.scala | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala index 2f3a1ba64a..7c26a4e325 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala @@ -1,6 +1,6 @@ package com.twitter.finagle.stats -import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} /** * Interface used via the LoadService mechanism to obtain an @@ -13,6 +13,6 @@ private[twitter] trait SchemaRegistry { def schemas(): Map[String, MetricSchema] - def expressions(): Map[String, ExpressionSchema] + def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index aa23727b72..38e72afa25 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -52,8 +52,24 @@ case class ExpressionSchema private ( case _ => // should not happen } } + + def schemaKey(): ExpressionSchemaKey = { + ExpressionSchemaKey(name, labels.serviceName, namespaces) + } } +/** + * ExpressionSchemaKey is a class that exists to serve as a key into a Map of ExpressionSchemas. + * It is simply a subset of the fields of the ExpressionSchema. Namely: + * @param name + * @param serviceName + * @param namespaces + */ +case class ExpressionSchemaKey( + name: String, + serviceName: Option[String], + namespaces: Seq[String]) + // expose for testing in twitter-server private[twitter] object ExpressionSchema { def apply(name: String, expr: Expression): ExpressionSchema = From 4956ef0dcc44ecd2ed70f9357cf9287f9a38eaf7 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 20 May 2021 20:16:10 +0000 Subject: [PATCH 485/796] Tests: Migrate to AnyFunSuite Problem FunSuite is deprecated for a while, and most importantly, there are so many warning messages in the sbt build output. Solution Migrate them to AnyFunSuite via autofix: https://github.com/scalatest/autofix/tree/master/3.1.x Differential Revision: https://phabricator.twitter.biz/D672801 --- .../finagle/stats/RoleConfiguredStatsReceiverTest.scala | 4 ++-- .../com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala index b52585d438..7750dcc135 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala @@ -1,8 +1,8 @@ package com.twitter.finagle.stats -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite -class RoleConfiguredStatsReceiverTest extends FunSuite { +class RoleConfiguredStatsReceiverTest extends AnyFunSuite { test("RoleConfiguredSR configures metadata role correctly") { val mem = new InMemoryStatsReceiver val clientSR = new RoleConfiguredStatsReceiver(mem, Client) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 198a13042b..d7b18116ff 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -3,10 +3,10 @@ package com.twitter.finagle.stats.exp import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats._ import com.twitter.util.{Stopwatch, Time, TimeControl} -import org.scalatest.FunSuite import scala.util.control.NonFatal +import org.scalatest.funsuite.AnyFunSuite -class ExpressionSchemaTest extends FunSuite { +class ExpressionSchemaTest extends AnyFunSuite { trait Ctx { val sr = new InMemoryStatsReceiver val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) From a70d61c41df1269440432034a97906fcafca5534 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 21 May 2021 23:00:28 +0000 Subject: [PATCH 486/796] metrics: Introduce MetricBuilderRegistry for instrumenting expressions in server stack Problem Metrics creations are scattered in many stack modules, each stack module has its filter and independent from others. However, instrumenting an expression needs access to all component metrics. Solution Introduce a stack param to cache metrics as well as providing lazy vals to create expressions. Result The original approach was having a stack.module that sits at the end of the stack where all needed metrics are generated. the downside is it registers the same expressions many times (= connections), and the last override the previous ones. The current approach should reduce the expression evaluation time, but requires implementors to have more context on the finagle stack for metrics creation sequence. Differential Revision: https://phabricator.twitter.biz/D663915 --- .../finagle/stats/InMemoryStatsReceiver.scala | 1 + .../com/twitter/finagle/stats/MetricBuilder.scala | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 6dab7158cc..91fd48e7b6 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -257,6 +257,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { stats.clear() gauges.clear() schemas.clear() + preloadExpressions.clear() } private[this] def toHistogramDetail(addedValues: Seq[Float]): HistogramDetail = { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index b00761f0c1..ddc9180790 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -85,6 +85,20 @@ object MetricBuilder { ) } } +object Metadata { + + /** + * extract the MetricBuilder from Metadata + */ + def getMetricBuilder(metadata: Metadata): Option[MetricBuilder] = { + metadata match { + case metricBuilder: MetricBuilder => Some(metricBuilder) + case NoMetadata => None + case MultiMetadata(schemas) => + schemas.find(_ != NoMetadata).asInstanceOf[Option[MetricBuilder]] + } + } +} sealed trait Metadata From 95f1189dfd302a353baee14d4e371594a538dd80 Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 27 May 2021 15:01:50 +0000 Subject: [PATCH 487/796] Twitter-oss: Prepare OSS libraries for release 21.5.0 Problem We want to release the next versions of our Twitter OSS libraries 21.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D677481 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 15b3a12276..bb32a87f79 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.5.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 8d16368213..288be24b2e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.4.0" +val utilCore = "com.twitter" %% "util-core" % "21.5.0" ``` # Units diff --git a/build.sbt b/build.sbt index 0d3f198ac1..d88832ac1f 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.5.0-SNAPSHOT" +val releaseVersion = "21.5.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 0846bb8b5432529217b346a17dcb74643511baa8 Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 27 May 2021 16:44:13 +0000 Subject: [PATCH 488/796] Twitter-oss: update OSS libraries post-release to 21.6.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.6.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D678026 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 288be24b2e..ed4aeede24 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index d88832ac1f..5df28d3cee 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ import scoverage.ScoverageKeys // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.5.0" +val releaseVersion = "21.6.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 75be5bedbf6f01391ee91da34244fe3d0f5eacac Mon Sep 17 00:00:00 2001 From: Matt Dannenberg Date: Fri, 28 May 2021 19:32:49 +0000 Subject: [PATCH 489/796] twitter-server: add name and namespace params to ExpressionHandler Problem Consumers of the /admin/metric/expressions.json endpoint would like to be able to grab a subset of the expression by name or namespace. There is no way to do so currently. Solution Add optional query params name and namespace which are used to filter the list of expressions returned by the /admin/metric/expressions.json endpoint. JIRA Issues: CSL-10622 Differential Revision: https://phabricator.twitter.biz/D672594 --- .../finagle/stats/InMemoryStatsReceiver.scala | 2 +- .../twitter/finagle/stats/exp/ExpressionSchema.scala | 12 ++++++------ .../finagle/stats/exp/ExpressionSchemaTest.scala | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 91fd48e7b6..3ea627f9a7 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -289,7 +289,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { val expressionId = (schema.labels.serviceName match { case Some(serviceName) => schema.name + "_" + serviceName case None => schema.name - }) + schema.namespaces.mkString("_") + }) + schema.namespace.mkString("_") preloadExpressions.put(expressionId, schema) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index 38e72afa25..258a463b89 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -7,7 +7,7 @@ import com.twitter.finagle.stats.{MetricUnit, SourceRole, StatsReceiver, Unspeci * * @param name this is going to be an important query key when fetching expressions * @param labels service related information, see [[ExpressionLabels]] - * @param namespaces a list of namespaces the expression belongs to, this is usually + * @param namespace a list of namespaces the expression belongs to, this is usually * used to indicate a tenant in a multi-tenancy systems or similar concepts. * For standalone services, this should be empty. * @param expr class representation of the expression, see [[Expression]] @@ -20,7 +20,7 @@ case class ExpressionSchema private ( name: String, labels: ExpressionLabels, expr: Expression, - namespaces: Seq[String], + namespace: Seq[String], bounds: Bounds, description: String, unit: MetricUnit, @@ -36,8 +36,8 @@ case class ExpressionSchema private ( * a single string. This is for multi-tenancy system tenants or similar concepts and should * remain unset for standalone services. */ - def withNamespaces(name: String*): ExpressionSchema = - copy(namespaces = this.namespaces ++ name) + def withNamespace(name: String*): ExpressionSchema = + copy(namespace = this.namespace ++ name) private[finagle] def withRole(role: SourceRole): ExpressionSchema = copy(labels = labels.copy(role = role)) @@ -54,7 +54,7 @@ case class ExpressionSchema private ( } def schemaKey(): ExpressionSchemaKey = { - ExpressionSchemaKey(name, labels.serviceName, namespaces) + ExpressionSchemaKey(name, labels.serviceName, namespace) } } @@ -76,7 +76,7 @@ private[twitter] object ExpressionSchema { ExpressionSchema( name = name, labels = ExpressionLabels.empty, - namespaces = Seq.empty, + namespace = Seq.empty, expr = expr, bounds = Unbounded.get, description = "Unspecified", diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index d7b18116ff..d67534bc22 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -88,14 +88,14 @@ class ExpressionSchemaTest extends AnyFunSuite { test("expressions with different namespaces are all stored") { new Ctx { val successRate1 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) - .withNamespaces("section1") + .withNamespace("section1") .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) .withUnit(Percentage) .withDescription("The success rate of the slow query") .register() val successRate2 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) - .withNamespaces("section2", "a") + .withNamespace("section2", "a") .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) .withUnit(Percentage) .withDescription("The success rate of the slow query") @@ -103,7 +103,7 @@ class ExpressionSchemaTest extends AnyFunSuite { assert(sr.expressions.values.size == 2) val namespaces = sr.expressions.values.toSeq.map { exprSchema => - exprSchema.namespaces + exprSchema.namespace } assert(namespaces.contains(Seq("section1"))) assert(namespaces.contains(Seq("section2", "a"))) From 2fa17e27d018caec8b6e865fcadbc5d0d9f2950c Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 3 Jun 2021 03:26:30 +0000 Subject: [PATCH 490/796] metrics: Remove MetricSchema and make it an attribute of MetricBuilder Problem The metric types are defined in a wrapper class MetricSchema - CounterSchema, GaugeSchema, and HistogramSchema. This is becoming inconvenient that we need to re-wrap MetricBuilders when we made a change to them. And requires StatsReceiver to carry over the metric type info, while ideally, it should stick with MetricBuilder itself. Solution Remove the wrapper and add enum fields in MetricBuilder. JIRA Issues: CSL-10923 Differential Revision: https://phabricator.twitter.biz/D668739 --- CHANGELOG.rst | 7 ++ .../main/scala/com/twitter/jvm/JvmStats.scala | 7 +- .../stats/BroadcastStatsReceiver.scala | 30 +++---- .../finagle/stats/CumulativeGauge.scala | 16 ++-- .../finagle/stats/DenylistStatsReceiver.scala | 12 +-- .../finagle/stats/InMemoryStatsReceiver.scala | 87 ++++++++++--------- .../KeyIndicatorMarkingStatsReceiver.scala | 12 +-- .../finagle/stats/LazyStatsReceiver.scala | 24 +++-- .../twitter/finagle/stats/MetricBuilder.scala | 43 ++++++--- .../twitter/finagle/stats/MetricSchema.scala | 24 ----- .../stats/NameTranslatingStatsReceiver.scala | 18 ++-- .../finagle/stats/NullStatsReceiver.scala | 6 +- .../RelativeNameMarkingStatsReceiver.scala | 21 ++--- .../stats/RoleConfiguredStatsReceiver.scala | 12 +-- .../finagle/stats/RollupStatsReceiver.scala | 15 ++-- .../finagle/stats/SchemaRegistry.scala | 2 +- .../twitter/finagle/stats/StatsReceiver.scala | 36 +++++--- .../finagle/stats/StatsReceiverProxy.scala | 6 +- .../VerbosityAdjustingStatsReceiver.scala | 12 +-- .../finagle/stats/exp/Expression.scala | 56 ++++++------ .../stats/MetricBuilderCompilationTest.java | 8 +- .../stats/InMemoryStatsReceiverTest.scala | 34 +++++--- .../finagle/stats/MetricBuilderTest.scala | 17 +++- .../RoleConfiguredStatsReceiverTest.scala | 20 ++--- .../stats/exp/ExpressionSchemaTest.scala | 16 +++- 25 files changed, 286 insertions(+), 255 deletions(-) delete mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bb32a87f79..3b2a228da0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-stats: Removed MetricSchema trait (CounterSchema, GaugeSchema and HistogramSchema). + StatReceiver derived classes use MetricBuilder directly to create counters, gauges and stats. + ``PHAB_ID=D668739`` + 21.5.0 ------ diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index ca23be1707..6254f007bb 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -1,8 +1,9 @@ package com.twitter.jvm import com.twitter.conversions.StringOps._ +import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.StatsReceiver -import java.lang.management.{ManagementFactory, BufferPoolMXBean} +import java.lang.management.{BufferPoolMXBean, ManagementFactory} import scala.collection.mutable import scala.jdk.CollectionConverters._ @@ -125,10 +126,10 @@ object JvmStats { val gcStats = stats.scope("gc") gcPool.foreach { gc => val name = gc.getName.regexSub("""[^\w]""".r) { m => "_" } - gauges.add(gcStats.metricBuilder().withCounterishGauge.gauge(name, "cycles") { + gauges.add(gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "cycles") { gc.getCollectionCount }) - gauges.add(gcStats.metricBuilder().withCounterishGauge.gauge(name, "msec") { + gauges.add(gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "msec") { gc.getCollectionTime }) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala index ca7a504eff..729114ea56 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala @@ -19,19 +19,19 @@ object BroadcastStatsReceiver { with DelegatingStatsReceiver { val repr: AnyRef = this - def counter(schema: CounterSchema) = new BroadcastCounter.Two( - first.counter(schema), - second.counter(schema) + def counter(metricBuilder: MetricBuilder) = new BroadcastCounter.Two( + first.counter(metricBuilder), + second.counter(metricBuilder) ) - def stat(schema: HistogramSchema) = new BroadcastStat.Two( - first.stat(schema), - second.stat(schema) + def stat(metricBuilder: MetricBuilder) = new BroadcastStat.Two( + first.stat(metricBuilder), + second.stat(metricBuilder) ) - def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { - val firstGauge = first.addGauge(schema)(f) - val secondGauge = second.addGauge(schema)(f) + def addGauge(metricBuilder: MetricBuilder)(f: => Float) = new Gauge { + val firstGauge = first.addGauge(metricBuilder)(f) + val secondGauge = second.addGauge(metricBuilder)(f) def remove(): Unit = { firstGauge.remove() secondGauge.remove() @@ -53,14 +53,14 @@ object BroadcastStatsReceiver { private class N(srs: Seq[StatsReceiver]) extends StatsReceiver with DelegatingStatsReceiver { val repr: AnyRef = this - def counter(schema: CounterSchema) = - BroadcastCounter(srs.map { _.counter(schema) }) + def counter(metricBuilder: MetricBuilder) = + BroadcastCounter(srs.map { _.counter(metricBuilder) }) - def stat(schema: HistogramSchema) = - BroadcastStat(srs.map { _.stat(schema) }) + def stat(metricBuilder: MetricBuilder) = + BroadcastStat(srs.map { _.stat(metricBuilder) }) - def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { - val gauges = srs.map { _.addGauge(schema)(f) } + def addGauge(metricBuilder: MetricBuilder)(f: => Float) = new Gauge { + val gauges = srs.map { _.addGauge(metricBuilder)(f) } def remove(): Unit = gauges.foreach { _.remove() } def metadata: Metadata = MultiMetadata(gauges.map(_.metadata)) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index bacafab967..28bdce7871 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -1,6 +1,7 @@ package com.twitter.finagle.stats import com.github.benmanes.caffeine.cache.{Cache, Caffeine, RemovalCause, RemovalListener} +import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.util.lint._ import java.lang.{Boolean => JBoolean} import java.util.concurrent.{ConcurrentHashMap, Executor, ForkJoinPool} @@ -120,18 +121,18 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => * The StatsReceiver implements these. They provide the cumulative * gauges. */ - protected[this] def registerGauge(schema: GaugeSchema, f: => Float): Unit + protected[this] def registerGauge(metricBuilder: MetricBuilder, f: => Float): Unit protected[this] def deregisterGauge(name: Seq[String]): Unit - private[this] def getWhenNotPresent(schema: GaugeSchema) = whenNotPresent(schema) + private[this] def getWhenNotPresent(metricBuilder: MetricBuilder) = whenNotPresent(metricBuilder) /** The executor that will be used for expiring gauges */ def executor: Executor = ForkJoinPool.commonPool() - private[this] def whenNotPresent(schema: GaugeSchema) = + private[this] def whenNotPresent(metricBuilder: MetricBuilder) = new JFunction[Seq[String], CumulativeGauge] { def apply(key: Seq[String]): CumulativeGauge = new CumulativeGauge(executor) { - self.registerGauge(schema, getValue) + self.registerGauge(metricBuilder, getValue) // The number of registers starts at `0` because every new gauge will cause a // registration, even the one that created the `CumulativeGauge`. Because 0 is @@ -167,12 +168,13 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => } } - def addGauge(schema: GaugeSchema)(f: => Float): Gauge = { + def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { + validateMetricType(metricBuilder, GaugeType) var gauge: Gauge = null while (gauge == null) { val cumulativeGauge = - gauges.computeIfAbsent(schema.metricBuilder.name, getWhenNotPresent(schema)) - gauge = cumulativeGauge.addGauge(f, schema.metricBuilder) + gauges.computeIfAbsent(metricBuilder.name, getWhenNotPresent(metricBuilder)) + gauge = cumulativeGauge.addGauge(f, metricBuilder) } gauge } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala index df6074abed..3708a59e0f 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/DenylistStatsReceiver.scala @@ -45,14 +45,14 @@ object DenylistStatsReceiver { class DenylistStatsReceiver(protected val self: StatsReceiver, denylisted: Seq[String] => Boolean) extends StatsReceiverProxy { - override def counter(schema: CounterSchema) = - getStatsReceiver(schema.metricBuilder.name).counter(schema) + override def counter(metricBuilder: MetricBuilder) = + getStatsReceiver(metricBuilder.name).counter(metricBuilder) - override def stat(schema: HistogramSchema) = - getStatsReceiver(schema.metricBuilder.name).stat(schema) + override def stat(metricBuilder: MetricBuilder) = + getStatsReceiver(metricBuilder.name).stat(metricBuilder) - override def addGauge(schema: GaugeSchema)(f: => Float) = - getStatsReceiver(schema.metricBuilder.name).addGauge(schema)(f) + override def addGauge(metricBuilder: MetricBuilder)(f: => Float) = + getStatsReceiver(metricBuilder.name).addGauge(metricBuilder)(f) private[this] def getStatsReceiver(name: Seq[String]): StatsReceiver = if (denylisted(name)) NullStatsReceiver else self diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 3ea627f9a7..e2cd333193 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -1,5 +1,6 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} import com.twitter.finagle.stats.exp.Expression.replaceExpression import com.twitter.finagle.stats.exp.ExpressionSchema import java.io.PrintStream @@ -68,8 +69,8 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { val gauges: mutable.Map[Seq[String], () => Float] = new ConcurrentHashMap[Seq[String], () => Float]().asScala - val schemas: mutable.Map[Seq[String], MetricSchema] = - new ConcurrentHashMap[Seq[String], MetricSchema]().asScala + val schemas: mutable.Map[Seq[String], MetricBuilder] = + new ConcurrentHashMap[Seq[String], MetricBuilder]().asScala private val metricBuilders: ConcurrentHashMap[Int, MetricBuilder] = new ConcurrentHashMap[Int, MetricBuilder]() @@ -86,110 +87,116 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } override def counter(name: String*): ReadableCounter = - counter(CounterSchema(this.metricBuilder().withName(name: _*))) + counter(this.metricBuilder(CounterType).withName(name: _*)) /** * Creates a [[ReadableCounter]] of the given `name`. */ - def counter(schema: CounterSchema): ReadableCounter = + def counter(metricBuilder: MetricBuilder): ReadableCounter = { + validateMetricType(metricBuilder, CounterType) new ReadableCounter { - verbosity += schema.metricBuilder.name -> schema.metricBuilder.verbosity + verbosity += metricBuilder.name -> metricBuilder.verbosity // eagerly initialize counters.synchronized { - if (!counters.contains(schema.metricBuilder.name)) { - counters(schema.metricBuilder.name) = 0 - schemas(schema.metricBuilder.name) = schema - storeMetricBuilder(schema) + if (!counters.contains(metricBuilder.name)) { + counters(metricBuilder.name) = 0 + schemas(metricBuilder.name) = metricBuilder + storeMetricBuilder(metricBuilder) } } def incr(delta: Long): Unit = counters.synchronized { val oldValue = apply() - counters(schema.metricBuilder.name) = oldValue + delta + counters(metricBuilder.name) = oldValue + delta } - def apply(): Long = counters.getOrElse(schema.metricBuilder.name, 0) + def apply(): Long = counters.getOrElse(metricBuilder.name, 0) - def metadata: Metadata = schema.metricBuilder + def metadata: Metadata = metricBuilder override def toString: String = - s"Counter(${schema.metricBuilder.name.mkString("/")}=${apply()})" + s"Counter(${metricBuilder.name.mkString("/")}=${apply()})" } + } override def stat(name: String*): ReadableStat = - stat(HistogramSchema(this.metricBuilder().withName(name: _*))) + stat(this.metricBuilder(HistogramType).withName(name: _*)) /** * Creates a [[ReadableStat]] of the given `name`. */ - def stat(schema: HistogramSchema): ReadableStat = + def stat(metricBuilder: MetricBuilder): ReadableStat = { + validateMetricType(metricBuilder, HistogramType) new ReadableStat { - verbosity += schema.metricBuilder.name -> schema.metricBuilder.verbosity + verbosity += metricBuilder.name -> metricBuilder.verbosity // eagerly initialize stats.synchronized { - if (!stats.contains(schema.metricBuilder.name)) { - stats(schema.metricBuilder.name) = Nil - schemas(schema.metricBuilder.name) = schema - storeMetricBuilder(schema) + if (!stats.contains(metricBuilder.name)) { + stats(metricBuilder.name) = Nil + schemas(metricBuilder.name) = metricBuilder + storeMetricBuilder(metricBuilder) } } def add(value: Float): Unit = stats.synchronized { val oldValue = apply() - stats(schema.metricBuilder.name) = oldValue :+ value + stats(metricBuilder.name) = oldValue :+ value } - def apply(): Seq[Float] = stats.getOrElse(schema.metricBuilder.name, Seq.empty) + def apply(): Seq[Float] = stats.getOrElse(metricBuilder.name, Seq.empty) - def metadata: Metadata = schema.metricBuilder + def metadata: Metadata = metricBuilder override def toString: String = { val vals = apply() - s"Stat(${schema.metricBuilder.name.mkString("/")}=${statValuesToStr(vals)})" + s"Stat(${metricBuilder.name.mkString("/")}=${statValuesToStr(vals)})" } } + } /** * Creates a [[Gauge]] of the given `name`. */ - def addGauge(schema: GaugeSchema)(f: => Float) = + def addGauge(metricBuilder: MetricBuilder)(f: => Float) = { + validateMetricType(metricBuilder, GaugeType) new Gauge { - gauges += schema.metricBuilder.name -> (() => f) - schemas += schema.metricBuilder.name -> schema - verbosity += schema.metricBuilder.name -> schema.metricBuilder.verbosity - storeMetricBuilder(schema) + gauges += metricBuilder.name -> (() => f) + schemas += metricBuilder.name -> metricBuilder + verbosity += metricBuilder.name -> metricBuilder.verbosity + storeMetricBuilder(metricBuilder) def remove(): Unit = { - gauges -= schema.metricBuilder.name - removeMetricBuilder(schema) - schemas -= schema.metricBuilder.name + gauges -= metricBuilder.name + removeMetricBuilder(metricBuilder) + schemas -= metricBuilder.name } - def metadata: Metadata = schema.metricBuilder + def metadata: Metadata = metricBuilder override def toString: String = { // avoid holding a reference to `f` - val current = gauges.get(schema.metricBuilder.name) match { + val current = gauges.get(metricBuilder.name) match { case Some(fn) => fn() case None => -0.0f } - s"Gauge(${schema.metricBuilder.name.mkString("/")}=$current)" + s"Gauge(${metricBuilder.name.mkString("/")}=$current)" } } + } - private[this] def storeMetricBuilder(schema: MetricSchema): Unit = { - schema.metricBuilder.kernel match { - case Some(kernel) => metricBuilders.put(kernel, schema.metricBuilder) + private[this] def storeMetricBuilder(metricBuilder: MetricBuilder): Unit = { + metricBuilder.kernel match { + case Some(kernel) => metricBuilders.put(kernel, metricBuilder) case None => } } - private[this] def removeMetricBuilder(schema: MetricSchema): Unit = { - schema.metricBuilder.kernel match { + private[this] def removeMetricBuilder(metricBuilder: MetricBuilder): Unit = { + metricBuilder.kernel match { case Some(kernel) => metricBuilders.remove(kernel) case None => } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/KeyIndicatorMarkingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/KeyIndicatorMarkingStatsReceiver.scala index eb4239280d..e9d7a8989a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/KeyIndicatorMarkingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/KeyIndicatorMarkingStatsReceiver.scala @@ -9,15 +9,15 @@ package com.twitter.finagle.stats private[finagle] class KeyIndicatorMarkingStatsReceiver(protected val self: StatsReceiver) extends StatsReceiverProxy { - override def counter(counterSchema: CounterSchema): Counter = { - self.counter(CounterSchema(counterSchema.metricBuilder.withKeyIndicator())) + override def counter(metricBuilder: MetricBuilder): Counter = { + self.counter(metricBuilder.withKeyIndicator()) } - override def stat(histogramSchema: HistogramSchema): Stat = { - self.stat(HistogramSchema(histogramSchema.metricBuilder.withKeyIndicator())) + override def stat(metricBuilder: MetricBuilder): Stat = { + self.stat(metricBuilder.withKeyIndicator()) } - override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { - self.addGauge(GaugeSchema(gaugeSchema.metricBuilder.withKeyIndicator()))(f) + override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { + self.addGauge(metricBuilder.withKeyIndicator())(f) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala index 8765bc27ec..f87c3f4bc8 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/LazyStatsReceiver.scala @@ -1,5 +1,7 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.MetricBuilder.{CounterType, HistogramType} + /** * Wraps an underlying [[StatsReceiver]] to ensure that derived counters and * stats will not start exporting metrics until `incr` or `add` is first called @@ -19,15 +21,21 @@ package com.twitter.finagle.stats * modeling whether a gauge is "used" or not. */ final class LazyStatsReceiver(val self: StatsReceiver) extends StatsReceiverProxy { - override def counter(schema: CounterSchema): Counter = new Counter { - private[this] lazy val underlying = self.counter(schema) - def incr(delta: Long): Unit = underlying.incr(delta) - def metadata: Metadata = schema.metricBuilder + override def counter(metricBuilder: MetricBuilder): Counter = { + validateMetricType(metricBuilder, CounterType) + new Counter { + private[this] lazy val underlying = self.counter(metricBuilder) + def incr(delta: Long): Unit = underlying.incr(delta) + def metadata: Metadata = metricBuilder + } } - override def stat(schema: HistogramSchema): Stat = new Stat { - private[this] lazy val underlying = self.stat(schema) - def add(value: Float): Unit = underlying.add(value) - def metadata: Metadata = schema.metricBuilder + override def stat(metricBuilder: MetricBuilder): Stat = { + validateMetricType(metricBuilder, HistogramType) + new Stat { + private[this] lazy val underlying = self.stat(metricBuilder) + def add(value: Float): Unit = underlying.add(value) + def metadata: Metadata = metricBuilder + } } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index ddc9180790..e708b42c22 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -1,5 +1,6 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType, MetricType} import java.util.function.Supplier import scala.annotation.varargs @@ -66,6 +67,7 @@ object MetricBuilder { processPath: Option[String] = None, percentiles: IndexedSeq[Double] = IndexedSeq.empty, isCounterishGauge: Boolean = false, + metricType: MetricType, statsReceiver: StatsReceiver ): MetricBuilder = { new MetricBuilder( @@ -81,9 +83,19 @@ object MetricBuilder { percentiles, isCounterishGauge, kernel = None, + metricType, statsReceiver ) } + + /** + * Indicate the Metric type, [[CounterType]] will create a [[Counter]], + * [[GaugeType]] will create a [[Gauge]], and [[HistogramType]] will create a [[Stat]]. + */ + sealed trait MetricType + case object CounterType extends MetricType + case object GaugeType extends MetricType + case object HistogramType extends MetricType } object Metadata { @@ -126,6 +138,7 @@ class MetricBuilder private ( val isCounterishGauge: Boolean, // see withKernel val kernel: Option[Int], + val metricType: MetricType, val statsReceiver: StatsReceiver) extends Metadata { @@ -146,6 +159,7 @@ class MetricBuilder private ( processPath: Option[String] = this.processPath, isCounterishGauge: Boolean = this.isCounterishGauge, percentiles: IndexedSeq[Double] = this.percentiles, + metricType: MetricType = this.metricType, kernel: Option[Int] = this.kernel ): MetricBuilder = { new MetricBuilder( @@ -161,6 +175,7 @@ class MetricBuilder private ( percentiles = percentiles, isCounterishGauge = isCounterishGauge, statsReceiver = this.statsReceiver, + metricType = metricType, kernel = kernel ) } @@ -207,25 +222,27 @@ class MetricBuilder private ( def withNoCounterishGauge: MetricBuilder = this.copy(isCounterishGauge = false) /** - * Generates a CounterSchema which can be used to create a counter in a StatsReceiver. + * Generates a CounterType MetricBuilder which can be used to create a counter in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. - * @return a CounterSchema describing a counter. + * @return a Countertype MetricBuilder. */ - private[MetricBuilder] final def counterSchema: CounterSchema = CounterSchema(this) + private[MetricBuilder] final def counterSchema: MetricBuilder = + this.copy(metricType = CounterType) /** - * Generates a GaugeSchema which can be used to create a gauge in a StatsReceiver. + * Generates a GaugeType MetricBuilder which can be used to create a gauge in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. - * @return a GaugeSchema describing a gauge. + * @return a GaugeType MetricBuilder. */ - private[MetricBuilder] final def gaugeSchema: GaugeSchema = GaugeSchema(this) + private[MetricBuilder] final def gaugeSchema: MetricBuilder = this.copy(metricType = GaugeType) /** - * Generates a HistogramSchema which can be used to create a histogram in a StatsReceiver. + * Generates a HistogramType MetricBuilder which can be used to create a histogram in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. - * @return a HistogramSchema describing a histogram. + * @return a HistogramType MetricBuilder. */ - private[MetricBuilder] final def histogramSchema: HistogramSchema = HistogramSchema(this) + private[MetricBuilder] final def histogramSchema: MetricBuilder = + this.copy(metricType = HistogramType) /** * Produce a counter as described by the builder inside the underlying StatsReceiver. @@ -281,7 +298,8 @@ class MetricBuilder private ( name == that.name && relativeName == that.relativeName && processPath == that.processPath && - percentiles == that.percentiles + percentiles == that.percentiles && + metricType == that.metricType case _ => false } @@ -296,13 +314,14 @@ class MetricBuilder private ( name, relativeName, processPath, - percentiles) + percentiles, + metricType) val hashCodes = keyIndicator.hashCode() +: state.map(_.hashCode()) hashCodes.foldLeft(0)((a, b) => 31 * a + b) } override def toString(): String = { val nameString = name.mkString(metadataScopeSeparator()) - s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $nameString, $relativeName, $processPath, $percentiles, $statsReceiver)" + s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $nameString, $relativeName, $processPath, $percentiles, $metricType, $statsReceiver)" } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala deleted file mode 100644 index f5d49f81ac..0000000000 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricSchema.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.finagle.stats - -/** - * This trait represents a metric with the full set of its metadata. - */ -sealed abstract class MetricSchema(val metricBuilder: MetricBuilder) - -/** - * Represents a counter with its assorted metadata. - */ -case class CounterSchema(override val metricBuilder: MetricBuilder) - extends MetricSchema(metricBuilder) - -/** - * Represents a gauge with its assorted metadata. - */ -case class GaugeSchema private (override val metricBuilder: MetricBuilder) - extends MetricSchema(metricBuilder) - -/** - * Represents a histogram with its assorted metadata. - */ -case class HistogramSchema private (override val metricBuilder: MetricBuilder) - extends MetricSchema(metricBuilder) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index 0c64344227..99a57d5cbe 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -17,22 +17,16 @@ abstract class NameTranslatingStatsReceiver( protected def translate(name: Seq[String]): Seq[String] - override def counter(counterSchema: CounterSchema): Counter = { - self.counter( - CounterSchema( - counterSchema.metricBuilder.withName(translate(counterSchema.metricBuilder.name): _*))) + override def counter(metricBuilder: MetricBuilder): Counter = { + self.counter(metricBuilder.withName(translate(metricBuilder.name): _*)) } - override def stat(histogramSchema: HistogramSchema): Stat = { - self.stat( - HistogramSchema( - histogramSchema.metricBuilder.withName(translate(histogramSchema.metricBuilder.name): _*))) + override def stat(metricBuilder: MetricBuilder): Stat = { + self.stat(metricBuilder.withName(translate(metricBuilder.name): _*)) } - override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { - self.addGauge( - GaugeSchema( - gaugeSchema.metricBuilder.withName(translate(gaugeSchema.metricBuilder.name): _*)))(f) + override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { + self.addGauge(metricBuilder.withName(translate(metricBuilder.name): _*))(f) } override def toString: String = s"$self/$namespacePrefix" diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala index 0f8ba9b610..0926044b20 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NullStatsReceiver.scala @@ -29,9 +29,9 @@ class NullStatsReceiver extends StatsReceiver { def repr: NullStatsReceiver = this - def counter(schema: CounterSchema) = NullCounter - def stat(schema: HistogramSchema) = NullStat - def addGauge(schema: GaugeSchema)(f: => Float) = NullGauge + def counter(metricBuilder: MetricBuilder) = NullCounter + def stat(metricBuilder: MetricBuilder) = NullStat + def addGauge(metricBuilder: MetricBuilder)(f: => Float) = NullGauge override def scope(namespace: String): StatsReceiver = this diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala index dcaf9423c4..b32b4ed849 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala @@ -10,26 +10,19 @@ class RelativeNameMarkingStatsReceiver( protected val self: StatsReceiver) extends StatsReceiverProxy { - override def counter(counterSchema: CounterSchema): Counter = { - val schema = CounterSchema( - counterSchema.metricBuilder - .withRelativeName(counterSchema.metricBuilder.name: _*)) + override def counter(metricBuilder: MetricBuilder): Counter = { + val schema = metricBuilder.withRelativeName(metricBuilder.name: _*) self.counter(schema) } - override def stat(histogramSchema: HistogramSchema): Stat = { - val schema = - HistogramSchema( - histogramSchema.metricBuilder - .withRelativeName(histogramSchema.metricBuilder.name: _*)) + override def stat(metricBuilder: MetricBuilder): Stat = { + val schema = metricBuilder.withRelativeName(metricBuilder.name: _*) self.stat(schema) } - override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { - val schema = - GaugeSchema( - gaugeSchema.metricBuilder - .withRelativeName(gaugeSchema.metricBuilder.name: _*)) + override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { + val schema = metricBuilder + .withRelativeName(metricBuilder.name: _*) self.addGauge(schema)(f) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala index 6846e440dd..612a47e6de 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala @@ -15,16 +15,16 @@ case class RoleConfiguredStatsReceiver( name: Option[String] = None) extends StatsReceiverProxy { - override def counter(counterSchema: CounterSchema): Counter = { - self.counter(CounterSchema(counterSchema.metricBuilder.withRole(role))) + override def counter(metricBuilder: MetricBuilder): Counter = { + self.counter(metricBuilder.withRole(role)) } - override def stat(histogramSchema: HistogramSchema): Stat = { - self.stat(HistogramSchema(histogramSchema.metricBuilder.withRole(role))) + override def stat(metricBuilder: MetricBuilder): Stat = { + self.stat(metricBuilder.withRole(role)) } - override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { - self.addGauge(GaugeSchema(gaugeSchema.metricBuilder.withRole(role)))(f) + override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { + self.addGauge(metricBuilder.withRole(role))(f) } override def registerExpression(expressionSchema: ExpressionSchema): Unit = { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala index 3e30dea072..88c49b11fb 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala @@ -22,27 +22,24 @@ class RollupStatsReceiver(protected val self: StatsReceiver) extends StatsReceiv } } - override def counter(schema: CounterSchema) = new Counter { + override def counter(metricBuilder: MetricBuilder) = new Counter { private[this] val allCounters = BroadcastCounter( - tails(schema.metricBuilder.name).map(n => - self.counter(CounterSchema(schema.metricBuilder.withName(n: _*)))) + tails(metricBuilder.name).map(n => self.counter(metricBuilder.withName(n: _*))) ) def incr(delta: Long): Unit = allCounters.incr(delta) def metadata: Metadata = allCounters.metadata } - override def stat(schema: HistogramSchema) = new Stat { + override def stat(metricBuilder: MetricBuilder) = new Stat { private[this] val allStats = BroadcastStat( - tails(schema.metricBuilder.name).map(n => - self.stat(HistogramSchema(schema.metricBuilder.withName(n: _*)))) + tails(metricBuilder.name).map(n => self.stat(metricBuilder.withName(n: _*))) ) def add(value: Float): Unit = allStats.add(value) def metadata: Metadata = allStats.metadata } - override def addGauge(schema: GaugeSchema)(f: => Float) = new Gauge { + override def addGauge(metricBuilder: MetricBuilder)(f: => Float) = new Gauge { private[this] val underlying = - tails(schema.metricBuilder.name).map(n => - self.addGauge(GaugeSchema(schema.metricBuilder.withName(n: _*)))(f)) + tails(metricBuilder.name).map(n => self.addGauge(metricBuilder.withName(n: _*))(f)) def remove(): Unit = underlying.foreach(_.remove()) def metadata: Metadata = MultiMetadata(underlying.map(_.metadata)) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala index 7c26a4e325..615e426c13 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala @@ -11,7 +11,7 @@ private[twitter] trait SchemaRegistry { /** Whether or not the counters are latched. */ def hasLatchedCounters: Boolean - def schemas(): Map[String, MetricSchema] + def schemas(): Map[String, MetricBuilder] def expressions(): Map[ExpressionSchemaKey, ExpressionSchema] diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 6bc1b0c771..42e28875a2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -1,5 +1,6 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType, MetricType} import com.twitter.finagle.stats.exp.ExpressionSchema import java.lang.{Float => JFloat} import java.util.concurrent.Callable @@ -108,7 +109,8 @@ trait StatsReceiver { /** * Get a [[MetricBuilder metricBuilder]] for this StatsReceiver. */ - def metricBuilder(): MetricBuilder = MetricBuilder(statsReceiver = this) + def metricBuilder(metricType: MetricType): MetricBuilder = + MetricBuilder(metricType = metricType, statsReceiver = this) /** * Get a [[Counter counter]] with the given `name`. @@ -121,12 +123,12 @@ trait StatsReceiver { */ @varargs def counter(verbosity: Verbosity, name: String*): Counter = - counter(CounterSchema(this.metricBuilder().withVerbosity(verbosity).withName(name: _*))) + counter(this.metricBuilder(CounterType).withVerbosity(verbosity).withName(name: _*)) /** * Get a [[Counter counter]] with the given schema. */ - def counter(schema: CounterSchema): Counter + def counter(schema: MetricBuilder): Counter /** * Get a [[Stat stat]] with the given name. @@ -139,12 +141,12 @@ trait StatsReceiver { */ @varargs def stat(verbosity: Verbosity, name: String*): Stat = - stat(HistogramSchema(this.metricBuilder().withVerbosity(verbosity).withName(name: _*))) + stat(this.metricBuilder(HistogramType).withVerbosity(verbosity).withName(name: _*)) /** * Get a [[Stat stat]] with the given schema. */ - def stat(schema: HistogramSchema): Stat + def stat(schema: MetricBuilder): Stat /** * Register a function `f` as a [[Gauge gauge]] with the given name that has @@ -214,7 +216,7 @@ trait StatsReceiver { * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = - addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name: _*)))(f) + addGauge(this.metricBuilder(GaugeType).withVerbosity(verbosity).withName(name: _*))(f) /** * Just like $AddGaugeScaladocLink but optimized for better Java experience. @@ -227,7 +229,7 @@ trait StatsReceiver { */ @varargs def addGauge(f: Supplier[Float], verbosity: Verbosity, name: String*): Gauge = - addGauge(GaugeSchema(this.metricBuilder().withVerbosity(verbosity).withName(name: _*)))(f.get()) + addGauge(this.metricBuilder(GaugeType).withVerbosity(verbosity).withName(name: _*))(f.get()) /** * Add the function `f` as a [[Gauge gauge]] with the given name. @@ -245,7 +247,7 @@ trait StatsReceiver { * good location to store the returned [[Gauge gauge]] that can give the desired lifecycle. * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ - def addGauge(schema: GaugeSchema)(f: => Float): Gauge + def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge /** * Prepend `namespace` to the names of the returned [[StatsReceiver]]. @@ -340,22 +342,28 @@ trait StatsReceiver { // We'll remove them once zk is upgraded: CSL-4710. private[stats] def counter0(name: String): Counter = counter(name) private[stats] def stat0(name: String): Stat = stat(name) + + protected def validateMetricType(metricBuilder: MetricBuilder, expectedType: MetricType): Unit = { + require( + metricBuilder.metricType == expectedType, + s"creating a $expectedType using wrong MetricBuilder: ${metricBuilder.metricType}") + } } abstract class AbstractStatsReceiver extends StatsReceiver { - final def counter(schema: CounterSchema): Counter = - counterImpl(schema.metricBuilder.verbosity, schema.metricBuilder.name) + final def counter(metricBuilder: MetricBuilder): Counter = + counterImpl(metricBuilder.verbosity, metricBuilder.name) protected def counterImpl(verbosity: Verbosity, name: scala.collection.Seq[String]): Counter - final def stat(schema: HistogramSchema): Stat = - statImpl(schema.metricBuilder.verbosity, schema.metricBuilder.name) + final def stat(metricBuilder: MetricBuilder): Stat = + statImpl(metricBuilder.verbosity, metricBuilder.name) protected def statImpl(verbosity: Verbosity, name: scala.collection.Seq[String]): Stat - final def addGauge(schema: GaugeSchema)(f: => Float): Gauge = - addGaugeImpl(schema.metricBuilder.verbosity, schema.metricBuilder.name, f) + final def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = + addGaugeImpl(metricBuilder.verbosity, metricBuilder.name, f) protected def addGaugeImpl( verbosity: Verbosity, diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala index a4311b72fd..c058020512 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala @@ -11,9 +11,9 @@ trait StatsReceiverProxy extends StatsReceiver with DelegatingStatsReceiver { def repr: AnyRef = self.repr - def counter(schema: CounterSchema): Counter = self.counter(schema) - def stat(schema: HistogramSchema): Stat = self.stat(schema) - def addGauge(schema: GaugeSchema)(f: => Float): Gauge = self.addGauge(schema)(f) + def counter(metricBuilder: MetricBuilder): Counter = self.counter(metricBuilder) + def stat(metricBuilder: MetricBuilder): Stat = self.stat(metricBuilder) + def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = self.addGauge(metricBuilder)(f) override protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = self.registerExpression(expressionSchema) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala index 3ddb35795b..81a074a9b9 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/VerbosityAdjustingStatsReceiver.scala @@ -9,16 +9,16 @@ class VerbosityAdjustingStatsReceiver( defaultVerbosity: Verbosity) extends StatsReceiverProxy { - override def stat(histogramSchema: HistogramSchema): Stat = { - self.stat(HistogramSchema(histogramSchema.metricBuilder.withVerbosity(defaultVerbosity))) + override def stat(metricBuilder: MetricBuilder): Stat = { + self.stat(metricBuilder.withVerbosity(defaultVerbosity)) } - override def counter(counterSchema: CounterSchema): Counter = { - self.counter(CounterSchema(counterSchema.metricBuilder.withVerbosity(defaultVerbosity))) + override def counter(metricBuilder: MetricBuilder): Counter = { + self.counter(metricBuilder.withVerbosity(defaultVerbosity)) } - override def addGauge(gaugeSchema: GaugeSchema)(f: => Float): Gauge = { - self.addGauge(GaugeSchema(gaugeSchema.metricBuilder.withVerbosity(defaultVerbosity)))(f) + override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { + self.addGauge(metricBuilder.withVerbosity(defaultVerbosity))(f) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index bb66fa6e09..0989cf7f59 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -1,14 +1,8 @@ package com.twitter.finagle.stats.exp +import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.exp.Expression.HistogramComponent -import com.twitter.finagle.stats.{ - CounterSchema, - GaugeSchema, - HistogramSchema, - MetricBuilder, - MetricSchema, - StatsReceiver -} +import com.twitter.finagle.stats.{MetricBuilder, StatsReceiver} import java.util.concurrent.ConcurrentHashMap import scala.annotation.varargs @@ -27,8 +21,8 @@ private[twitter] object Expression { case (acc, expr) => getStatsReceivers(expr) ++ acc } - case MetricExpression(schema) => Set(schema.metricBuilder.statsReceiver) - case HistogramExpression(schema, _) => Set(schema.metricBuilder.statsReceiver) + case MetricExpression(metricBuilder) => Set(metricBuilder.statsReceiver) + case HistogramExpression(metricBuilder, _) => Set(metricBuilder.statsReceiver) case ConstantExpression(_) => Set.empty } @@ -42,15 +36,22 @@ private[twitter] object Expression { * @param component the histogram component either a [[HistogramComponent]] for Left * or a percentile in Double for Right. */ - def apply(schema: HistogramSchema, component: Either[HistogramComponent, Double]) = - HistogramExpression(schema, component) + def apply( + metricBuilder: MetricBuilder, + component: Either[HistogramComponent, Double] + ): HistogramExpression = { + require( + metricBuilder.metricType == HistogramType, + "this method is for creating histogram expression") + HistogramExpression(metricBuilder, component) + } /** * Create an single expression wrapping a counter or gauge. */ - def apply(schema: MetricSchema): Expression = { - require(!schema.isInstanceOf[HistogramSchema], "provide a component for histogram") - MetricExpression(schema) + def apply(metricBuilder: MetricBuilder): Expression = { + require(metricBuilder.metricType != HistogramType, "provide a component for histogram") + MetricExpression(metricBuilder) } // utility methods shared by Metrics.scala and InMemoryStatsReceiver @@ -61,26 +62,19 @@ private[twitter] object Expression { expression match { case f @ FunctionExpression(_, exprs) => f.copy(exprs = exprs.map(replaceExpression(_, metricBuilders))) - case m @ MetricExpression(schema) if schema.metricBuilder.kernel.isDefined => - val builder = metricBuilders.get(schema.metricBuilder.kernel.get) - if (builder != null) m.copy(schema = reformSchema(schema, builder)) + case m @ MetricExpression(metricBuilder) if metricBuilder.kernel.isDefined => + val builder = metricBuilders.get(metricBuilder.kernel.get) + if (builder != null) m.copy(builder) else m - case h @ HistogramExpression(schema, _) if schema.metricBuilder.kernel.isDefined => - val builder = metricBuilders.get(schema.metricBuilder.kernel.get) - if (builder != null) - h.copy(schema = reformSchema(schema, builder).asInstanceOf[HistogramSchema]) + case h @ HistogramExpression(metricBuilder, _) if metricBuilder.kernel.isDefined => + val builder = metricBuilders.get(metricBuilder.kernel.get) + if (builder != null) h.copy(builder) else h case otherExpression => otherExpression } } - private[this] def reformSchema(schema: MetricSchema, builder: MetricBuilder): MetricSchema = { - schema match { - case CounterSchema(_) => CounterSchema(builder) - case HistogramSchema(_) => HistogramSchema(builder) - case GaugeSchema(_) => GaugeSchema(builder) - } - } + private[this] def reformSchema(builder: MetricBuilder): MetricBuilder = builder } /** @@ -116,13 +110,13 @@ case class FunctionExpression private (fnName: String, exprs: Seq[Expression]) e /** * Represents the leaf metrics */ -case class MetricExpression private (schema: MetricSchema) extends Expression +case class MetricExpression private (metricBuilder: MetricBuilder) extends Expression /** * Represent a histogram expression with specified component, for example the average, or a percentile * @param component either a [[HistogramComponent]] or a percentile in Double */ case class HistogramExpression private ( - schema: HistogramSchema, + metricBuilder: MetricBuilder, component: Either[HistogramComponent, Double]) extends Expression diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java b/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java index 6261e249bc..7beb7e8db4 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java @@ -2,8 +2,6 @@ import org.junit.Test; -import java.util.Arrays; - import scala.Some; /** @@ -13,13 +11,13 @@ public final class MetricBuilderCompilationTest { @Test public void testMetricBuilderConstruction() { StatsReceiver sr = new InMemoryStatsReceiver(); - MetricBuilder mb = sr.metricBuilder(); + MetricBuilder mb = sr.metricBuilder(MetricBuilder.CounterType$.MODULE$); } @Test public void testWithMethods() { StatsReceiver sr = new InMemoryStatsReceiver(); - MetricBuilder mb = sr.metricBuilder() + MetricBuilder mb = sr.metricBuilder(MetricBuilder.CounterType$.MODULE$) .withKeyIndicator(true) .withDescription("my cool metric") .withVerbosity(Verbosity.Debug()) @@ -35,7 +33,7 @@ public void testWithMethods() { @Test public void testConstructingMetrics() { StatsReceiver sr = new InMemoryStatsReceiver(); - MetricBuilder mb = sr.metricBuilder(); + MetricBuilder mb = sr.metricBuilder(MetricBuilder.CounterType$.MODULE$); Gauge g = mb.gauge(() -> 3.0f, "my", "cool", "gauge"); Stat s = mb.histogram("my", "cool", "histogram"); Counter c = mb.counter("my", "cool", "counter"); diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 01a6894cdc..9bfcea9a3d 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -1,5 +1,6 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema} import java.io.{ByteArrayOutputStream, PrintStream} import java.nio.charset.StandardCharsets @@ -124,7 +125,8 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra val stats = new InMemoryStatsReceiver() ExpressionSchema( "a", - Expression(CounterSchema(MetricBuilder(name = Seq("counter"), statsReceiver = stats)))) + Expression( + MetricBuilder(name = Seq("counter"), metricType = CounterType, statsReceiver = stats))) stats.expressions.contains("a") } @@ -326,11 +328,11 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra test("creating metrics stores their metadata and clear()ing should remove that metadata") { val stats = new InMemoryStatsReceiver() stats.addGauge("coolGauge") { 3 } - assert(stats.schemas(Seq("coolGauge")).isInstanceOf[MetricSchema]) + assert(stats.schemas(Seq("coolGauge")).isInstanceOf[MetricBuilder]) stats.counter("sweetCounter") - assert(stats.schemas(Seq("sweetCounter")).isInstanceOf[MetricSchema]) + assert(stats.schemas(Seq("sweetCounter")).isInstanceOf[MetricBuilder]) stats.stat("radHisto") - assert(stats.schemas(Seq("radHisto")).isInstanceOf[MetricSchema]) + assert(stats.schemas(Seq("radHisto")).isInstanceOf[MetricBuilder]) stats.clear() assert(stats.schemas.isEmpty) @@ -353,13 +355,13 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(parts.length == 3) assert( parts( - 0) == "coolGauge GaugeSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, coolGauge, List(), None, Vector(), InMemoryStatsReceiver))") + 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, coolGauge, List(), None, Vector(), GaugeType, InMemoryStatsReceiver)") assert( parts( - 1) == "rad/histo HistogramSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, rad/histo, List(), None, Vector(), InMemoryStatsReceiver/rad))") + 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, rad/histo, List(), None, Vector(), HistogramType, InMemoryStatsReceiver/rad)") assert( parts( - 2) == "sweet/counter CounterSchema(MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, sweet/counter, List(), None, Vector(), InMemoryStatsReceiver))") + 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, sweet/counter, List(), None, Vector(), CounterType, InMemoryStatsReceiver)") } finally { ps.close() } @@ -368,9 +370,12 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra test("expression are reloaded with fully scoped names") { val sr = new InMemoryStatsReceiver - val aSchema = CounterSchema(MetricBuilder(name = Seq("a"), statsReceiver = sr).withKernel) - val bSchema = HistogramSchema(MetricBuilder(name = Seq("b"), statsReceiver = sr).withKernel) - val cSchema = GaugeSchema(MetricBuilder(name = Seq("c"), statsReceiver = sr).withKernel) + val aSchema = + MetricBuilder(name = Seq("a"), metricType = CounterType, statsReceiver = sr).withKernel + val bSchema = + MetricBuilder(name = Seq("b"), metricType = HistogramType, statsReceiver = sr).withKernel + val cSchema = + MetricBuilder(name = Seq("c"), metricType = GaugeType, statsReceiver = sr).withKernel val expression = ExpressionSchema( "test_expression", @@ -382,9 +387,12 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra val cGauge = sr.scope(("test")).addGauge(cSchema) { 1 } // what we expected as hydrated metric builders - val aaSchema = CounterSchema(MetricBuilder(name = Seq("test", "a"), statsReceiver = sr)) - val bbSchema = HistogramSchema(MetricBuilder(name = Seq("test", "b"), statsReceiver = sr)) - val ccSchema = GaugeSchema(MetricBuilder(name = Seq("test", "c"), statsReceiver = sr)) + val aaSchema = + MetricBuilder(name = Seq("test", "a"), metricType = CounterType, statsReceiver = sr) + val bbSchema = + MetricBuilder(name = Seq("test", "b"), metricType = HistogramType, statsReceiver = sr) + val ccSchema = + MetricBuilder(name = Seq("test", "c"), metricType = GaugeType, statsReceiver = sr) val expected_expression = ExpressionSchema( "test_expression", diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala index 4474095f4c..6a5e4bfd52 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala @@ -1,11 +1,15 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.MetricBuilder.CounterType import org.scalatest.funsuite.AnyFunSuite class MetricBuilderTest extends AnyFunSuite { test("metadataScopeSeparator affects stringification") { - val mb = MetricBuilder(name = Seq("foo", "bar"), statsReceiver = new InMemoryStatsReceiver) + val mb = MetricBuilder( + name = Seq("foo", "bar"), + metricType = CounterType, + statsReceiver = new InMemoryStatsReceiver) assert(mb.toString.contains("foo/bar")) metadataScopeSeparator.setSeparator("-") assert(mb.toString.contains("foo-bar")) @@ -15,13 +19,18 @@ class MetricBuilderTest extends AnyFunSuite { test("kernels are object reference hashCode") { val sr = new InMemoryStatsReceiver - val mb1 = MetricBuilder(name = Seq("a"), statsReceiver = sr).withKernel - val mb2 = MetricBuilder(name = Seq("a"), statsReceiver = sr).withKernel + val mb1 = + MetricBuilder(name = Seq("a"), metricType = CounterType, statsReceiver = sr).withKernel + val mb2 = + MetricBuilder(name = Seq("a"), metricType = CounterType, statsReceiver = sr).withKernel assert(mb2.kernel != mb1.kernel) } test("kernels remain the same after copy") { - val mb1 = MetricBuilder(name = Seq("a"), statsReceiver = new InMemoryStatsReceiver).withKernel + val mb1 = MetricBuilder( + name = Seq("a"), + metricType = CounterType, + statsReceiver = new InMemoryStatsReceiver).withKernel val mb2 = mb1.withDescription("I am a description").withName("a", "b") assert(mb1.kernel == mb2.kernel) } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala index 7750dcc135..1ff7e4fb8e 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala @@ -9,26 +9,26 @@ class RoleConfiguredStatsReceiverTest extends AnyFunSuite { val serverSR = new RoleConfiguredStatsReceiver(mem, Server) clientSR.counter("foo") - assert(mem.schemas.get(Seq("foo")).get.metricBuilder.role == Client) + assert(mem.schemas.get(Seq("foo")).get.role == Client) clientSR.addGauge("bar")(3) - assert(mem.schemas.get(Seq("bar")).get.metricBuilder.role == Client) + assert(mem.schemas.get(Seq("bar")).get.role == Client) clientSR.stat("baz") - assert(mem.schemas.get(Seq("baz")).get.metricBuilder.role == Client) + assert(mem.schemas.get(Seq("baz")).get.role == Client) serverSR.counter("fee") - assert(mem.schemas.get(Seq("fee")).get.metricBuilder.role == Server) + assert(mem.schemas.get(Seq("fee")).get.role == Server) serverSR.addGauge("fi")(3) - assert(mem.schemas.get(Seq("fi")).get.metricBuilder.role == Server) + assert(mem.schemas.get(Seq("fi")).get.role == Server) serverSR.stat("foe") - assert(mem.schemas.get(Seq("foe")).get.metricBuilder.role == Server) + assert(mem.schemas.get(Seq("foe")).get.role == Server) } test("RoleConfiguredSR equality") { val mem = new InMemoryStatsReceiver - val clientA = new RoleConfiguredStatsReceiver(mem, Client) - val clientB = new RoleConfiguredStatsReceiver(mem, Client) - val serverA = new RoleConfiguredStatsReceiver(mem, Server) - val serverB = new RoleConfiguredStatsReceiver(mem, Server) + val clientA = RoleConfiguredStatsReceiver(mem, Client) + val clientB = RoleConfiguredStatsReceiver(mem, Client) + val serverA = RoleConfiguredStatsReceiver(mem, Server) + val serverB = RoleConfiguredStatsReceiver(mem, Server) assert(clientA == clientB) assert(clientA != serverA) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index d67534bc22..2db57e6599 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -1,6 +1,7 @@ package com.twitter.finagle.stats.exp import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.MetricBuilder.{CounterType, HistogramType} import com.twitter.finagle.stats._ import com.twitter.util.{Stopwatch, Time, TimeControl} import scala.util.control.NonFatal @@ -12,11 +13,20 @@ class ExpressionSchemaTest extends AnyFunSuite { val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) val successMb = - CounterSchema(MetricBuilder(name = Seq("success"), statsReceiver = clientSR).withKernel) + MetricBuilder( + name = Seq("success"), + metricType = CounterType, + statsReceiver = clientSR).withKernel val failuresMb = - CounterSchema(MetricBuilder(name = Seq("failures"), statsReceiver = clientSR).withKernel) + MetricBuilder( + name = Seq("failures"), + metricType = CounterType, + statsReceiver = clientSR).withKernel val latencyMb = - HistogramSchema(MetricBuilder(name = Seq("latency"), statsReceiver = clientSR).withKernel) + MetricBuilder( + name = Seq("latency"), + metricType = HistogramType, + statsReceiver = clientSR).withKernel val sum = Expression(successMb).plus(Expression(failuresMb)) def slowQuery(ctl: TimeControl, succeed: Boolean): Unit = { From 659e818caf3ce173466c68d50e4382e1b2ef9892 Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Fri, 4 Jun 2021 19:35:30 +0000 Subject: [PATCH 491/796] *: Upgrade sbt to 1.5.3 Problem sbt supports Scala 3 natively in the recent versions (e.g. 1.5.x) and I want the build to be ready for Scala 3. Solution Updating sbt from 1.3.10 to 1.5.3 Result sbt is updated to 1.5.3 and most warnings addressed (i.e. auto-reloading, unused settings, and migration to slash-syntax for settings) Signed-off-by: Moses Nakamura Differential Revision: https://phabricator.twitter.biz/D680505 --- build.sbt | 35 +++++++++++++++++++---------------- project/build.properties | 2 +- project/plugins.sbt | 1 - sbt | 18 ++++++++++-------- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/build.sbt b/build.sbt index 5df28d3cee..38d7689655 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,8 @@ import scoverage.ScoverageKeys +Global / onChangedBuildSource := ReloadOnSourceChanges +Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure + // All Twitter library releases are date versioned as YY.MM.patch val releaseVersion = "21.6.0-SNAPSHOT" @@ -86,7 +89,7 @@ val baseSettings = Seq( version := releaseVersion, organization := "com.twitter", // Workaround for a scaladoc bug which causes it to choke on empty classpaths. - unmanagedClasspath in Compile += Attributed.blank(new java.io.File("doesnotexist")), + Compile / unmanagedClasspath += Attributed.blank(new java.io.File("doesnotexist")), libraryDependencies ++= Seq( "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.2", // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit @@ -94,11 +97,11 @@ val baseSettings = Seq( "org.scalatest" %% "scalatest" % "3.1.2" % "test", "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test" ), - fork in Test := true, // We have to fork to get the JavaOptions + Test / fork := true, // We have to fork to get the JavaOptions // Workaround for cross building HealthyQueue.scala, which is not compatible between // 2.12- with 2.13+. - unmanagedSourceDirectories in Compile += { - val sourceDir = (sourceDirectory in Compile).value + Compile / unmanagedSourceDirectories += { + val sourceDir = (Compile / sourceDirectory).value CrossVersion.partialVersion(scalaVersion.value) match { case Some((2, n)) if n >= 13 => sourceDir / "scala-2.13+" case _ => sourceDir / "scala-2.12-" @@ -121,20 +124,20 @@ val baseSettings = Seq( // Note: Use -Xlint rather than -Xlint:unchecked when TestThriftStructure // warnings are resolved javacOptions ++= Seq("-Xlint:unchecked", "-source", "1.8", "-target", "1.8"), - javacOptions in doc := Seq("-source", "1.8"), + doc / javacOptions := Seq("-source", "1.8"), javaOptions ++= Seq( "-Djava.net.preferIPv4Stack=true", "-XX:+AggressiveOpts", "-server" ), javaOptions ++= gcJavaOptions, - javaOptions in Test ++= travisTestJavaOptions, + Test / javaOptions ++= travisTestJavaOptions, // -a: print stack traces for failing asserts testOptions += Tests.Argument(TestFrameworks.JUnit, "-a"), // This is bad news for things like com.twitter.util.Time - parallelExecution in Test := false, + Test / parallelExecution := false, // Sonatype publishing - publishArtifact in Test := false, + Test / publishArtifact := false, pomIncludeRepository := { _ => false }, publishMavenStyle := true, publishConfiguration := publishConfiguration.value.withOverwrite(true), @@ -172,7 +175,7 @@ val baseSettings = Seq( val sharedSettings = defaultProjectSettings ++ baseSettings lazy val noPublishSettings = Seq( - skip in publish := true + publish / skip := true ) lazy val util = Project( @@ -184,7 +187,7 @@ lazy val util = Project( sharedSettings ++ noPublishSettings ++ Seq( - unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(utilBenchmark) + ScalaUnidoc / unidoc / unidocProjectFilter := inAnyProject -- inProjects(utilBenchmark) ) ).aggregate( utilApp, @@ -303,7 +306,7 @@ lazy val utilCore = Project( name := "util-core", // Moved some code to 'concurrent-extra' to conform to Pants' 1:1:1 principle (https://www.pantsbuild.org/build_files.html#target-granularity) // so that util-core would work better for Pants projects in IntelliJ. - unmanagedSourceDirectories in Compile += baseDirectory.value / "concurrent-extra", + Compile / unmanagedSourceDirectories += baseDirectory.value / "concurrent-extra", libraryDependencies ++= Seq( caffeineLib % "test", scalacheckLib, @@ -313,7 +316,7 @@ lazy val utilCore = Project( "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" ), - resourceGenerators in Compile += Def.task { + Compile / resourceGenerators += Def.task { val projectName = name.value val file = resourceManaged.value / "com" / "twitter" / projectName / "build.properties" val buildRev = scala.sys.process.Process("git" :: "rev-parse" :: "HEAD" :: Nil).!!.trim @@ -323,10 +326,10 @@ lazy val utilCore = Project( IO.write(file, contents) Seq(file) }.taskValue, - sources in (Compile, doc) := { + Compile / doc / sources := { // Monitors.java causes "not found: type Monitor$" (CSL-5034) // so exclude it from the sources used for scaladoc - val previous = (sources in (Compile, doc)).value + val previous = (Compile / doc / sources).value previous.filterNot(file => file.getName() == "Monitors.java") } ).dependsOn(utilFunction) @@ -339,8 +342,8 @@ lazy val utilDoc = Project( ).settings( sharedSettings ++ noPublishSettings ++ Seq( - scalacOptions in doc ++= Seq("-doc-title", "Util", "-doc-version", version.value), - includeFilter in Sphinx := ("*.html" | "*.png" | "*.svg" | "*.js" | "*.css" | "*.gif" | "*.txt") + doc / scalacOptions ++= Seq("-doc-title", "Util", "-doc-version", version.value), + Sphinx / includeFilter := ("*.html" | "*.png" | "*.svg" | "*.js" | "*.css" | "*.gif" | "*.txt") )) lazy val utilFunction = Project( diff --git a/project/build.properties b/project/build.properties index 797e7ccfdb..67d27a1dfe 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.3.10 +sbt.version=1.5.3 diff --git a/project/plugins.sbt b/project/plugins.sbt index b7c24e916b..e2cefb38da 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,6 +2,5 @@ resolvers += Classpaths.sbtPluginReleases resolvers += Resolver.sonatypeRepo("snapshots") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.0") -addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC13") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.0") diff --git a/sbt b/sbt index 4cb7d05ede..2441ede7e4 100755 --- a/sbt +++ b/sbt @@ -1,21 +1,23 @@ #!/bin/bash -sbtver=1.3.10 -sbtjar=sbt-launch.jar -sbtsha128=f49861befe36746728182ea7efb30096c724b0d7 +set -eo pipefail + +sbtver="1.5.3" +sbtjar="sbt-launch-$sbtver.jar" +sbtsha128="1ca2d0ee419a1f82f512f2aa8556d6e262b37961" sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" -if [ ! -f $sbtjar ]; then +if [ ! -f "sbt-launch.jar" ]; then echo "downloading $PWD/$sbtjar" 1>&2 - if ! curl --location --silent --fail --remote-name $sbtrepo/$sbtver/$sbtjar; then + if ! curl --location --silent --fail -o "sbt-launch.jar" "$sbtrepo/$sbtver/$sbtjar"; then exit 1 fi fi -checksum=`openssl dgst -sha1 $sbtjar | awk '{ print $2 }'` +checksum=`openssl dgst -sha1 sbt-launch.jar | awk '{ print $2 }'` if [ "$checksum" != $sbtsha128 ]; then - echo "bad $PWD/$sbtjar. delete $PWD/$sbtjar and run $0 again." + echo "bad $PWD/sbt-launch.jar. delete $PWD/sbt-launch.jar and run $0 again." exit 1 fi @@ -24,4 +26,4 @@ fi java -ea \ $SBT_OPTS \ $JAVA_OPTS \ - -jar $sbtjar "$@" + -jar "sbt-launch.jar" "$@" From d9c900a1c8e6862dda114f553a13c712601dc079 Mon Sep 17 00:00:00 2001 From: Flavio Brasil Date: Fri, 4 Jun 2021 22:02:54 +0000 Subject: [PATCH 492/796] util-core finagle-core: introduces forking scheduler Problem The adaptive executor project is being developed in a separate module. One of the goals of the project is deprecating the need to use separate `FuturePool`s for the execution of CPU intensive and blocking tasks, allowing users to rely on the scheduler to handle the different types of tasks in a finagle application. It's not possible to provide a seamless migration path to this new behavior without changing the `FuturePool` implementation. Solution This review introduces a `ForkingScheduler` interface to be extended by the fiber scheduler (implementation of the adaptive executor project) and introduces a mechanism for the scheduler to define if the future pool calls should be redirected to the forking scheduler. Notes - I placed the new interface in util since I expect that util code might rely on it in the future - There's no behavior change in case the application doesn't use a forking scheduler - I fixed the misplaced comment about the tracing filter in the server stack JIRA Issues: VM-4488 Differential Revision: https://phabricator.twitter.biz/D644985 --- .../twitter/concurrent/ForkingScheduler.scala | 81 +++++++++++++ .../scala/com/twitter/util/FuturePool.scala | 111 +++++++++++------- .../com/twitter/util/FuturePoolTest.scala | 73 ++++++++++++ 3 files changed, 221 insertions(+), 44 deletions(-) create mode 100644 util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala new file mode 100644 index 0000000000..10027f24b4 --- /dev/null +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala @@ -0,0 +1,81 @@ +package com.twitter.concurrent + +import com.twitter.util.Future +import java.util.concurrent.{Executor, ExecutorService} + +/** + * A scheduler that provides methods to fork the execution of + * `Future` computations and behaves similarly to a thread pool + */ +trait ForkingScheduler extends Scheduler { + + /** + * Forks the execution of a `Future` computation. + * + * @param f the Future computation to be forked + * @tparam T the type of the `Future` computation + * @return a `Future` to be satisfied when the forked `Future` + * is completed + */ + def fork[T](f: => Future[T]): Future[T] + + /** + * Forks the execution of a `Future` computation using the + * provided executor. + * + * @param executor the executor to be used to run the `Future` + * computation + * @param f the `Future` computation to be forked + * @tparam T the type of the `Future` computation + * @return a `Future` to be satisfied when the forked `Future` + * is completed + */ + def fork[T](executor: Executor)(f: => Future[T]): Future[T] + + /** + * Creates a wrapper of this scheduler with a fixed max + * concurrency. The original forking scheduler remains + * unchanged and is used as the underlying implementation + * of the returned scheduler. + * + * @param v the max concurrency + * @return the wrapper with a fixed max concurrency + */ + def withMaxConcurrency(v: Int): ForkingScheduler + + /** + * Creates an `ExecutorService` wrapper for this forking + * scheduler. The original forking scheduler remains + * unchanged and is used as the underlying implementation + * of the returned executor. If the executor service is + * shut down, only the wrapper is shut down and the original + * forking scheduler continues operating. + * + * @return the executor service wrapper + */ + def asExecutorService(): ExecutorService + + /** + * Indicates if `FuturePool`s should have their tasks + * redirected to the forking scheduler. + * + * @return if enabled + */ + def redirectFuturePools(): Boolean +} + +object ForkingScheduler { + + /** + * Returns empty if the current scheduler isn't a forking + * scheduler. Returns the forking scheduler otherwise. + * @return an optional of the forking scheduler + */ + def apply(): Option[ForkingScheduler] = + Scheduler() match { + case f: ForkingScheduler => + Some(f) + case _ => + None + } +} diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index 52ed0d464b..72901a10f7 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -1,6 +1,6 @@ package com.twitter.util -import com.twitter.concurrent.NamedPoolThreadFactory +import com.twitter.concurrent.{ForkingScheduler, NamedPoolThreadFactory} import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.{Future => _, _} import scala.runtime.NonLocalReturnControl @@ -129,59 +129,82 @@ class ExecutorServiceFuturePool protected[this] ( extends FuturePool { def this(executor: ExecutorService) = this(executor, false) - def apply[T](f: => T): Future[T] = { - val runOk = new AtomicBoolean(true) - val p = new Promise[T] - val task = new Runnable { - private[this] val saved = Local.save() - - def run(): Unit = { - // Make an effort to skip work in the case the promise - // has been cancelled or already defined. - if (!runOk.compareAndSet(true, false)) - return + /** + * Not null if the current scheduler supports forking and indicates that + * `FuturePool`s should be redirected to it. If the provided executor is a + * `ThreadPoolExecutor`, the max concurrency of the forking scheduler is + * set to the size of the thread pool, maintaining a similar concurrency behavior + */ + private[this] lazy val forkingScheduler: ForkingScheduler = { + ForkingScheduler() + .filter(_.redirectFuturePools()) + .map { s => + executor match { + case e: ThreadPoolExecutor => + s.withMaxConcurrency(e.getMaximumPoolSize) + case _ => + s + } + }.getOrElse(null) + } - val current = Local.save() - Local.restore(saved) + def apply[T](f: => T): Future[T] = { + if (forkingScheduler != null) { + forkingScheduler.fork(Future.value(f)) + } else { + val runOk = new AtomicBoolean(true) + val p = new Promise[T] + val task = new Runnable { + private[this] val saved = Local.save() + + def run(): Unit = { + // Make an effort to skip work in the case the promise + // has been cancelled or already defined. + if (!runOk.compareAndSet(true, false)) + return + + val current = Local.save() + Local.restore(saved) + + try p.updateIfEmpty(Try(f)) + catch { + case nlrc: NonLocalReturnControl[_] => + val fnlrc = new FutureNonLocalReturnControl(nlrc) + p.updateIfEmpty(Throw(fnlrc)) + throw fnlrc + case e: Throwable => + p.updateIfEmpty(Throw(new ExecutionException(e))) + throw e + } finally Local.restore(current) + } + } - try p.updateIfEmpty(Try(f)) + // This is safe: the only thing that can call task.run() is + // executor, the only thing that can raise an interrupt is the + // receiver of this value, which will then be fully initialized. + val javaFuture = + try executor.submit(task) catch { - case nlrc: NonLocalReturnControl[_] => - val fnlrc = new FutureNonLocalReturnControl(nlrc) - p.updateIfEmpty(Throw(fnlrc)) - throw fnlrc - case e: Throwable => - p.updateIfEmpty(Throw(new ExecutionException(e))) - throw e - } finally Local.restore(current) - } - } + case e: RejectedExecutionException => + runOk.set(false) + p.setException(e) + null + } - // This is safe: the only thing that can call task.run() is - // executor, the only thing that can raise an interrupt is the - // receiver of this value, which will then be fully initialized. - val javaFuture = - try executor.submit(task) - catch { - case e: RejectedExecutionException => - runOk.set(false) - p.setException(e) - null + p.setInterruptHandler { + case cause => + if (interruptible || runOk.compareAndSet(true, false)) { + if (p.updateIfEmpty(Throw(cause))) + javaFuture.cancel(true) + } } - p.setInterruptHandler { - case cause => - if (interruptible || runOk.compareAndSet(true, false)) { - if (p.updateIfEmpty(Throw(cause))) - javaFuture.cancel(true) - } + p } - - p } override def toString: String = - s"ExecutorServiceFuturePool(interruptible=$interruptible, executor=$executor)" + s"ExecutorServiceFuturePool(interruptible=$interruptible, executor=$executor, forkingScheduler=$forkingScheduler)" override def poolSize: Int = executor match { case tpe: ThreadPoolExecutor => tpe.getPoolSize diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index f8bf342179..89fa97ce47 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -1,5 +1,6 @@ package com.twitter.util +import com.twitter.concurrent.{Scheduler, ForkingScheduler} import com.twitter.conversions.DurationOps._ import java.util.concurrent.{Future => JFuture, _} import org.scalatest.concurrent.Eventually @@ -233,4 +234,76 @@ class FuturePoolTest extends AnyFunSuite with Eventually { executor.shutdown() } + class TestForkingScheduler(redirect: Boolean) extends ForkingScheduler { + var concurrency = Option.empty[Int] + var forked = Option.empty[Future[_]] + override def fork[T](f: => Future[T]) = { + val r = f + forked = Some(r) + r + } + override def redirectFuturePools(): Boolean = redirect + override def blocking[T](f: => T)(implicit perm: Awaitable.CanAwait): T = f + override def flush(): Unit = {} + override def withMaxConcurrency(v: Int): ForkingScheduler = { + concurrency = Some(v) + this + } + + override def submit(r: Runnable): Unit = r.run() + + override def fork[T](executor: Executor)(f: => Future[T]) = ??? + override def asExecutorService(): ExecutorService = ??? + override def numDispatches: Long = ??? + } + + test("FuturePool should be redirected to the forking scheduler if specified by the scheduler") { + val concurrency = 1 + val scheduler = new TestForkingScheduler(redirect = true) + val prevScheduler = Scheduler() + val executor = Executors.newFixedThreadPool(concurrency) + Scheduler.setUnsafe(scheduler) + try { + val pool = FuturePool(executor) + var executed = false + val result = + pool { + executed = true + 1 + } + assert(Await.result(result, 1.second) == 1) + assert(Await.result(scheduler.forked.get, 1.second) == 1) + assert(scheduler.concurrency == Some(concurrency)) + assert(executed) + } finally { + Scheduler.setUnsafe(prevScheduler) + executor.shutdown() + } + } + + test( + "FuturePool should not be redirected to the forking scheduler if specified by the scheduler") { + val concurrency = 1 + val scheduler = new TestForkingScheduler(redirect = false) + val executor = Executors.newFixedThreadPool(concurrency) + val prevScheduler = Scheduler() + Scheduler.setUnsafe(scheduler) + try { + val pool = FuturePool(executor) + var executed = false + val result = + pool { + executed = true + 1 + } + assert(Await.result(result, 1.second) == 1) + assert(scheduler.forked.isEmpty) + assert(scheduler.concurrency.isEmpty) + assert(executed) + } finally { + Scheduler.setUnsafe(prevScheduler) + executor.shutdown() + } + } + } From 8aaf226e40243af920ea0632d11f8e10d27ad230 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Sat, 5 Jun 2021 01:54:19 +0000 Subject: [PATCH 493/796] util: Add util-jackson Problem We'd like a general purpose JSON parsing library which provides a level of support over the Jackson and `jackson-module-scala` libraries. Solution The core of the Finatra Jackson support has proven to be useful in providing utility for JSON serialization and deserialization. Thus we are lifting the core of the Finatra Jackson library to be general purpose JSON parsing with Jackson in Scala. Result A usable JSON parsing library with an enhanced Scala API and improved case class deserializer for collecting deserialization failures. Differential Revision: https://phabricator.twitter.biz/D664962 --- CHANGELOG.rst | 14 +- build.sbt | 43 +- doc/src/sphinx/index.rst | 1 + doc/src/sphinx/util-jackson/index.rst | 1005 +++++++ util-benchmark/src/main/scala/BUILD | 7 +- .../com/twitter/json/JacksonBenchmark.scala | 75 + .../com/twitter/json/TestDemographic.java | 6 + .../scala/com/twitter/json/TestFormat.java | 7 + .../scala/com/twitter/util/WrappedValue.scala | 28 + util-jackson-annotations/PROJECT | 8 + .../com/twitter/util/jackson/annotation/BUILD | 10 + .../jackson/annotation/InjectableValue.java | 19 + util-jackson/PROJECT | 8 + .../com/fasterxml/jackson/databind/BUILD | 18 + .../DeserializationContextAccessor.scala | 7 + .../jackson/databind/ObjectMapperCopier.scala | 11 + .../main/scala/com/twitter/util/jackson/BUILD | 35 + .../scala/com/twitter/util/jackson/JSON.scala | 28 + .../com/twitter/util/jackson/JsonDiff.scala | 227 ++ .../util/jackson/ScalaObjectMapper.scala | 636 +++++ .../com/twitter/util/jackson/caseclass/BUILD | 31 + .../caseclass/CaseClassBeanProperty.scala | 8 + .../caseclass/CaseClassDeserializer.scala | 834 ++++++ .../CaseClassDeserializerResolver.scala | 22 + .../jackson/caseclass/CaseClassField.scala | 459 +++ .../caseclass/CaseClassJacksonModule.scala | 14 + .../util/jackson/caseclass/SerdeLogging.scala | 24 + .../util/jackson/caseclass/Types.scala | 144 + .../util/jackson/caseclass/exceptions/BUILD | 20 + .../CaseClassFieldMappingException.scala | 103 + .../CaseClassMappingException.scala | 77 + .../InjectableValuesException.scala | 22 + .../caseclass/exceptions/package.scala | 27 + .../util/jackson/caseclass/package.scala | 148 + .../com/twitter/util/jackson/package.scala | 8 + .../com/twitter/util/jackson/serde/BUILD | 19 + .../jackson/serde/DefaultSerdeModule.scala | 12 + .../serde/DurationStringDeserializer.scala | 43 + .../serde/DurationStringSerializer.scala | 20 + .../jackson/serde/LongKeyDeserializer.scala | 52 + .../serde/TimeStringDeserializer.scala | 99 + .../jackson/serde/TimeStringSerializer.scala | 73 + .../serde/WrappedValueSerializer.scala | 18 + .../test/java/com/twitter/util/jackson/BUILD | 8 + .../com/twitter/util/jackson/CarMake.java | 8 + .../com/twitter/util/jackson/CarMakeEnum.java | 9 + .../util/jackson/TestInjectableValue.java | 19 + .../java/com/twitter/util/jackson/tests/BUILD | 9 + .../util/jackson/tests/JsonDiffJavaTest.java | 86 + .../tests/ScalaObjectMapperJavaTest.java | 47 + .../test/scala/com/twitter/util/jackson/BUILD | 41 + .../com/twitter/util/jackson/JSONTest.scala | 81 + .../twitter/util/jackson/JsonDiffTest.scala | 256 ++ .../twitter/util/jackson/JsonGenerator.scala | 72 + .../jackson/NoValidatorImplConstraint.java | 38 + .../jackson/ScalaObjectMapperFunctions.scala | 95 + .../util/jackson/ScalaObjectMapperTest.scala | 2509 +++++++++++++++++ .../ThrowsRuntimeExceptionConstraint.java | 38 + ...sRuntimeExceptionConstraintValidator.scala | 14 + .../caseclass/CaseClassFieldTest.scala | 226 ++ .../caseclass/CaseClassValidationsTest.scala | 501 ++++ .../caseclass/OptionalValidationTest.scala | 87 + .../RichJsonProcessingExceptionTest.scala | 92 + .../util/jackson/caseclass/TypesTest.scala | 34 + .../twitter/util/jackson/caseclasses.scala | 841 ++++++ .../DurationStringDeserializerTest.scala | 52 + .../serde/LocalDateTimeSerdeTest.scala | 118 + .../serde/TimeStringDeserializerTest.scala | 113 + .../serde/TimeStringSerializerTest.scala | 139 + 69 files changed, 9999 insertions(+), 4 deletions(-) create mode 100644 doc/src/sphinx/util-jackson/index.rst create mode 100644 util-benchmark/src/main/scala/com/twitter/json/JacksonBenchmark.scala create mode 100644 util-benchmark/src/main/scala/com/twitter/json/TestDemographic.java create mode 100644 util-benchmark/src/main/scala/com/twitter/json/TestFormat.java create mode 100644 util-core/src/main/scala/com/twitter/util/WrappedValue.scala create mode 100644 util-jackson-annotations/PROJECT create mode 100644 util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD create mode 100644 util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/InjectableValue.java create mode 100644 util-jackson/PROJECT create mode 100644 util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD create mode 100644 util-jackson/src/main/scala/com/fasterxml/jackson/databind/DeserializationContextAccessor.scala create mode 100644 util-jackson/src/main/scala/com/fasterxml/jackson/databind/ObjectMapperCopier.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/BUILD create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/JsonDiff.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassBeanProperty.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassDeserializer.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassDeserializerResolver.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassField.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassJacksonModule.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/SerdeLogging.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/Types.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/CaseClassFieldMappingException.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/CaseClassMappingException.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/InjectableValuesException.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/package.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/package.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/package.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/serde/DefaultSerdeModule.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/serde/DurationStringDeserializer.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/serde/DurationStringSerializer.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/serde/LongKeyDeserializer.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/serde/TimeStringDeserializer.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/serde/TimeStringSerializer.scala create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/serde/WrappedValueSerializer.scala create mode 100644 util-jackson/src/test/java/com/twitter/util/jackson/BUILD create mode 100644 util-jackson/src/test/java/com/twitter/util/jackson/CarMake.java create mode 100644 util-jackson/src/test/java/com/twitter/util/jackson/CarMakeEnum.java create mode 100644 util-jackson/src/test/java/com/twitter/util/jackson/TestInjectableValue.java create mode 100644 util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD create mode 100644 util-jackson/src/test/java/com/twitter/util/jackson/tests/JsonDiffJavaTest.java create mode 100644 util-jackson/src/test/java/com/twitter/util/jackson/tests/ScalaObjectMapperJavaTest.java create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/BUILD create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/JsonDiffTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/JsonGenerator.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/NoValidatorImplConstraint.java create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperFunctions.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/ThrowsRuntimeExceptionConstraint.java create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/ThrowsRuntimeExceptionConstraintValidator.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/CaseClassFieldTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/CaseClassValidationsTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/OptionalValidationTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/RichJsonProcessingExceptionTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/TypesTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/serde/DurationStringDeserializerTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/serde/LocalDateTimeSerdeTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/serde/TimeStringDeserializerTest.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/serde/TimeStringSerializerTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3b2a228da0..ec244827ec 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,16 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-jackson: Introduce a new library for JSON serialization and deserialization based on the + Jackson integration in `Finatra `__. + + This includes a custom case class deserializer which "fails slow" to collect all mapping failures + for error reporting. This deserializer is also natively integrated with the util-validator library + to provide for performing case class validations during deserialization. ``PHAB_ID=D664962`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ @@ -26,8 +36,8 @@ New Features validators for annotating and validating fields of Scala case classes. ``PHAB_ID=D638603`` * util-app: Introduce a Java-friendly API `c.t.app.App#runOnExit(Runnable)` and -`c.t.app.App#runOnExitLast(Runnable)` to allow Java 8 users to call `c.t.app.App#runOnExit` -and `c.t.app.App#runOnExitLast` with lambda expressions. ``PHAB_ID=D511536`` + `c.t.app.App#runOnExitLast(Runnable)` to allow Java 8 users to call `c.t.app.App#runOnExit` + and `c.t.app.App#runOnExitLast` with lambda expressions. ``PHAB_ID=D511536`` 21.4.0 ------ diff --git a/build.sbt b/build.sbt index 38d7689655..e0ae40bba0 100644 --- a/build.sbt +++ b/build.sbt @@ -201,6 +201,8 @@ lazy val util = Project( utilDoc, utilFunction, utilHashing, + utilJacksonAnnotations, + utilJackson, utilJvm, utilLint, utilLogging, @@ -252,7 +254,15 @@ lazy val utilBenchmark = Project( JmhPlugin ).settings( name := "util-benchmark" - ).dependsOn(utilCore, utilHashing, utilJvm, utilReflect, utilStats, utilValidator) + ).dependsOn( + utilCore, + utilHashing, + utilJackson, + utilJvm, + utilReflect, + utilStats, + utilValidator + ) lazy val utilCache = Project( id = "util-cache", @@ -377,6 +387,37 @@ lazy val utilHashing = Project( ) ).dependsOn(utilCore % "test") +lazy val utilJacksonAnnotations = Project( + id = "util-jackson-annotations", + base = file("util-jackson-annotations") +).settings( + sharedSettings + ).settings( + name := "util-jackson-annotations" + ) + +lazy val utilJackson = Project( + id = "util-jackson", + base = file("util-jackson") +).settings( + sharedSettings + ).settings( + name := "util-jackson", + libraryDependencies ++= Seq( + "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion, + "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion, + "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava"), + "jakarta.validation" % "jakarta.validation-api" % "3.0.0", + "org.json4s" %% "json4s-core" % "3.6.7", + "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion % "test", + scalacheckLib, + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", + "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" + ) + ).dependsOn(utilCore, utilJacksonAnnotations, utilMock % Test, utilReflect, utilSlf4jApi, utilValidator) + lazy val utilJvm = Project( id = "util-jvm", base = file("util-jvm") diff --git a/doc/src/sphinx/index.rst b/doc/src/sphinx/index.rst index 6d24186b36..3380ba6da5 100644 --- a/doc/src/sphinx/index.rst +++ b/doc/src/sphinx/index.rst @@ -13,6 +13,7 @@ Guides util-stats/index util-cookbook/index + util-jackson/index util-validator/index Changelog diff --git a/doc/src/sphinx/util-jackson/index.rst b/doc/src/sphinx/util-jackson/index.rst new file mode 100644 index 0000000000..448668c89b --- /dev/null +++ b/doc/src/sphinx/util-jackson/index.rst @@ -0,0 +1,1005 @@ +.. _util-jackson-index: + +util-jackson Guide +================== + +The library builds upon the excellent |jackson-module-scala|_ for `JSON `_ +support by wrapping the `Jackson ScalaObjectMapper `__ +to provide an API very similar to the `Jerkson `__ `Parser `__. + +Additionally, the library provides a default improved `case class deserializer <#improved-case-class-deserializer>`_ +which *accumulates* deserialization errors while parsing JSON into a `case class` instead of failing-fast, +such that all errors can be reported at once. + +.. admonition :: 🚨 This documentation assumes some level of familiarity with `Jackson JSON Processing `__ + + Specifically Jackson `databinding `__ + and the Jackson `ObjectMapper `__. + + There are several `tutorials `__ (and `external documentation `__) + which may be useful if you are unfamiliar with Jackson. + + Additionally, you may want to familiarize yourself with the `Jackson Annotations `_ + as they allow for finer-grain customization of Jackson databinding. + +Case Classes +------------ + +As mentioned, `Jackson `__ is a JSON processing library. We +generally use Jackson for `databinding `__, +or more specifically: + +- object serialization: converting an object **into a JSON String** and +- object deserialization: converting a JSON String **into an object** + +This library integration is primarily centered around serializing and deserializing Scala +`case classes `__. This is because Scala +`case classes `__ map well to the two JSON +structures [`reference `__]: + +- A collection of name/value pairs. Generally termed an *object*. Here, the name/value pairs are the case field name to field value but can also be an actual Scala `Map[T, U]` as well. +- An ordered list of values. Typically an *array*, *vector*, *list*, or *sequence*, which for case classes can be represented by a Scala `Iterable`. + +Library Features +---------------- + +- Usable as a replacement for the |jackson-module-scala|_ or `Jerkson `__ in many cases. +- A `c.t.util.jackson.ScalaObjectMapper `__ which provides additional Scala friendly methods not found in the |jackson-module-scala|_ `ScalaObjectMapper`. +- A custom `case class deserializer `__ which overcomes some of the limitations in |jackson-module-scala|_. +- Integration with the `util-validator `__ Bean Validation 2.0 (`beanvalidation.org `_) style validations during JSON deserialization. +- Utility for quick JSON parsing with `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__ +- Utility for easily comparing JSON strings. + +Basic Usage +----------- + +Let's assume we have these two case classes: + +.. code:: scala + + case class Bar(d: String) + case class Foo(a: String, b: Int, c: Bar) + +To **serialize** a case class into a JSON string, use + +.. code:: scala + + ScalaObjectMapper#writeValueAsString(any: Any): String // or + JSON#write(any: Any) + +For example: + +.. code:: scala + :emphasize-lines: 10, 15, 30, 33, 38 + + Welcome to Scala 2.12.12 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> val foo = Foo("Hello, World", 42, Bar("Goodbye, World")) + foo: Foo = Foo(Hello, World,42,Bar(Goodbye, World)) + + scala> import com.twitter.util.jackson.JSON + import com.twitter.util.jackson.JSON + + scala> JSON.write(foo) + res3: String = {"a":"Hello, World","b":42,"c":{"d":"Goodbye, World"}} + + scala> // or use the configured "pretty print mapper" + + scala> JSON.prettyPrint(foo) + res4: String = + { + "a" : "Hello, World", + "b" : 42, + "c" : { + "d" : "Goodbye, World" + } + } + + scala> // or use a configured ScalaObjectMapper + + scala> import com.twitter.util.jackson.ScalaObjectMapper + import com.twitter.util.jackson.ScalaObjectMapper + + scala> val mapper = ScalaObjectMapper() + mapper: com.twitter.util.jackson.ScalaObjectMapper = com.twitter.util.jackson.ScalaObjectMapper@490d9c41 + + scala> mapper.writeValueAsString(foo) + res0: String = {"a":"Hello, World","b":42,"c":{"d":"Goodbye, World"}} + + scala> // or use the configured "pretty print mapper" + + scala> mapper.writePrettyString(foo) + res1: String = + { + "a" : "Hello, World", + "b" : 42, + "c" : { + "d" : "Goodbye, World" + } + } + +To **deserialize** a JSON string into a case class, use + +.. code:: scala + + ScalaObjectMapper#parse[T](s: String): T // or + JSON#parse[T](s: String): T + +For example, assuming the same `Bar` and `Foo` case classes defined above: + +.. code:: scala + :emphasize-lines: 10, 21 + + Welcome to Scala 2.12.12 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> val s = """{"a": "Hello, World", "b": 42, "c": {"d": "Goodbye, World"}}""" + s: String = {"a": "Hello, World", "b": 42, "c": {"d": "Goodbye, World"}} + + scala> import com.twitter.util.jackson.JSON + import com.twitter.util.jackson.JSON + + scala> val foo = JSON.parse[Foo](s) + foo: Option[Foo] = Some(Foo(Hello, World,42,Bar(Goodbye, World))) + + scala> // or use a configured ScalaObjectMapper + + scala> import com.twitter.util.jackson.ScalaObjectMapper + import com.twitter.util.jackson.ScalaObjectMapper + + scala> val mapper = ScalaObjectMapper() + mapper: com.twitter.util.jackson.ScalaObjectMapper = com.twitter.util.jackson.ScalaObjectMapper@3b64f131 + + scala> val foo = mapper.parse[Foo](s) + foo: Foo = Foo(Hello, World,42,Bar(Goodbye, World)) + +.. tip:: + + As seen above you can use the `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__ utility for + general JSON serde operations with a default configured `ScalaObjectMapper <#defaults>`__. + + This can be useful when you do not need to use a specifically configured `ScalaObjectMapper` or + do not wish to perform any Bean Validation 2.0 style validations during JSON deserialization since + the `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__ utility **specifically disables** + validation support on its underlying `c.t.util.jackson.ScalaObjectMapper`. + + See the `documentation <#c-t-util-jackson-json>`__ for more information. + +You can find many examples of using the `ScalaObjectMapper` in the various framework tests: + +- Scala `example `__. +- Java `example `__. + +As mentioned above, there is also a plethora of Jackson `tutorials `__ and `HOW-TOs `__ +available online which provide more in-depth examples of how to use a Jackson `ObjectMapper `__. + +`ScalaObjectMapper` +------------------- + +The |ScalaObjectMapper|_ is a thin wrapper around a configured |jackson-module-scala|_ `ScalaObjectMapper`. +However, the util-jackson |ScalaObjectMapper|_ comes configured with several defaults when instantiated. + +Defaults +~~~~~~~~ + +The following integrations are provided by default when using the |ScalaObjectMapper|_: + +- The Jackson `DefaultScalaModule `__. +- The Jackson `JSR310 `__ `c.f.jackson.datatype.jsr310.JavaTimeModule `__. +- A `LongKeyDeserializer `__ which allows for deserializing Scala Maps with Long keys. +- A `WrappedValueSerializer `__ (more information on "WrappedValues" `here `__). +- Twitter `c.t.util.Time `_ and `c.t.util.Duration `_ serializers [`1 `__, `2 `__] and deserializers [`1 `__, `2 `__]. +- An improved `CaseClassDeserializer `__: see details `below <#improved-case-class-deserializer>`__. +- Integration with the `util-validator `__ Bean Validation 2.0 style validations during JSON deserialization. + +Instantiation +~~~~~~~~~~~~~ + +Instantiation of a new |ScalaObjectMapper|_ can done either via a companion object method or via +the `ScalaObject#Builder` to specify custom configuration. + +`ScalaObjectMapper#apply` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The companion object defines `apply` methods for creation of a |ScalaObjectMapper|_ configured +with the defaults listed above: + +.. code:: scala + + import com.twitter.util.jackson.ScalaObjectMapper + import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper} + import com.fasterxml.jackson.module.scala.{ScalaObjectMapper => JacksonScalaObjectMapper} + + val objectMapper: ScalaObjectMapper = ScalaObjectMapper() + + val underlying: JacksonObjectMapper with JacksonScalaObjectMapper = ??? + val objectMapper: ScalaObjectMapper = ScalaObjectMapper(underlying) + +.. important:: + + The above `#apply` which takes an underlying Jackson `ObjectMapper` will **mutate the configuration** + of the underlying Jackson `ObjectMapper` to apply the default configuration to the given Jackson + `ObjectMapper`. Thus it is not expected that this underlying Jackson `ObjectMapper` be a shared + resource. + +Companion Object Wrappers +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The companion object also defines other methods to easily obtain some specifically configured +|ScalaObjectMapper|_ which wraps an *already configured* Jackson `ObjectMapper`: + +.. code:: scala + + import com.twitter.util.jackson.ScalaObjectMapper + import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper} + import com.fasterxml.jackson.module.scala.{ScalaObjectMapper => JacksonScalaObjectMapper} + + val underlying: JacksonObjectMapper with JacksonScalaObjectMapper = ??? + + // different from `#apply(underlying)`. wraps a copy of the given Jackson mapper + // and does not apply any configuration. + val objectMapper: ScalaObjectMapper = ScalaObjectMapper.objectMapper(underlying) + + // merely wraps a copy of the given Jackson mapper that is expected to be configured + // with a YAMLFactory, does not apply any configuration. + val objectMapper: ScalaObjectMapper = ScalaObjectMapper.yamlObjectMapper(underlying) + + // only sets the PropertyNamingStrategy to be PropertyNamingStrategy.LOWER_CAMEL_CASE + // to a copy of the given Jackson mapper, does not apply any other configuration. + val objectMapper: ScalaObjectMapper = ScalaObjectMapper.camelCaseObjectMapper(underlying) + + // only sets the PropertyNamingStrategy to be PropertyNamingStrategy.SNAKE_CASE + // to a copy of the given Jackson mapper, does not apply any other configuration. + val objectMapper: ScalaObjectMapper = ScalaObjectMapper.snakeCaseObjectMapper(underlying) + +.. admonition:: These methods clone the underlying mapper, they do not mutate the given Jaskson mapper. + + Note that these methods will *copy* the underlying Jackson mapper (not mutate it) to apply any + necessary configuration to produce a new |ScalaObjectMapper|_, which in this case is only to + change the `PropertyNamingStrategy` accordingly. No other configuration changes are made + to the copy of the underlying Jackson mapper. + + Specifically, note that the `ScalaObjectMapper.objectMapper(underlying)` **wraps a copy but does not mutate the original** + configuration of the given underlying Jackson `ObjectMapper`. This is different from the + `ScalaObjectMapper(underlying)` which **mutates** the given underlying Jackson `ObjectMapper` to + apply all of the default configuration to produce a `ScalaObjectMapper`. + +`ScalaObjectMapper#Builder` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can use the `ScalaObjectMapper#Builder` for more advanced control over configuration options for +producing a configured `ScalaObjectMapper`. For example, to create an instance of a |ScalaObjectMapper|_ +with case class validation via the util-validator `ScalaValidator` **disabled**: + +.. code :: scala + + import com.twitter.util.jackson.ScalaObjectMapper + + val objectMapper: ScalaObjectMapper = ScalaObjectMapper.builder.withNoValidation.objectMapper + +See the `Advanced Configuration <#advanced-configuration>`__ section for more information. + +Advanced Configuration +~~~~~~~~~~~~~~~~~~~~~~ + +To apply more custom configuration to create a |ScalaObjectMapper|_, there is a builder for +constructing a customized mapper. + +E.g., to set a `PropertyNamingStrategy` different than the default: + +.. code:: scala + :emphasize-lines: 3 + + val objectMapper: ScalaObjectMapper = + ScalaObjectMapper.builder + .withPropertyNamingStrategy(PropertyNamingStrategy.KebabCaseStrategy) + .objectMapper + +Or to set additional modules or configuration: + +.. code:: scala + :emphasize-lines: 4, 5, 6, 7 + + val objectMapper: ScalaObjectMapper = + ScalaObjectMapper.builder + .withPropertyNamingStrategy(PropertyNamingStrategy.KebabCaseStrategy) + .withAdditionalJacksonModules(Seq(MySimpleJacksonModule)) + .withAdditionalMapperConfigurationFn( + _.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + ) + .objectMapper + +You can also get a `camelCase`, `snake_case`, or even a YAML configured mapper. + +.. code:: scala + :emphasize-lines: 7, 15, 23 + + val camelCaseObjectMapper: ScalaObjectMapper = + ScalaObjectMapper.builder + .withAdditionalJacksonModules(Seq(MySimpleJacksonModule)) + .withAdditionalMapperConfigurationFn( + _.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + ) + .camelCaseObjectMapper + + val snakeCaseObjectMapper: ScalaObjectMapper = + ScalaObjectMapper.builder + .withAdditionalJacksonModules(Seq(MySimpleJacksonModule)) + .withAdditionalMapperConfigurationFn( + _.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + ) + .snakeCaseObjectMapper + + val yamlObjectMapper: ScalaObjectMapper = + ScalaObjectMapper.builder + .withAdditionalJacksonModules(Seq(MySimpleJacksonModule)) + .withAdditionalMapperConfigurationFn( + _.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true) + ) + .yamlObjectMapper + +Access to the underlying Jackson Object Mapper +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The |ScalaObjectMapper|_ is a thin wrapper around a configured Jackson |jackson-module-scala|_ +`com.fasterxml.jackson.module.scala.ScalaObjectMapper`, thus you can always access the underlying +Jackson object mapper by calling `underlying`: + +.. code:: scala + :emphasize-lines: 7 + + import com.fasterxml.jackson.databind.ObjectMapper + import com.fasterxml.jackson.module.scala.{ScalaObjectMapper => JacksonScalaObjectMapper} + import com.twitter.util.jackson.ScalaObjectMapper + + val objectMapper: ScalaObjectMapper = ??? + + val jacksonObjectMapper: ObjectMapper with JacksonScalaObjectMapper = objectMapper.underlying + +Adding a Custom Serializer or Deserializer +------------------------------------------ + +For more information see the Jackson documentation for +`Custom Serializers `__. + +Add a Jackson Module to a |ScalaObjectMapper|_ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Follow the steps to create a Jackson Module for the custom serializer or deserializer then register +the module to the underlying Jackson mapper from the |ScalaObjectMapper|_ instance: + +.. code:: scala + :emphasize-lines: 13, 14, 15, 43, 44, 45, 46, 47, 48, 52, 53 + + import com.fasterxml.jackson.databind.JsonDeserializer + import com.fasterxml.jackson.databind.deser.Deserializers + import com.fasterxml.jackson.databind.module.SimpleModule + import com.fasterxml.jackson.module.scala.JacksonModule + import com.twitter.util.jackson.ScalaObjectMapper + + // custom deserializer + class FooDeserializer extends JsonDeserializer[Foo] { + override def deserialize(...) + } + + // Jackson SimpleModule for custom deserializer + class FooDeserializerModule extends SimpleModule { + addDeserializer(FooDeserializer) + } + + // custom parameterized deserializer + class MapIntIntDeserializer extends JsonDeserializer[Map[Int, Int]] { + override def deserialize(...) + } + + // custom parameterized deserializer resolver + class MapIntIntDeserializerResolver extends Deserializers.Base { + override def findBeanDeserializer( + javaType: JavaType, + config: DeserializationConfig, + beanDesc: BeanDescription + ): MapIntIntDeserializer = { + if (javaType.isMapLikeType && javaType.hasGenericTypes && hasIntTypes(javaType)) { + new MapIntIntDeserializer + } else null + } + + private[this] def hasIntTypes(javaType: JavaType): Boolean = { + val k = javaType.containedType(0) + val v = javaType.containedType(1) + k.isPrimitive && k.getRawClass == classOf[Integer] && + v.isPrimitive && v.getRawClass == classOf[Integer] + } + } + + // Jackson SimpleModule for custom deserializer + class MapIntIntDeserializerModule extends JacksonModule { + override def getModuleName: String = this.getClass.getName + this += { + _.addDeserializers(new MapIntIntDeserializerResolver) + } + } + + ... + + val mapper: ScalaObjectMapper = ??? + mapper.registerModules(new FooDeserializerModule, new MapIntIntDeserializerModule) + +Improved `case class` deserializer +---------------------------------- + +The library provides a `case class deserializer `__ +which overcomes some limitations in |jackson-module-scala|_: + +- Throws a `JsonMappingException` when required fields are missing from the parsed JSON. +- Uses specified `case class` default values when fields are missing in the incoming JSON. +- Properly deserializes a `Seq[Long]` (see: https://github.com/FasterXML/jackson-module-scala/issues/62). +- Supports `"wrapped values" `__ using `c.t.util.jackson.WrappedValue `_. +- Support for field and method level validations via integration with the `util-validator `__ Bean Validation 2.0 style validations during JSON deserialization. +- Accumulates all JSON deserialization errors (instead of failing fast) in a returned sub-class of `JsonMappingException` (see: `CaseClassMappingException `_). + +The `case class deserializer `__ +is added by default when constructing a new |ScalaObjectMapper|_. + +.. tip:: + + Note: with the |CaseClassDeserializer|_, non-option fields without default values are + **considered required**. If a required field is missing, a `CaseClassMappingException` is thrown. + + JSON `null` values are **not allowed and will be treated as a "missing" value**. If necessary, users + can specify a custom deserializer for a field if they want to be able to parse a JSON `null` into + a Scala `null` type for a field. Define your deserializer, `NullAllowedDeserializer` then annotate + the field with `@JsonDeserialize(using = classOf[NullAllowedDeserializer])`. + +`@JsonCreator` Support +~~~~~~~~~~~~~~~~~~~~~~ + +The |CaseClassDeserializer|_ supports specification of a constructor or static factory +method annotated with the Jackson Annotation, `@JsonCreator `_ +(an annotation for indicating a specific constructor or static factory method to use for +instantiation of the case class during deserialization). + +For example, you can annotate a method on the companion object for the case class as a static +factory for instantiation. Any static factory method to use for instantiation **MUST** be specified +on the companion object for case class: + +.. code:: scala + :emphasize-lines: 4 + + case class MySimpleCaseClass(int: Int) + + object MySimpleCaseClass { + @JsonCreator + def apply(s: String): MySimpleCaseClass = MySimpleCaseClass(s.toInt) + } + +Or to specify a secondary constructor to use for case class instantiation: + +.. code:: scala + :emphasize-lines: 2 + + case class MyCaseClassWithMultipleConstructors(number1: Long, number2: Long, number3: Long) { + @JsonCreator + def this(numberAsString1: String, numberAsString2: String, numberAsString3: String) { + this(numberAsString1.toLong, numberAsString2.toLong, numberAsString3.toLong) + } + } + +.. note:: + + If you define multiple constructors on a case class, it is **required** to annotate one of the + constructors with `@JsonCreator`. + + To annotate the primary constructor (as the syntax can seem non-intuitive because the `()` is + required): + + .. code:: scala + :emphasize-lines: 1 + + case class MyCaseClassWithMultipleConstructors @JsonCreator()(number1: Long, number2: Long, number3: Long) { + def this(numberAsString1: String, numberAsString2: String, numberAsString3: String) { + this(numberAsString1.toLong, numberAsString2.toLong, numberAsString3.toLong) + } + } + + The parens are needed because the Scala class constructor syntax requires constructor + annotations to have exactly one parameter list, possibly empty. + + If you define multiple case class constructors with no visible `@JsonCreator` constructor or + static factory method via a companion, deserialization will error. + +`@JsonFormat` Support +~~~~~~~~~~~~~~~~~~~~~ + +The |CaseClassDeserializer|_ supports `@JsonFormat`-annotated case class fields to properly +contextualize deserialization based on the values in the annotation. + +A common use case is to be able to support deserializing a JSON string into a "time" representation +class based on a specific pattern independent of the time format configured on the `ObjectMapper` or +even the default format for a given deserializer for the type. + +For instance, the library provides a `specific deserializer `_ +for the `com.twitter.util.Time `_ +class. This deserializer is a Jackson `ContextualDeserializer `_ +and will properly take into account a `@JsonFormat`-annotated field. + +However, the |CaseClassDeserializer|_ is invoked first and acts as a proxy for deserializing the time +value. The case class deserializer properly contextualizes the field for correct deserialization by +the `TimeStringDeserializer`. + +Thus if you had a case class defined: + +.. code:: scala + :emphasize-lines: 7 + + import com.fasterxml.jackson.annotation.JsonFormat + import com.twitter.util.Time + + case class Event( + id: Long, + description: String, + @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") when: Time + ) + +The following JSON: + +.. code:: json + + { + "id": 42, + "description": "Something happened.", + "when": "2018-09-14T23:20:08.000-07:00" + } + +Will always deserialize properly into the case class regardless of the pattern configured on the +`ObjectMapper` or as the default of a contextualized deserializer: + +.. code:: scala + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> import com.fasterxml.jackson.annotation.JsonFormat + import com.fasterxml.jackson.annotation.JsonFormat + + scala> import com.twitter.util.Time + import com.twitter.util.Time + + scala> case class Event( + | id: Long, + | description: String, + | @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") when: Time + | ) + defined class Event + + scala> val json = """ + | { + | "id": 42, + | "description": "Something happened.", + | "when": "2018-09-14T23:20:08.000-07:00" + | }""".stripMargin + json: String = + " + { + "id": 42, + "description": "Something happened.", + "when": "2018-09-14T23:20:08.000-07:00" + }" + + scala> import com.twitter.util.jackson.ScalaObjectMapper + import com.twitter.util.jackson.ScalaObjectMapper + + scala> val mapper = ScalaObjectMapper() + mapper: com.twitter.util.jackson.ScalaObjectMapper = com.twitter.util.jackson.ScalaObjectMapper@52dc71b2 + + scala> val event: Event = mapper.parse[Event](json) + event: Event = Event(42,Something happened.,2018-09-15 06:20:08 +0000) + +Jackson InjectableValues Support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, the library does not configure any `com.fasterxml.jackson.databind.InjectableValues `__ +implementation. + +`@InjectableValue` +^^^^^^^^^^^^^^^^^^ + +It does however provide the `c.t.util.jackson.annotation.InjectableValue `__ annotation which can be used to mark *other* `java.lang.annotation.Annotation` interfaces as annotations +which support case class field injection via Jackson `com.fasterxml.jackson.databind.InjectableValues`. + +That is, users can create custom annotations and annotate them with `@InjectableValue`. This +then allows for a configured Jackson `com.fasterxml.jackson.databind.InjectableValues` implementation +to be able to treat these annotations similar to the `@JacksonInject `__. + +This means a custom Jackson `InjectableValues` implementation can use the `@InjectableValue` marker +annotation to resolve fields annotated with annotations that have the `@InjectableValue` marker +annotation as injectable fields. + +For more information on the Jackson `@JacksonInject` or `c.f.databind.InjectableValues` support see the +tutorial `here `__. + +`Mix-in Annotations `_ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Jackson `Mix-in Annotations `_ +provide a way to associate annotations to classes without needing to modify the target classes +themselves. It is intended to help support 3rd party datatypes where the user cannot modify the +sources to add annotations. + +The |CaseClassDeserializer|_ supports Jackson `Mix-in Annotations `_ +for specifying field annotations during deserialization with the `case class deserializer `_. + +For example, to deserialize JSON into the following classes that are not yours to annotate: + +.. code:: scala + + case class Point(x: Int, y: Int) { + def area: Int = x * y + } + + case class Points(points: Seq[Point]) + +However, you want to enforce field constraints with `validations <../util-validator/index.html>`_ +during deserialization. You can define a `Mix-in`, + +.. code:: scala + + import com.fasterxml.jackson.annotation.JsonIgnore + import jakarta.validation.constraints.{Max, Min} + + trait PointMixIn { + @Min(0) @Max(100) def x: Int + @Min(0) @Max(100) def y: Int + @JsonIgnore def area: Int + } + +Then register this `Mix-in` for the `Point` class type. There are several ways to do this: + +Follow the steps to create a Jackson Module for the `Mix-in` then register the module to the +underlying Jackson mapper from the |ScalaObjectMapper|_ instance: + +.. code:: scala + + import com.fasterxml.jackson.databind.module.SimpleModule + import com.twitter.util.jackson.ScalaObjectMapper + + object PointMixInModule extends SimpleModule { + setMixInAnnotation(classOf[Point], classOf[PointMixIn]); + } + + ... + + val objectMapper: ScalaObjectMapper = ??? + objectMapper.registerModule(PointMixInModule) + +Or register the `Mix-in` for the class type directly on the underlying Jackson mapper (without a Jackson Module): + +.. code:: scala + + import com.twitter.util.jackson.ScalaObjectMapper + + val objectMapper: ScalaObjectMapper = ??? + objectMapper.underlying.addMixin[Point, PointMixIn] + +Deserializing this JSON would then error with failed validations: + +.. code:: json + + { + "points": [ + {"x": -1, "y": 120}, + {"x": 4, "y": 99} + ] + } + +.. code:: scala + :emphasize-lines: 51, 52, 53 + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> case class Point(x: Int, y: Int) { + | def area: Int = x * y + | } + defined class Point + + scala> case class Points(points: Seq[Point]) + defined class Points + + scala> import com.fasterxml.jackson.annotation.JsonIgnore + import com.fasterxml.jackson.annotation.JsonIgnore + + scala> import jakarta.validation.constraints.{Max, Min} + import jakarta.validation.constraints.{Max, Min} + + scala> trait PointMixIn { + | @Min(0) @Max(100) def x: Int + | @Min(0) @Max(100) def y: Int + | @JsonIgnore def area: Int + | } + defined trait PointMixIn + + scala> import com.twitter.util.jackson.ScalaObjectMapper + import com.twitter.util.jackson.ScalaObjectMapper + + scala> val objectMapper: ScalaObjectMapper = ScalaObjectMapper() + objectMapper: com.twitter.util.jackson.ScalaObjectMapper = com.twitter.util.jackson.ScalaObjectMapper@2389f546 + + scala> objectMapper.underlying.addMixin[Point, PointMixIn] + res0: com.fasterxml.jackson.databind.ObjectMapper = com.twitter.util.jackson.ScalaObjectMapper$Builder$$anon$1@5ae22651 + + scala> val json = """ + | { + | "points": [ + | {"x": -1, "y": 120}, + | {"x": 4, "y": 99} + | ] + | }""".stripMargin + json: String = + " + { + "points": [ + {"x": -1, "y": 120}, + {"x": 4, "y": 99} + ] + }" + + scala> val points = objectMapper.parse[Points](json) + com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException: 2 errors encountered during deserialization. + Errors: com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException: points.x: must be greater than or equal to 0 + com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException: points.y: must be less than or equal to 100 + at com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException$.apply(CaseClassMappingException.scala:21) + at com.twitter.util.jackson.caseclass.CaseClassDeserializer.deserialize(CaseClassDeserializer.scala:431) + at com.twitter.util.jackson.caseclass.CaseClassDeserializer.deserializeNonWrapperClass(CaseClassDeserializer.scala:408) + at com.twitter.util.jackson.caseclass.CaseClassDeserializer.deserialize(CaseClassDeserializer.scala:373) + at com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4524) + at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3466) + at com.fasterxml.jackson.module.scala.ScalaObjectMapper.readValue(ScalaObjectMapper.scala:191) + at com.fasterxml.jackson.module.scala.ScalaObjectMapper.readValue$(ScalaObjectMapper.scala:190) + at com.twitter.util.jackson.ScalaObjectMapper$Builder$$anon$1.readValue(ScalaObjectMapper.scala:382) + at com.twitter.util.jackson.ScalaObjectMapper.parse(ScalaObjectMapper.scala:463) + ... 34 elided + +As the first `Point` instance has an x-value less than the minimum of 0 and a y-value greater than +the maximum of 100. + +Known `CaseClassDeserializer` Limitations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The |CaseClassDeserializer|_ provides a fair amount of utility but can not and does not +support all Jackson Annotations. The behavior of supporting a Jackson Annotation can at times be +ambiguous (or even nonsensical), especially when it comes to combining Jackson Annotations and +injectable field annotations. + +Java Enums +~~~~~~~~~~ + +We recommend the use of `Java Enums `__ +for representing enumerations since they integrate well with Jackson's ObjectMapper and have +exhaustiveness checking as of Scala 2.10. + +The following `Jackson annotations `__ may be +useful when working with Enums: + +- `@JsonValue`: can be used for an overridden `toString` method. +- `@JsonEnumDefaultValue`: can be used for defining a default value when deserializing unknown Enum values. Note that this requires `READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE `_ feature to be enabled. + +`c.t.util.jackson.JSON` +----------------------- + +The library provides a utility for default mapping of JSON to an object or writing an object as a JSON +string. This is largely inspired by the `scala.util.parsing.json.JSON `__ +from the Scala Parser Combinators library. + +However, the `c.t.util.jackson.JSON `__ +utility uses a default configured `ScalaObjectMapper <#defaults>`__ and is thus more full featured +than the `scala.util.parsing.json.JSON` utility. + +.. important:: + + The `c.t.util.jackson.JSON` API does not return an exception when parsing but rather returns an + `Option[T]` result. When parsing is successful, this is a `Some(T)`, otherwise it is a `None`. But + note that the specifics of any failure are lost. + + It is thus also important to note that for this reason that the `c.t.util.jackson.JSON` uses a + default configured `ScalaObjectMapper <#defaults>`__ **with validation specifically disabled**, + such that no `Bean Validation 2.0 <../util-validator>`__ style validations are performed when + parsing with `c.t.util.jackson.JSON`. Users should prefer using a configured `ScalaObjectMapper` + to perform validations in order to be able to properly handle validation exceptions. + +.. code:: scala + :emphasize-lines: 7, 13, 22, 25 + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> import com.twitter.util.jackson.JSON + import com.twitter.util.jackson.JSON + + scala> val result = JSON.parse[Map[String, Int]]("""{"a": 1, "b": 2}""") + result: Option[Map[String,Int]] = Some(Map(a -> 1, b -> 2)) + + scala> case class FooClass(id: String) + defined class FooClass + + scala> val result = JSON.parse[FooClass]("""{"id": "abcd1234"}""") + result: Option[FooClass] = Some(FooClass(abcd1234)) + + scala> result.get + res0: FooClass = FooClass(abcd1234) + + scala> val f = FooClass("99999999") + f: FooClass = FooClass(99999999) + + scala> JSON.write(f) + res1: String = {"id":"99999999"} + + scala> JSON.prettyPrint(f) + res2: String = + { + "id" : "99999999" + } + +`c.t.util.jackson.JsonDiff` +--------------------------- + +The library provides a utility for comparing JSON strings or structures which can be serialized as +JSON strings via the `c.t.util.jackson.JsonDiff `__ utility. + +`JsonDiff` provides two functions: `diff` and `assertDiff`. The `diff` method allows the user to +decide how to handle JSON differences by returning an `Option[JsonDiff.Result]` while `assertDiff` +throws an `AssertionError` when a difference is encountered. + +The `JsonDiff.Result#toString` contains a textual representation meant to show when both the +`expected` and `actual` JSON strings are sorted lexicographically where the first encountered +difference occurs. When an `AssertError` is thrown, the `JsonDiff.Result#toString` is used to +populate the exception message. + +For example: + +.. code:: scala + :emphasize-lines: 13, 19, 54 + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> import com.twitter.util.jackson.JsonDiff + import com.twitter.util.jackson.JsonDiff + + scala> val a ="""{"a":1,"b":2}""" + a: String = {"a":1,"b":2} + + scala> val b ="""{"b": 2,"a": 1}""" + b: String = {"b": 2,"a": 1} + + scala> val result = JsonDiff.diff(a, b) // no difference + result: Option[com.twitter.util.jackson.JsonDiff.Result] = None + + scala> val b ="""{"b": 3,"a": 1}""" + b: String = {"b": 3,"a": 1} + + scala> val result = JsonDiff.diff(a, b) // b-values are different + result: Option[com.twitter.util.jackson.JsonDiff.Result] = + Some( * + Expected: {"a":1,"b":2} + Actual: {"a":1,"b":3}) + + scala> result.get.toString + res1: String = + " * + Expected: {"a":1,"b":2} + Actual: {"a":1,"b":3}" + + scala> result.get.expected + res3: com.fasterxml.jackson.databind.JsonNode = {"a":1,"b":2} + + scala> result.get.actual + res4: com.fasterxml.jackson.databind.JsonNode = {"b":3,"a":1} + + scala> result.get.expectedPrettyString + res5: String = + { + "a" : 1, + "b" : 2 + } + + scala> result.get.actualPrettyString + res6: String = + { + "b" : 3, + "a" : 1 + } + + scala> import com.twitter.util.Try + import com.twitter.util.Try + + scala> val t = Try(JsonDiff.assertDiff(a, b)) // throws an AssertionError + JSON DIFF FAILED! + * + Expected: {"a":1,"b":2} + Actual: {"a":1,"b":3} + + scala> t.isThrow + res0: Boolean = true + + scala> t.throwable.getMessage + res2: String = + " * + Expected: {"a":1,"b":2} + Actual: {"a":1,"b":3}" + +Normalization +~~~~~~~~~~~~~ + +Both API methods accept a "normalize function" which is a function to apply on the *actual* to "normalize" any +fields -- such as a timestamp -- before comparing to the *expected*. + +.. code:: scala + :emphasize-lines: 43, 44, 45, 46, 49, 52 + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> import com.twitter.util.jackson.JsonDiff + import com.twitter.util.jackson.JsonDiff + + scala> val a ="""{"a":1,"b":2}""" + a: String = {"a":1,"b":2} + + scala> val b ="""{"b": 3,"a": 1}""" + b: String = {"b": 3,"a": 1} + + scala> val result = JsonDiff.diff(expected = a, actual = b) // b-values are different + result: Option[com.twitter.util.jackson.JsonDiff.Result] = + Some( * + Expected: {"a":1,"b":2} + Actual: {"a":1,"b":3}) + + scala> result.get.toString + res1: String = + " * + Expected: {"a":1,"b":2} + Actual: {"a":1,"b":3}" + + scala> JsonDiff.assertDiff(expected = a, actual = b) // throws an AssertionError + JSON DIFF FAILED! + * + Expected: {"a":1,"b":2} + Actual: {"a":1,"b":3} + java.lang.AssertionError: com.twitter.util.jackson.JsonDiff$ failure + * + Expected: {"a":1,"b":2} + Actual: {"a":1,"b":3} + at com.twitter.util.jackson.JsonDiff$.assert(JsonDiff.scala:206) + at com.twitter.util.jackson.JsonDiff$.assertDiff(JsonDiff.scala:114) + ... 34 elided + + scala> import com.fasterxml.jackson.databind.JsonNode + import com.fasterxml.jackson.databind.JsonNode + + scala> import com.fasterxml.jackson.databind.node.ObjectNode + import com.fasterxml.jackson.databind.node.ObjectNode + + scala> val normalizeFn: JsonNode => JsonNode = { jsonNode: JsonNode => + | jsonNode.asInstanceOf[ObjectNode].put("b", 2) + | } + normalizeFn: com.fasterxml.jackson.databind.JsonNode => com.fasterxml.jackson.databind.JsonNode = $Lambda$1162/1826212603@3da20c42 + + scala> val result = JsonDiff.diff(expected = a, actual = b, normalizeFn) // normalize fn updates b-value of 'actual' to match the 'expected' + result: Option[com.twitter.util.jackson.JsonDiff.Result] = None + + scala> JsonDiff.assertDiff(expected = a, actual = b, normalizeFn) // no exception when 'actual' is normalized + + scala> + +.. |ScalaObjectMapper| replace:: `ScalaObjectMapper` +.. _ScalaObjectMapper: https://github.com/twitter/util/blob/develop/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala + +.. |CaseClassDeserializer| replace:: `util-jackson case class deserializer` +.. _CaseClassDeserializer: https://github.com/twitter/util/blob/develop/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassDeserializer.scala + +.. |jackson-module-scala| replace:: `jackson-module-scala` +.. _jackson-module-scala: https://github.com/FasterXML/jackson-module-scala + + diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index bcacbf9800..e884e380b1 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -1,5 +1,8 @@ scala_library( - sources = ["**/*.scala"], + sources = [ + "**/*.java", + "**/*.scala", + ], compiler_option_sets = ["fatal_warnings"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", @@ -8,7 +11,9 @@ scala_library( "3rdparty/jvm/org/scala-lang:scala-library", "util/util-core:scala", "util/util-hashing/src/main/scala", + "util/util-jackson/src/main/scala/com/twitter/util/jackson", "util/util-reflect/src/main/scala/com/twitter/util/reflect", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", "util/util-stats/src/main/scala", "util/util-validator/src/main/scala/com/twitter/util/validation", ], diff --git a/util-benchmark/src/main/scala/com/twitter/json/JacksonBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/json/JacksonBenchmark.scala new file mode 100644 index 0000000000..4bc32a2212 --- /dev/null +++ b/util-benchmark/src/main/scala/com/twitter/json/JacksonBenchmark.scala @@ -0,0 +1,75 @@ +package com.twitter.json + +import com.fasterxml.jackson.module.scala.{ScalaObjectMapper => JacksonScalaObjectMapper} +import com.twitter.util.StdBenchAnnotations +import com.twitter.util.jackson.ScalaObjectMapper +import java.io.ByteArrayInputStream +import java.time.LocalDateTime +import org.openjdk.jmh.annotations._ + +object JacksonBenchmark { + case class TestTask(request_id: String, group_ids: Seq[String], params: TestTaskParams) + + case class TestTaskParams( + results: TaskTaskResults, + start_time: LocalDateTime, + end_time: LocalDateTime, + priority: String) + + case class TaskTaskResults( + country_codes: Seq[String], + group_by: Seq[String], + format: Option[TestFormat], + demographics: Seq[TestDemographic]) + +} + +// ./sbt 'project util-benchmark' 'jmh:run JacksonBenchmark' +@State(Scope.Benchmark) +class JacksonBenchmark extends StdBenchAnnotations { + import JacksonBenchmark._ + + private[this] val json = + """{ + "request_id": "00000000-1111-2222-3333-444444444444", + "type": "impression", + "params": { + "priority": "normal", + "start_time": "2013-01-01T00:02:00.000Z", + "end_time": "2013-01-01T00:03:00.000Z", + "results": { + "demographics": ["age", "gender"], + "country_codes": ["US"], + "group_by": ["group"] + } + }, + "group_ids":["grp-1"] + }""" + + private[this] val bytes = json.getBytes("UTF-8") + + /** create framework scala object mapper with all defaults */ + private[this] val mapperWithCaseClassDeserializer: ScalaObjectMapper = + ScalaObjectMapper.builder.objectMapper + + /** create framework scala object mapper without case class deserializer */ + private[this] val mapperWithoutCaseClassDeserializer: ScalaObjectMapper = { + // configure the underlying Jackson ScalaObjectMapper with everything but the CaseClassJacksonModule + val underlying = new com.fasterxml.jackson.databind.ObjectMapper with JacksonScalaObjectMapper + underlying.registerModules(ScalaObjectMapper.DefaultJacksonModules: _*) + // do not use apply which will always install the default jackson modules on the underlying mapper + new ScalaObjectMapper(underlying) + } + + @Benchmark + def withCaseClassDeserializer(): TestTask = { + val is = new ByteArrayInputStream(bytes) + mapperWithCaseClassDeserializer.parse[TestTask](is) + } + + @Benchmark + def withoutCaseClassDeserializer(): TestTask = { + val is = new ByteArrayInputStream(bytes) + mapperWithoutCaseClassDeserializer.parse[TestTask](is) + } +} diff --git a/util-benchmark/src/main/scala/com/twitter/json/TestDemographic.java b/util-benchmark/src/main/scala/com/twitter/json/TestDemographic.java new file mode 100644 index 0000000000..63808a4dc6 --- /dev/null +++ b/util-benchmark/src/main/scala/com/twitter/json/TestDemographic.java @@ -0,0 +1,6 @@ +package com.twitter.json; + +public enum TestDemographic { + gender, + age; +} diff --git a/util-benchmark/src/main/scala/com/twitter/json/TestFormat.java b/util-benchmark/src/main/scala/com/twitter/json/TestFormat.java new file mode 100644 index 0000000000..ca8387f78f --- /dev/null +++ b/util-benchmark/src/main/scala/com/twitter/json/TestFormat.java @@ -0,0 +1,7 @@ +package com.twitter.json; + +public enum TestFormat { + json, + csv, + tsv; +} diff --git a/util-core/src/main/scala/com/twitter/util/WrappedValue.scala b/util-core/src/main/scala/com/twitter/util/WrappedValue.scala new file mode 100644 index 0000000000..95feac3e99 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/WrappedValue.scala @@ -0,0 +1,28 @@ +package com.twitter.util + +/** + * [[WrappedValue]] is a marker interface intended for use by case classes wrapping a single value. + * The intent is to signal that the underlying value should be directly serialized + * or deserialized ignoring the wrapping case class. + * + * [[WrappedValue]] is a [[https://docs.scala-lang.org/overviews/core/value-classes.html Universal Trait]] + * so that it can be used by [[https://docs.scala-lang.org/overviews/core/value-classes.html value classes]]. + * + * ==Usage== + * {{{ + * case class WrapperA(underlying: Int) extends WrappedValue[Int] + * case class WrapperB(underlying: Int) extends AnyVal with WrappedValue[Int] + * }}} + */ +trait WrappedValue[T] extends Any { self: Product => + + def onlyValue: T = self.productElement(0).asInstanceOf[T] + + def asString: String = onlyValue.toString + + /** + * @note we'd rather not override `toString`, but this is required to correctly handle + * [[WrappedValue]] instances used as Map keys. + */ + override def toString: String = asString +} diff --git a/util-jackson-annotations/PROJECT b/util-jackson-annotations/PROJECT new file mode 100644 index 0000000000..2a8a1534c3 --- /dev/null +++ b/util-jackson-annotations/PROJECT @@ -0,0 +1,8 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - ccoco + - ibennett + - jyan + - mnakamura + - roanta diff --git a/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD b/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD new file mode 100644 index 0000000000..f3f0df4f7c --- /dev/null +++ b/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD @@ -0,0 +1,10 @@ +java_library( + sources = ["*.java"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-jackson-annotations", + repo = artifactory, + ), + strict_deps = True, +) diff --git a/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/InjectableValue.java b/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/InjectableValue.java new file mode 100644 index 0000000000..5c8a353697 --- /dev/null +++ b/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/InjectableValue.java @@ -0,0 +1,19 @@ +package com.twitter.util.jackson.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marker annotation for {@link java.lang.annotation.Annotation} interfaces which define + * a field that should be injected into a case class via Jackson + * `com.fasterxml.jackson.databind.InjectableValues`. + * + * @see com.fasterxml.jackson.databind.InjectableValues + * @see Finatra User's Guide - JSON Injectable Values + * @see Finatra User's Guide - HTTP Request Field Annotations + */ +@Target(ElementType.ANNOTATION_TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface InjectableValue {} diff --git a/util-jackson/PROJECT b/util-jackson/PROJECT new file mode 100644 index 0000000000..2a8a1534c3 --- /dev/null +++ b/util-jackson/PROJECT @@ -0,0 +1,8 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - ccoco + - ibennett + - jyan + - mnakamura + - roanta diff --git a/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD new file mode 100644 index 0000000000..f3a51ec8a5 --- /dev/null +++ b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD @@ -0,0 +1,18 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-jackson-objectmapper-copy", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + ], + exports = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + ], +) diff --git a/util-jackson/src/main/scala/com/fasterxml/jackson/databind/DeserializationContextAccessor.scala b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/DeserializationContextAccessor.scala new file mode 100644 index 0000000000..dc6c032557 --- /dev/null +++ b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/DeserializationContextAccessor.scala @@ -0,0 +1,7 @@ +package com.fasterxml.jackson.databind + +/** INTERNAL API ONLY */ +class DeserializationContextAccessor(context: DeserializationContext) { + // Workaround to get access to the DeserializationContext#injectableValues member + def getInjectableValues: Option[InjectableValues] = Option(context._injectableValues) +} diff --git a/util-jackson/src/main/scala/com/fasterxml/jackson/databind/ObjectMapperCopier.scala b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/ObjectMapperCopier.scala new file mode 100644 index 0000000000..369175d89e --- /dev/null +++ b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/ObjectMapperCopier.scala @@ -0,0 +1,11 @@ +package com.fasterxml.jackson.databind + +import com.fasterxml.jackson.module.scala.ScalaObjectMapper + +/** INTERNAL API ONLY */ +object ObjectMapperCopier { + // Workaround since calling objectMapper.copy on "ObjectMapper with ScalaObjectMapper" fails the _checkInvalidCopy check + def copy(objectMapper: ObjectMapper): ObjectMapper with ScalaObjectMapper = { + new ObjectMapper(objectMapper) with ScalaObjectMapper + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD new file mode 100644 index 0000000000..f352f749f0 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD @@ -0,0 +1,35 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-jackson-core", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson-dataformat-yaml", + "3rdparty/jvm/com/fasterxml/jackson/datatype:jackson-datatype-jsr310", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-jackson/src/main/scala/com/fasterxml/jackson/databind", + "util/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass", + "util/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions", + "util/util-jackson/src/main/scala/com/twitter/util/jackson/serde", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + "util/util-validator/src/main/scala/com/twitter/util/validation", + ], + exports = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson-dataformat-yaml", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + ], +) diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala new file mode 100644 index 0000000000..8afa7f8036 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala @@ -0,0 +1,28 @@ +package com.twitter.util.jackson + +import com.twitter.util.Try + +/** + * Uses an instance of a [[ScalaObjectMapper]] configured to not perform any type of validation. + * Inspired by [[https://www.scala-lang.org/api/2.12.6/scala-parser-combinators/scala/util/parsing/json/JSON$.html]]. + * + * @note This is only intended for use from Scala (not Java). + * + * @see [[ScalaObjectMapper]] + */ +object JSON { + private final val Mapper: ScalaObjectMapper = + ScalaObjectMapper.builder.withNoValidation.objectMapper + + /** Simple utility to parse a JSON string into an Option[T] type. */ + def parse[T: Manifest](input: String): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** Simple utility to write a value as a JSON encoded String. */ + def write(any: Any): String = + Mapper.writeValueAsString(any) + + /** Simple utility to pretty print a JSON encoded String from the given instance. */ + def prettyPrint(any: Any): String = + Mapper.writePrettyString(any) +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/JsonDiff.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/JsonDiff.scala new file mode 100644 index 0000000000..f2e8081272 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/JsonDiff.scala @@ -0,0 +1,227 @@ +package com.twitter.util.jackson + +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapperCopier, SerializationFeature} +import com.fasterxml.jackson.databind.node.TextNode +import com.twitter.util.logging.Logging +import java.io.PrintStream +import scala.util.control.NonFatal + +object JsonDiff extends Logging { + + private[this] lazy val mapper: ScalaObjectMapper = + ScalaObjectMapper.builder.objectMapper + + private[this] lazy val sortingObjectMapper: JacksonScalaObjectMapperType = { + val newMapper = ObjectMapperCopier.copy(mapper.underlying) + newMapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + newMapper + } + + /** + * Creates a string representation of the given [[JsonNode]] with entries + * sorted alphabetically by key. + * + * @param jsonNode - input [[JsonNode]] + * @return string representation of the [[JsonNode]]. + */ + def toSortedString(jsonNode: JsonNode): String = { + if (jsonNode.isTextual) { + jsonNode.textValue() + } else { + val node = sortingObjectMapper.treeToValue(jsonNode, classOf[Object]) + sortingObjectMapper.writeValueAsString(node) + } + } + + /** A [[JsonDiff]] result */ + case class Result( + expected: JsonNode, + expectedPrettyString: String, + actual: JsonNode, + actualPrettyString: String) { + + private[this] val expectedJsonSorted = JsonDiff.toSortedString(expected) + private[this] val actualJsonSorted = JsonDiff.toSortedString(actual) + + override val toString: String = { + val expectedHeader = "Expected: " + val diffStartIdx = + actualJsonSorted.zip(expectedJsonSorted).indexWhere { case (x, y) => x != y } + + val message = new StringBuilder + message.append(" " * (expectedHeader.length + diffStartIdx) + "*\n") + message.append(expectedHeader + expectedJsonSorted + "\n") + message.append(s"Actual: $actualJsonSorted") + + message.toString() + } + } + + /** + * Computes the diff for two snippets of json both of expected type `T`. + * If a difference is detected a [[Result]] is returned otherwise a None. + * + * @param expected the expected json + * @param actual the actual or received json + * @return if a difference is detected a [[Result]] + * is returned otherwise a None. + */ + def diff[T](expected: T, actual: T): Option[Result] = + diff[T](expected, actual, None) + + /** + * Computes the diff for two snippets of json both of expected type `T`. + * If a difference is detected a [[Result]] is returned otherwise a None. + * + * @param expected the expected json + * @param actual the actual or received json + * @param normalizeFn a function to apply to the actual json in order to "normalize" values. + * + * @return if a difference is detected a [[Result]] + * is returned otherwise a None. + * + * ==Usage== + * {{{ + * + * private def normalize(jsonNode: JsonNode): JsonNode = jsonNode match { + * case on: ObjectNode => on.put("time", "1970-01-01T00:00:00Z") + * case _ => jsonNode + * } + * + * val expected = """{"foo": "bar", "time": ""1970-01-01T00:00:00Z"}""" + * val actual = ??? ({"foo": "bar", "time": ""2021-05-14T00:00:00Z"}) + * val result: Option[JsonDiff.Result] = JsonDiff.diff(expected, actual, normalize) + * }}} + */ + def diff[T]( + expected: T, + actual: T, + normalizeFn: JsonNode => JsonNode + ): Option[Result] = + diff[T](expected, actual, Option(normalizeFn)) + + /** + * Asserts that the actual equals the expected. Will throw an [[AssertionError]] with details + * printed to [[System.out]]. + * + * @param expected the expected json + * @param actual the actual or received json + * + * @throws AssertionError - when the expected does not match the actual. + */ + @throws[AssertionError] + def assertDiff[T](expected: T, actual: T): Unit = + assert[T](expected, actual, None, System.out) + + /** + * Asserts that the actual equals the expected. Will throw an [[AssertionError]] with details + * printed to [[System.out]] using the given normalizeFn to normalize the actual contents. + * + * @param expected the expected json + * @param actual the actual or received json + * @param normalizeFn a function to apply to the actual json in order to "normalize" values. + * + * @throws AssertionError - when the expected does not match the actual. + */ + @throws[AssertionError] + def assertDiff[T]( + expected: T, + actual: T, + normalizeFn: JsonNode => JsonNode + ): Unit = assert[T](expected, actual, Option(normalizeFn), System.out) + + /** + * Asserts that the actual equals the expected. Will throw an [[AssertionError]] with details + * printed to the given [[PrintStream]]. + * + * @param expected the expected json + * @param actual the actual or received json + * @param p the [[PrintStream]] for reporting details + * + * @throws AssertionError - when the expected does not match the actual. + */ + @throws[AssertionError] + def assertDiff[T](expected: T, actual: T, p: PrintStream): Unit = + assert[T](expected, actual, None, p) + + /** + * Asserts that the actual equals the expected. Will throw an [[AssertionError]] with details + * printed to the given [[PrintStream]] using the given normalizeFn to normalize the actual contents. + * + * @param expected the expected json + * @param actual the actual or received json + * @param p the [[PrintStream]] for reporting details + * @param normalizeFn a function to apply to the actual json in order to "normalize" values. + * + * @throws AssertionError - when the expected does not match the actual. + */ + @throws[AssertionError] + def assertDiff[T]( + expected: T, + actual: T, + normalizeFn: JsonNode => JsonNode, + p: PrintStream + ): Unit = assert[T](expected, actual, Option(normalizeFn), p) + + /* Private */ + + private[this] def diff[T]( + expected: T, + actual: T, + normalizer: Option[JsonNode => JsonNode] + ): Option[Result] = { + val actualJson = jsonString(actual) + val expectedJson = jsonString(expected) + + val actualJsonNode: JsonNode = { + val jsonNode = tryJsonNodeParse(actualJson) + normalizer match { + case Some(normalizerFn) => normalizerFn(jsonNode) + case _ => jsonNode + } + } + + val expectedJsonNode: JsonNode = tryJsonNodeParse(expectedJson) + if (actualJsonNode != expectedJsonNode) { + val result = Result( + expected = expectedJsonNode, + expectedPrettyString = mapper.writePrettyString(expectedJsonNode), + actual = actualJsonNode, + actualPrettyString = mapper.writePrettyString(actualJsonNode) + ) + Some(result) + } else None + } + + private[this] def assert[T]( + expected: T, + actual: T, + normalizer: Option[JsonNode => JsonNode], + p: PrintStream + ): Unit = { + diff(expected, actual, normalizer) match { + case Some(result) => + p.println("JSON DIFF FAILED!") + p.println(result) + throw new AssertionError(s"${JsonDiff.getClass.getName} failure\n$result") + case _ => // do nothing + } + } + + private[this] def tryJsonNodeParse(expectedJsonStr: String): JsonNode = { + try { + mapper.parse[JsonNode](expectedJsonStr) + } catch { + case NonFatal(e) => + warn(e.getMessage) + new TextNode(expectedJsonStr) + } + } + + private[this] def jsonString(receivedJson: Any): String = { + receivedJson match { + case str: String => str + case _ => mapper.writeValueAsString(receivedJson) + } + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala new file mode 100644 index 0000000000..7e73ca44d8 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala @@ -0,0 +1,636 @@ +package com.twitter.util.jackson + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.core.json.JsonWriteFeature +import com.fasterxml.jackson.core.util.{DefaultIndenter, DefaultPrettyPrinter} +import com.fasterxml.jackson.core.{JsonFactory, JsonFactoryBuilder, JsonParser, TSFBuilder} +import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream +import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper, _} +import com.fasterxml.jackson.dataformat.yaml.{YAMLFactory, YAMLFactoryBuilder} +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.scala.{ + DefaultScalaModule, + ScalaObjectMapper => JacksonScalaObjectMapper +} +import com.twitter.io.Buf +import com.twitter.util.jackson.caseclass.CaseClassJacksonModule +import com.twitter.util.jackson.serde.{DefaultSerdeModule, LongKeyDeserializers} +import com.twitter.util.validation.ScalaValidator +import java.io.{ByteArrayOutputStream, InputStream, OutputStream} +import java.nio.ByteBuffer + +object ScalaObjectMapper { + + /** The default [[ScalaValidator]] for a [[ScalaObjectMapper]] */ + private[twitter] val DefaultValidator: ScalaValidator = ScalaValidator() + + /** The default [[JsonWriteFeature.WRITE_NUMBERS_AS_STRINGS]] setting */ + private[twitter] val DefaultNumbersAsStrings: Boolean = false + + /** Framework modules need to be added 'last' so they can override existing ser/des */ + private[twitter] val DefaultJacksonModules: Seq[Module] = + Seq(DefaultScalaModule, new JavaTimeModule, LongKeyDeserializers, DefaultSerdeModule) + + /** The default [[PropertyNamingStrategy]] for a [[ScalaObjectMapper]] */ + private[twitter] val DefaultPropertyNamingStrategy: PropertyNamingStrategy = + PropertyNamingStrategy.SNAKE_CASE + + /** The default [[JsonInclude.Include]] for serialization for a [[ScalaObjectMapper]] */ + private[twitter] val DefaultSerializationInclude: JsonInclude.Include = + JsonInclude.Include.NON_ABSENT + + /** The default configuration for serialization as a `Map[SerializationFeature, Boolean]` */ + private[twitter] val DefaultSerializationConfig: Map[SerializationFeature, Boolean] = + Map( + SerializationFeature.WRITE_DATES_AS_TIMESTAMPS -> false, + SerializationFeature.WRITE_ENUMS_USING_TO_STRING -> true) + + /** The default configuration for deserialization as a `Map[DeserializationFeature, Boolean]` */ + private[twitter] val DefaultDeserializationConfig: Map[DeserializationFeature, Boolean] = + Map( + DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES -> true, + DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES -> false, + DeserializationFeature.READ_ENUMS_USING_TO_STRING -> true, + DeserializationFeature.FAIL_ON_READING_DUP_TREE_KEY -> true, + DeserializationFeature.USE_JAVA_ARRAY_FOR_JSON_ARRAY -> true /* see jackson-module-scala/issues/148 */ + ) + + /** The default for configuring additional modules on the underlying [[JacksonScalaObjectMapperType]] */ + private[twitter] val DefaultAdditionalJacksonModules: Seq[Module] = Seq.empty[Module] + + /** The default setting to enable case class validation during case class deserialization */ + private[twitter] val DefaultValidation: Boolean = true + + /** + * Returns a new [[ScalaObjectMapper]] configured with [[Builder]] defaults. + * + * @return a new [[ScalaObjectMapper]] instance. + * + * @see [[com.fasterxml.jackson.databind.InjectableValues]] + */ + def apply(): ScalaObjectMapper = + ScalaObjectMapper.builder.objectMapper + + /** + * Creates a new [[ScalaObjectMapper]] from an underlying [[JacksonScalaObjectMapperType]]. + * + * @note this mutates the underlying mapper to configure it with the [[ScalaObjectMapper]] defaults. + * + * @param underlying the [[JacksonScalaObjectMapperType]] to wrap. + * + * @return a new [[ScalaObjectMapper]] + */ + def apply(underlying: JacksonScalaObjectMapperType): ScalaObjectMapper = { + new ScalaObjectMapper( + ScalaObjectMapper.builder + .configureJacksonScalaObjectMapper(underlying) + ) + } + + /** + * Utility to create a new [[ScalaObjectMapper]] which simply wraps the given + * [[JacksonScalaObjectMapperType]]. + * + * @note the `underlying` mapper is not mutated to produce the new [[ScalaObjectMapper]] + */ + def objectMapper(underlying: JacksonScalaObjectMapperType): ScalaObjectMapper = { + val objectMapperCopy = ObjectMapperCopier.copy(underlying) + new ScalaObjectMapper(objectMapperCopy) + } + + /** + * Utility to create a new [[ScalaObjectMapper]] explicitly configured to serialize and deserialize + * YAML using the given [[JacksonScalaObjectMapperType]]. The resultant mapper [[PropertyNamingStrategy]] + * will be that configured on the `underlying` mapper. + * + * @note the `underlying` mapper is copied (not mutated) to produce the new [[ScalaObjectMapper]] + * to negotiate YAML serialization and deserialization. + */ + def yamlObjectMapper(underlying: JacksonScalaObjectMapperType): ScalaObjectMapper = + underlying.getFactory match { + case _: YAMLFactory => // correct + val objectMapperCopy = ObjectMapperCopier.copy(underlying) + new ScalaObjectMapper(objectMapperCopy) + case _ => // incorrect + throw new IllegalArgumentException( + s"The underlying mapper is not properly configured with a YAMLFactory") + } + + /** + * Utility to create a new [[ScalaObjectMapper]] explicitly configured with + * [[PropertyNamingStrategy.LOWER_CAMEL_CASE]] as a `PropertyNamingStrategy` wrapping the + * given [[JacksonScalaObjectMapperType]]. + * + * @note the `underlying` mapper is copied (not mutated) to produce the new [[ScalaObjectMapper]] + * with a [[PropertyNamingStrategy.LOWER_CAMEL_CASE]] PropertyNamingStrategy. + */ + def camelCaseObjectMapper(underlying: JacksonScalaObjectMapperType): ScalaObjectMapper = { + val objectMapperCopy = ObjectMapperCopier.copy(underlying) + objectMapperCopy.setPropertyNamingStrategy(PropertyNamingStrategy.LOWER_CAMEL_CASE) + new ScalaObjectMapper(objectMapperCopy) + } + + /** + * Utility to create a new [[ScalaObjectMapper]] explicitly configured with + * [[PropertyNamingStrategy.SNAKE_CASE]] as a `PropertyNamingStrategy` wrapping the + * given [[JacksonScalaObjectMapperType]]. + * + * @note the `underlying` mapper is copied (not mutated) to produce the new [[ScalaObjectMapper]] + * with a [[PropertyNamingStrategy.SNAKE_CASE]] PropertyNamingStrategy. + */ + def snakeCaseObjectMapper(underlying: JacksonScalaObjectMapperType): ScalaObjectMapper = { + val objectMapperCopy = ObjectMapperCopier.copy(underlying) + objectMapperCopy.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) + new ScalaObjectMapper(objectMapperCopy) + } + + /** + * + * Build a new instance of a [[ScalaObjectMapper]]. + * + * For example, + * {{{ + * ScalaObjectMapper.builder + * .withPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy) + * .withNumbersAsStrings(true) + * .withAdditionalJacksonModules(...) + * .objectMapper + * }}} + * + * or + * + * {{{ + * val builder = + * ScalaObjectMapper.builder + * .withPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy) + * .withNumbersAsStrings(true) + * .withAdditionalJacksonModules(...) + * + * val mapper = builder.objectMapper + * val camelCaseMapper = builder.camelCaseObjectMapper + * }}} + * + */ + def builder: ScalaObjectMapper.Builder = Builder() + + /** + * A Builder for creating a new [[ScalaObjectMapper]]. E.g., to build a new instance of + * a [[ScalaObjectMapper]]. + * + * For example, + * {{{ + * ScalaObjectMapper.builder + * .withPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy) + * .withNumbersAsStrings(true) + * .withAdditionalJacksonModules(...) + * .objectMapper + * }}} + * + * or + * + * {{{ + * val builder = + * ScalaObjectMapper.builder + * .withPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy) + * .withNumbersAsStrings(true) + * .withAdditionalJacksonModules(...) + * + * val mapper = builder.objectMapper + * val camelCaseMapper = builder.camelCaseObjectMapper + * }}} + */ + case class Builder private[jackson] ( + propertyNamingStrategy: PropertyNamingStrategy = DefaultPropertyNamingStrategy, + numbersAsStrings: Boolean = DefaultNumbersAsStrings, + serializationInclude: Include = DefaultSerializationInclude, + serializationConfig: Map[SerializationFeature, Boolean] = DefaultSerializationConfig, + deserializationConfig: Map[DeserializationFeature, Boolean] = DefaultDeserializationConfig, + defaultJacksonModules: Seq[Module] = DefaultJacksonModules, + validator: Option[ScalaValidator] = Some(DefaultValidator), + additionalJacksonModules: Seq[Module] = DefaultAdditionalJacksonModules, + additionalMapperConfigurationFns: Seq[JacksonObjectMapper => Unit] = Seq.empty, + validation: Boolean = DefaultValidation) { + + /* Public */ + + /** Create a new [[ScalaObjectMapper]] from this [[Builder]]. */ + final def objectMapper: ScalaObjectMapper = + new ScalaObjectMapper(jacksonScalaObjectMapper) + + /** Create a new [[ScalaObjectMapper]] from this [[Builder]] using the given [[JsonFactory]]. */ + final def objectMapper[F <: JsonFactory](factory: F): ScalaObjectMapper = + new ScalaObjectMapper(jacksonScalaObjectMapper(factory)) + + /** + * Create a new [[ScalaObjectMapper]] explicitly configured to serialize and deserialize + * YAML from this [[Builder]]. + * + * @note the used [[PropertyNamingStrategy]] is defined by the current [[Builder]] configuration. + */ + final def yamlObjectMapper: ScalaObjectMapper = + new ScalaObjectMapper( + configureJacksonScalaObjectMapper(new YAMLFactoryBuilder(new YAMLFactory()))) + + /** + * Creates a new [[ScalaObjectMapper]] explicitly configured with + * [[PropertyNamingStrategy.LOWER_CAMEL_CASE]] as a `PropertyNamingStrategy`. + */ + final def camelCaseObjectMapper: ScalaObjectMapper = + ScalaObjectMapper.camelCaseObjectMapper(jacksonScalaObjectMapper) + + /** + * Creates a new [[ScalaObjectMapper]] explicitly configured with + * [[PropertyNamingStrategy.SNAKE_CASE]] as a `PropertyNamingStrategy`. + */ + final def snakeCaseObjectMapper: ScalaObjectMapper = + ScalaObjectMapper.snakeCaseObjectMapper(jacksonScalaObjectMapper) + + /* Builder Methods */ + + /** + * Configure a [[PropertyNamingStrategy]] for this [[Builder]]. + * @note the default is [[PropertyNamingStrategy.SNAKE_CASE]] + * @see [[ScalaObjectMapper.DefaultPropertyNamingStrategy]] + */ + final def withPropertyNamingStrategy(propertyNamingStrategy: PropertyNamingStrategy): Builder = + this.copy(propertyNamingStrategy = propertyNamingStrategy) + + /** + * Enable the [[JsonWriteFeature.WRITE_NUMBERS_AS_STRINGS]] for this [[Builder]]. + * @note the default is false. + */ + final def withNumbersAsStrings(numbersAsStrings: Boolean): Builder = + this.copy(numbersAsStrings = numbersAsStrings) + + /** + * Configure a [[JsonInclude.Include]] for serialization for this [[Builder]]. + * @note the default is [[JsonInclude.Include.NON_ABSENT]] + * @see [[ScalaObjectMapper.DefaultSerializationInclude]] + */ + final def withSerializationInclude(serializationInclude: Include): Builder = + this.copy(serializationInclude = serializationInclude) + + /** + * Set the serialization configuration for this [[Builder]] as a `Map` of `SerializationFeature` + * to `Boolean` (enabled). + * @note the default is described by [[ScalaObjectMapper.DefaultSerializationConfig]]. + * @see [[ScalaObjectMapper.DefaultSerializationConfig]] + */ + final def withSerializationConfig( + serializationConfig: Map[SerializationFeature, Boolean] + ): Builder = + this.copy(serializationConfig = serializationConfig) + + /** + * Set the deserialization configuration for this [[Builder]] as a `Map` of `DeserializationFeature` + * to `Boolean` (enabled). + * @note this overwrites the default deserialization configuration of this [[Builder]]. + * @note the default is described by [[ScalaObjectMapper.DefaultDeserializationConfig]]. + * @see [[ScalaObjectMapper.DefaultDeserializationConfig]] + */ + final def withDeserializationConfig( + deserializationConfig: Map[DeserializationFeature, Boolean] + ): Builder = + this.copy(deserializationConfig = deserializationConfig) + + /** + * Configure a [[ScalaValidator]] for this [[Builder]] + * @see [[ScalaObjectMapper.DefaultValidator]] + * + * @note If you pass `withNoValidation` to the builder all case class validations will be + * bypassed, regardless of the `withValidator` configuration. + */ + final def withValidator(validator: ScalaValidator): Builder = + this.copy(validator = Some(validator)) + + /** + * Configure the list of additional Jackson [[Module]]s for this [[Builder]]. + * @note this will overwrite (not append) the list additional Jackson [[Module]]s of this [[Builder]]. + */ + final def withAdditionalJacksonModules(additionalJacksonModules: Seq[Module]): Builder = + this.copy(additionalJacksonModules = additionalJacksonModules) + + /** + * Configure additional [[JacksonObjectMapper]] functionality for the underlying mapper of this [[Builder]]. + * @note this will overwrite any previously set function. + */ + final def withAdditionalMapperConfigurationFn(mapperFn: JacksonObjectMapper => Unit): Builder = + this + .copy(additionalMapperConfigurationFns = this.additionalMapperConfigurationFns :+ mapperFn) + + /** Method to allow changing of the default Jackson Modules for use from the `ScalaObjectMapperModule` */ + private[twitter] final def withDefaultJacksonModules( + defaultJacksonModules: Seq[Module] + ): Builder = + this.copy(defaultJacksonModules = defaultJacksonModules) + + /** + * Disable case class validation during case class deserialization + * + * @see [[ScalaObjectMapper.DefaultValidation]] + * @note If you pass `withNoValidation` to the builder all case class validations will be + * bypassed, regardless of the `withValidator` configuration. + */ + final def withNoValidation: Builder = + this.copy(validation = false) + + /* Private */ + + private[this] def defaultMapperConfiguration(mapper: JacksonObjectMapper): Unit = { + /* Serialization Config */ + mapper.setDefaultPropertyInclusion( + JsonInclude.Value.construct(serializationInclude, serializationInclude)) + mapper + .configOverride(classOf[Option[_]]) + .setIncludeAsProperty(JsonInclude.Value.construct(serializationInclude, Include.ALWAYS)) + for ((feature, state) <- serializationConfig) { + mapper.configure(feature, state) + } + + /* Deserialization Config */ + for ((feature, state) <- deserializationConfig) { + mapper.configure(feature, state) + } + } + + /** Order is important: default + case class module + any additional */ + private[this] def jacksonModules: Seq[Module] = { + this.defaultJacksonModules ++ + Seq(new CaseClassJacksonModule(if (this.validation) this.validator else None)) ++ + this.additionalJacksonModules + } + + private[this] final def jacksonScalaObjectMapper: JacksonScalaObjectMapperType = + configureJacksonScalaObjectMapper(new JsonFactoryBuilder) + + private[this] final def jacksonScalaObjectMapper[F <: JsonFactory]( + jsonFactory: F + ): JacksonScalaObjectMapperType = configureJacksonScalaObjectMapper(jsonFactory) + + private[this] final def configureJacksonScalaObjectMapper[ + F <: JsonFactory, + B <: TSFBuilder[F, B] + ]( + builder: TSFBuilder[F, B] + ): JacksonScalaObjectMapperType = configureJacksonScalaObjectMapper(builder.build()) + + private[jackson] final def configureJacksonScalaObjectMapper( + factory: JsonFactory + ): JacksonScalaObjectMapperType = + configureJacksonScalaObjectMapper( + new JacksonObjectMapper(factory) with JacksonScalaObjectMapper) + + private[jackson] final def configureJacksonScalaObjectMapper( + underlying: JacksonScalaObjectMapperType + ): JacksonScalaObjectMapperType = { + if (this.numbersAsStrings) { + underlying.enable(JsonWriteFeature.WRITE_NUMBERS_AS_STRINGS.mappedFeature()) + } + + this.defaultMapperConfiguration(underlying) + this.additionalMapperConfigurationFns.foreach(_(underlying)) + + underlying.setPropertyNamingStrategy(this.propertyNamingStrategy) + this.jacksonModules.foreach(underlying.registerModule) + + underlying + } + } +} + +private[jackson] object ArrayElementsOnNewLinesPrettyPrinter extends DefaultPrettyPrinter { + _arrayIndenter = DefaultIndenter.SYSTEM_LINEFEED_INSTANCE + override def createInstance(): DefaultPrettyPrinter = this +} + +/** + * A thin wrapper over a [[https://github.com/FasterXML/jackson-module-scala jackson-module-scala]] + * [[com.fasterxml.jackson.module.scala.ScalaObjectMapper]] + * + * @note this API is inspired by the [[https://github.com/codahale/jerkson Jerkson]] + * [[https://github.com/codahale/jerkson/blob/master/src/main/scala/com/codahale/jerkson/Parser.scala Parser]] + * + * @param underlying a configured [[JacksonScalaObjectMapperType]] + */ +class ScalaObjectMapper(val underlying: JacksonScalaObjectMapperType) { + assert(underlying != null, "Underlying ScalaObjectMapper cannot be null.") + + /** + * Constructed [[ObjectWriter]] that will serialize objects using specified pretty printer + * for indentation (or if null, no pretty printer). + */ + lazy val prettyObjectMapper: ObjectWriter = + underlying.writer(ArrayElementsOnNewLinesPrettyPrinter) + + /** Returns the currently configured [[PropertyNamingStrategy]] */ + def propertyNamingStrategy: PropertyNamingStrategy = + underlying.getPropertyNamingStrategy + + /** + * Factory method for constructing a [[com.fasterxml.jackson.databind.ObjectReader]] that will + * read or update instances of specified type, `T`. + * @tparam T the type for which to create a [[com.fasterxml.jackson.databind.ObjectReader]] + * @return the created [[com.fasterxml.jackson.databind.ObjectReader]]. + */ + def reader[T: Manifest]: ObjectReader = + underlying.readerFor[T] + + /** Read a value from a [[Buf]] into a type `T`. */ + def parse[T: Manifest](buf: Buf): T = + parse[T](Buf.ByteBuffer.Shared.extract(buf)) + + /** Read a value from a [[ByteBuffer]] into a type `T`. */ + def parse[T: Manifest](byteBuffer: ByteBuffer): T = { + val is = new ByteBufferBackedInputStream(byteBuffer) + underlying.readValue[T](is) + } + + /** Convert from a [[JsonNode]] into a type `T`. */ + def parse[T: Manifest](jsonNode: JsonNode): T = + convert[T](jsonNode) + + /** Read a value from an [[InputStream]] (caller is responsible for closing the stream) into a type `T`. */ + def parse[T: Manifest](inputStream: InputStream): T = + underlying.readValue[T](inputStream) + + /** Read a value from an Array[Byte] into a type `T`. */ + def parse[T: Manifest](bytes: Array[Byte]): T = + underlying.readValue[T](bytes) + + /** Read a value from a String into a type `T`. */ + def parse[T: Manifest](string: String): T = + underlying.readValue[T](string) + + /** Read a value from a [[JsonParser]] into a type `T`. */ + def parse[T: Manifest](jsonParser: JsonParser): T = + underlying.readValue[T](jsonParser) + + /** + * Convenience method for doing two-step conversion from given value, into an instance of given + * value type, [[JavaType]] if (but only if!) conversion is needed. If given value is already of + * requested type, the value is returned as is. + * + * This method is functionally similar to first serializing a given value into JSON, and then + * binding JSON data into a value of the given type, but should be more efficient since full + * serialization does not (need to) occur. However, the same converters (serializers, + * deserializers) will be used for data binding, meaning the same object mapper configuration + * works. + * + * Note: it is possible that in some cases behavior does differ from full + * serialize-then-deserialize cycle. It is not guaranteed, however, that the behavior is 100% + * the same -- the goal is just to allow efficient value conversions for structurally compatible + * Objects, according to standard Jackson configuration. + * + * Further note that this functionality is not designed to support "advanced" use cases, such as + * conversion of polymorphic values, or cases where Object Identity is used. + * + * @param from value from which to convert. + * @param toValueType type to be converted into. + * @return a new instance of type [[JavaType]] converted from the given [[Any]] type. + */ + def convert(from: Any, toValueType: JavaType): AnyRef = { + try { + underlying.convertValue(from, toValueType) + } catch { + case e: IllegalArgumentException if e.getCause != null => + throw e.getCause + } + } + + /** + * Convenience method for doing two-step conversion from a given value, into an instance of a given + * type, `T`. This is functionality equivalent to first serializing the given value into JSON, + * then binding JSON data into a value of the given type, but may be executed without fully + * serializing into JSON. The same converters (serializers, deserializers) will be used for + * data binding, meaning the same object mapper configuration works. + * + * Note: when a [[com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException]] + * is thrown inside of the the `ObjectMapper#convertValue` method, Jackson wraps the exception + * inside of an [[IllegalArgumentException]]. As such we unwrap to restore the original + * exception here. The wrapping occurs because the [[com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException]] + * is a sub-type of [[java.io.IOException]] (through extension of [[com.fasterxml.jackson.databind.JsonMappingException]] + * --> [[com.fasterxml.jackson.core.JsonProcessingException]] --> [[java.io.IOException]]. + * + * @param any the value to be converted. + * @tparam T the type to which to be converted. + * @return a new instance of type `T` converted from the given [[Any]] type. + * + * @see [[https://github.com/FasterXML/jackson-databind/blob/d70b9e65c5e089094ec7583fa6a38b2f484a96da/src/main/java/com/fasterxml/jackson/databind/ObjectMapper.java#L2167]] + */ + def convert[T: Manifest](any: Any): T = { + try { + underlying.convertValue[T](any) + } catch { + case e: IllegalArgumentException if e.getCause != null => + throw e.getCause + } + } + + /** + * Method that can be used to serialize any value as JSON output, using the output stream + * provided (using an encoding of [[com.fasterxml.jackson.core.JsonEncoding#UTF8]]. + * + * Note: this method does not close the underlying stream explicitly here; however, the + * [[com.fasterxml.jackson.core.JsonFactory]] this mapper uses may choose to close the stream + * depending on its settings (by default, it will try to close it when + * [[com.fasterxml.jackson.core.JsonGenerator]] constructed is closed). + * + * @param any the value to serialize. + * @param outputStream the [[OutputStream]] to which to serialize. + */ + def writeValue(any: Any, outputStream: OutputStream): Unit = + underlying.writeValue(outputStream, any) + + /** + * Method that can be used to serialize any value as a `Array[Byte]`. Functionally equivalent + * to calling [[JacksonObjectMapper#writeValue(Writer,Object)]] with a [[java.io.ByteArrayOutputStream]] and + * getting bytes, but more efficient. Encoding used will be UTF-8. + * + * @param any the value to serialize. + * @return the `Array[Byte]` representing the serialized value. + */ + def writeValueAsBytes(any: Any): Array[Byte] = + underlying.writeValueAsBytes(any) + + /** + * Method that can be used to serialize any value as a String. Functionally equivalent to calling + * [[JacksonObjectMapper#writeValue(Writer,Object)]] with a [[java.io.StringWriter]] + * and constructing String, but more efficient. + * + * @param any the value to serialize. + * @return the String representing the serialized value. + */ + def writeValueAsString(any: Any): String = + underlying.writeValueAsString(any) + + /** + * Method that can be used to serialize any value as a pretty printed String. Uses the + * [[prettyObjectMapper]] and calls [[writeValueAsString(any: Any)]] on the given value. + * + * @param any the value to serialize. + * @return the pretty printed String representing the serialized value. + * + * @see [[prettyObjectMapper]] + * @see [[writeValueAsString(any: Any)]] + */ + def writePrettyString(any: Any): String = any match { + case str: String => + val jsonNode = underlying.readValue[JsonNode](str) + prettyObjectMapper.writeValueAsString(jsonNode) + case _ => + prettyObjectMapper.writeValueAsString(any) + } + + /** + * Method that can be used to serialize any value as a [[Buf]]. Functionally equivalent + * to calling [[writeValueAsBytes(any: Any)]] and then wrapping the results in an + * "owned" [[Buf.ByteArray]]. + * + * @param any the value to serialize. + * @return the [[Buf.ByteArray.Owned]] representing the serialized value. + * + * @see [[writeValueAsBytes(any: Any)]] + * @see [[Buf.ByteArray.Owned]] + */ + def writeValueAsBuf(any: Any): Buf = + Buf.ByteArray.Owned(underlying.writeValueAsBytes(any)) + + /** + * Convenience method for doing the multi-step process of serializing a `Map[String, String]` + * to a [[Buf]]. + * + * @param stringMap the `Map[String, String]` to convert. + * @return the [[Buf.ByteArray.Owned]] representing the serialized value. + */ + // optimized + def writeStringMapAsBuf(stringMap: Map[String, String]): Buf = { + val os = new ByteArrayOutputStream() + + val jsonGenerator = underlying.getFactory.createGenerator(os) + try { + jsonGenerator.writeStartObject() + for ((key, value) <- stringMap) { + jsonGenerator.writeStringField(key, value) + } + jsonGenerator.writeEndObject() + jsonGenerator.flush() + + Buf.ByteArray.Owned(os.toByteArray) + } finally { + jsonGenerator.close() + } + } + + /** + * Method for registering a module that can extend functionality provided by this mapper; for + * example, by adding providers for custom serializers and deserializers. + * + * @note this mutates the [[underlying]] [[com.fasterxml.jackson.databind.ObjectMapper]] of + * this [[ScalaObjectMapper]]. + * + * @param module [[com.fasterxml.jackson.databind.Module]] to register. + */ + def registerModule(module: Module): JacksonObjectMapper = + underlying.registerModule(module) +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD new file mode 100644 index 0000000000..356259ed5b --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD @@ -0,0 +1,31 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-jackson-caseclass", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + "3rdparty/jvm/org/json4s:json4s-core", + "3rdparty/jvm/org/scala-lang/modules:scala-collection-compat", + "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation", + "util/util-jackson/src/main/scala/com/fasterxml/jackson/databind", + "util/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions", + "util/util-reflect/src/main/scala/com/twitter/util/reflect", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + "util/util-validator/src/main/scala/com/twitter/util/validation", + "util/util-validator/src/main/scala/com/twitter/util/validation/conversions", + ], + exports = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/org/json4s:json4s-core", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + ], +) diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassBeanProperty.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassBeanProperty.scala new file mode 100644 index 0000000000..867af429b4 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassBeanProperty.scala @@ -0,0 +1,8 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.annotation.JacksonInject +import com.fasterxml.jackson.databind.BeanProperty + +private[twitter] case class CaseClassBeanProperty( + valueObj: Option[JacksonInject.Value], + property: BeanProperty) diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassDeserializer.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassDeserializer.scala new file mode 100644 index 0000000000..c80ebec7b3 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassDeserializer.scala @@ -0,0 +1,834 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.annotation.{JsonCreator, JsonIgnoreProperties} +import com.fasterxml.jackson.core.{ + JsonFactory, + JsonParseException, + JsonParser, + JsonProcessingException, + JsonToken +} +import com.fasterxml.jackson.databind._ +import com.fasterxml.jackson.databind.annotation.JsonNaming +import com.fasterxml.jackson.databind.exc.{ + InvalidDefinitionException, + InvalidFormatException, + MismatchedInputException, + UnrecognizedPropertyException +} +import com.fasterxml.jackson.databind.introspect._ +import com.fasterxml.jackson.databind.util.SimpleBeanPropertyDefinition +import com.twitter.util.WrappedValue +import com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException.ValidationError +import com.twitter.util.jackson.caseclass.exceptions.{ + CaseClassFieldMappingException, + CaseClassMappingException, + InjectableValuesException, + _ +} +import com.twitter.util.logging.Logging +import com.twitter.util.reflect.{Annotations, Types => ReflectionTypes} +import com.twitter.util.validation.ScalaValidator +import com.twitter.util.validation.conversions.ConstraintViolationOps._ +import com.twitter.util.validation.conversions.PathOps._ +import com.twitter.util.validation.metadata.{ExecutableDescriptor, MethodDescriptor} +import jakarta.validation.{ConstraintViolation, Payload} +import java.lang.annotation.Annotation +import java.lang.reflect.{ + Constructor, + Executable, + InvocationTargetException, + Method, + Parameter, + ParameterizedType, + Type, + TypeVariable +} +import org.json4s.reflect.{ClassDescriptor, ConstructorDescriptor, Reflector, ScalaType} +import scala.collection.mutable +import scala.jdk.CollectionConverters._ +import scala.util.control.NonFatal + +private object CaseClassDeserializer { + // For reporting an InvalidDefinitionException + val EmptyJsonParser: JsonParser = new JsonFactory().createParser("") + + /* For supporting JsonCreator */ + case class CaseClassCreator( + executable: Executable, + propertyDefinitions: Array[PropertyDefinition], + executableValidationDescription: Option[ExecutableDescriptor], + executableValidationMethodDescriptions: Option[Array[MethodDescriptor]]) + + /* Holder for a fully specified JavaType with generics and a Jackson BeanPropertyDefinition */ + case class PropertyDefinition( + javaType: JavaType, + scalaType: ScalaType, + beanPropertyDefinition: BeanPropertyDefinition) + + /* Holder of constructor arg to ScalaType */ + case class ConstructorParam(name: String, scalaType: ScalaType) + + private[this] def applyPropertyNamingStrategy( + config: DeserializationConfig, + fieldName: String + ): String = + config.getPropertyNamingStrategy.nameForField(config, null, fieldName) + + // Validate `Constraint` annotations for all fields. + def executeFieldValidations( + validatorOption: Option[ScalaValidator], + executableDescriptorOption: Option[ExecutableDescriptor], + config: DeserializationConfig, + constructorValues: Array[Object], + fields: Array[CaseClassField] + ): Seq[CaseClassFieldMappingException] = (validatorOption, executableDescriptorOption) match { + case (Some(validator), Some(executableDescriptor)) => + val violations: Set[ConstraintViolation[Any]] = + validator.forExecutables + .validateExecutableParameters[Any]( + executableDescriptor, + constructorValues.asInstanceOf[Array[Any]], + fields.map(_.name) + ) + if (violations.nonEmpty) { + val results = mutable.ListBuffer[CaseClassFieldMappingException]() + val iterator = violations.iterator + while (iterator.hasNext) { + val violation: ConstraintViolation[Any] = iterator.next() + results.append( + CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.leaf( + applyPropertyNamingStrategy(config, violation.getPropertyPath.getLeafNode.toString) + ), + CaseClassFieldMappingException.Reason( + message = violation.getMessage, + detail = ValidationError( + violation, + CaseClassFieldMappingException.ValidationError.Field, + violation.getDynamicPayload(classOf[Payload]) + ) + ) + ) + ) + } + results.toSeq + } else Seq.empty[CaseClassFieldMappingException] + case _ => + Seq.empty[CaseClassFieldMappingException] + } + + // We do not want to "leak" method names, so we only want to return the leaf node + // if the ConstraintViolation PropertyPath size is greater than 1 element. + private[this] def getMethodValidationViolationPropertyPath( + config: DeserializationConfig, + violation: ConstraintViolation[_] + ): CaseClassFieldMappingException.PropertyPath = { + val propertyPath = violation.getPropertyPath + val iterator = propertyPath.iterator.asScala + if (iterator.hasNext) { + iterator.next() // move the iterator + if (iterator.hasNext) { // has another element, return the leaf + CaseClassFieldMappingException.PropertyPath.leaf( + applyPropertyNamingStrategy(config, propertyPath.getLeafNode.toString) + ) + } else { // does not have another element, return empty + CaseClassFieldMappingException.PropertyPath.Empty + } + } else { + CaseClassFieldMappingException.PropertyPath.Empty + } + } + + // Validate all methods annotated with `MethodValidation` defined in the deserialized case class. + // This is called after the case class is created. + def executeMethodValidations( + validatorOption: Option[ScalaValidator], + methodDescriptorsOption: Option[Array[MethodDescriptor]], + config: DeserializationConfig, + obj: Any + ): Seq[CaseClassFieldMappingException] = (validatorOption, methodDescriptorsOption) match { + case (Some(validator), Some(methodDescriptors)) => + try { + validator.forExecutables + .validateMethods[Any](methodDescriptors, obj).map { violation: ConstraintViolation[Any] => + CaseClassFieldMappingException( + getMethodValidationViolationPropertyPath(config, violation), + CaseClassFieldMappingException.Reason( + message = violation.getMessage, + detail = ValidationError( + violation, + CaseClassFieldMappingException.ValidationError.Method, + violation.getDynamicPayload(classOf[Payload])) + ) + ) + }.toSeq + } catch { + case NonFatal(e: InvocationTargetException) if e.getCause != null => + // propagate the InvocationTargetException + throw e.getCause + } + case _ => + Seq.empty[CaseClassFieldMappingException] + } +} + +/** + * Custom case class deserializer which overcomes limitations in jackson-scala-module. + * + * Our improvements: + * - Throws a [[JsonMappingException]] when non-`Option` fields are missing in the incoming JSON. + * - Does not allow for case class constructor fields to ever be constructed with a `null` reference. + * Any field which is marked to not be read from the incoming JSON must either be injected with a + * configured InjectableValues implementation or have a default value which can be supplied when + * the deserializer constructs the case class. Otherwise an error will be returned. + * - Use default values when fields are missing in the incoming JSON. + * - Properly deserialize a `Seq[Long]` (see https://github.com/FasterXML/jackson-module-scala/issues/62) + * - Support "wrapped values" using [[WrappedValue]]. + * - Support for field and method level validations. + * + * The following Jackson annotations which affect deserialization are not explicitly supported by + * this deserializer (note this list may not be exhaustive): + * - @JsonPOJOBuilder + * - @JsonAlias + * - @JsonSetter + * - @JsonAnySetter + * - @JsonTypeName + * - @JsonUnwrapped + * - @JsonManagedReference + * - @JsonBackReference + * - @JsonIdentityInfo + * - @JsonTypeIdResolver + * + * For many, this is because the behavior of these annotations is ambiguous when it comes to application on + * constructor arguments of a Scala case class during deserialization. + * + * @see [[https://github.com/FasterXML/jackson-annotations/wiki/Jackson-Annotations]] + * + * @note This class is inspired by Jerkson's CaseClassDeserializer which can be found here: + * [[https://github.com/codahale/jerkson/blob/master/src/main/scala/com/codahale/jerkson/deser/CaseClassDeserializer.scala]] + */ +/* exposed for testing */ +private[jackson] class CaseClassDeserializer( + javaType: JavaType, + config: DeserializationConfig, + beanDescription: BeanDescription, + validator: Option[ScalaValidator]) + extends JsonDeserializer[AnyRef] + with Logging { + import CaseClassDeserializer._ + + private[this] val clazz: Class[_] = javaType.getRawClass + // we explicitly do not read a mix-in for a primitive type + private[this] val mixinClazz: Option[Class[_]] = + Option(config.findMixInClassFor(clazz)) + .flatMap(m => if (m.isPrimitive) None else Some(m)) + private[this] val clazzDescriptor: ClassDescriptor = + Reflector.describe(clazz).asInstanceOf[ClassDescriptor] + private[this] val clazzAnnotations: Array[Annotation] = + mixinClazz.map(_.getAnnotations) match { + case Some(mixinAnnotations) => + clazz.getAnnotations ++ mixinAnnotations + case _ => + clazz.getAnnotations + } + + private[this] val caseClazzCreator: CaseClassCreator = { + val fromCompanion: Option[AnnotatedMethod] = + beanDescription.getFactoryMethods.asScala.find(_.hasAnnotation(classOf[JsonCreator])) + val fromClazz: Option[AnnotatedConstructor] = + beanDescription.getConstructors.asScala.find(_.hasAnnotation(classOf[JsonCreator])) + + fromCompanion match { + case Some(jsonCreatorAnnotatedMethod) => + CaseClassCreator( + jsonCreatorAnnotatedMethod.getAnnotated, + getBeanPropertyDefinitions( + jsonCreatorAnnotatedMethod.getAnnotated.getParameters, + jsonCreatorAnnotatedMethod, + fromCompanion = true), + validator.map(_.describeExecutable(jsonCreatorAnnotatedMethod.getAnnotated, mixinClazz)), + validator.map(_.describeMethods(clazz)) + ) + case _ => + fromClazz match { + case Some(jsonCreatorAnnotatedConstructor) => + CaseClassCreator( + jsonCreatorAnnotatedConstructor.getAnnotated, + getBeanPropertyDefinitions( + jsonCreatorAnnotatedConstructor.getAnnotated.getParameters, + jsonCreatorAnnotatedConstructor), + validator.map( + _.describeExecutable(jsonCreatorAnnotatedConstructor.getAnnotated, mixinClazz)), + validator.map(_.describeMethods(clazz)) + ) + case _ => + // try to use what Jackson thinks is the default -- however Jackson does not + // seem to correctly track an empty default constructor for case classes, nor + // multiple un-annotated and we have no way to pick a proper constructor so we bail + val constructor = Option(beanDescription.getClassInfo.getDefaultConstructor) match { + case Some(ctor) => ctor + case _ => + val constructors = beanDescription.getBeanClass.getConstructors + assert(constructors.size == 1, "Multiple case class constructors not supported") + annotateConstructor(constructors.head, clazzAnnotations) + } + CaseClassCreator( + constructor.getAnnotated, + getBeanPropertyDefinitions(constructor.getAnnotated.getParameters, constructor), + validator.map(_.describeExecutable(constructor.getAnnotated, mixinClazz)), + validator.map(_.describeMethods(clazz)) + ) + } + } + } + // nested class inside another class is not supported, e.g., we do not support + // use of creators for non-static inner classes, + assert(!beanDescription.isNonStaticInnerClass, "Non-static inner case classes are not supported.") + + // all class annotations -- including inherited annotations for fields + private[this] val allAnnotations: scala.collection.Map[String, Array[Annotation]] = { + // use the carried scalaType in order to find the appropriate constructor by the + // unresolved scalaTypes -- e.g., before we resolve generic types to their bound type + Annotations.findAnnotations( + clazz, + caseClazzCreator.propertyDefinitions + .map(_.scalaType.erasure) + ) + } + + // support for reading annotations from Jackson Mix-ins + // see: https://github.com/FasterXML/jackson-docs/wiki/JacksonMixInAnnotations + private[this] val allMixinAnnotations: scala.collection.Map[String, Array[Annotation]] = + mixinClazz match { + case Some(clazz) => + clazz.getDeclaredMethods.map(m => m.getName -> m.getAnnotations).toMap + case _ => + scala.collection.Map.empty[String, Array[Annotation]] + } + + // optimized lookup of bean property definition annotations + private[this] def getBeanPropertyDefinitionAnnotations( + beanPropertyDefinition: BeanPropertyDefinition + ): Array[Annotation] = { + if (beanPropertyDefinition.getPrimaryMember.getAllAnnotations.size() > 0) { + beanPropertyDefinition.getPrimaryMember.getAllAnnotations.annotations().asScala.toArray + } else Array.empty[Annotation] + } + + // Field name to list of parsed annotations. Jackson only tracks JacksonAnnotations + // in the BeanPropertyDefinition AnnotatedMembers and we want to track all class annotations by field. + // Annotations are keyed by parameter name because the logic collapses annotations from the + // inheritance hierarchy where the discriminator is member name. + // note: Prefer while loop over Scala for loop for better performance. The scala for loop + // performance is optimized in 2.13.0 if we enable scalac: https://github.com/scala/bug/issues/1338 + private[this] val fieldAnnotations: scala.collection.Map[String, Array[Annotation]] = { + val fieldBeanPropertyDefinitions: Array[BeanPropertyDefinition] = + caseClazzCreator.propertyDefinitions.map(_.beanPropertyDefinition) + val collectedFieldAnnotations = mutable.HashMap[String, Array[Annotation]]() + var index = 0 + while (index < fieldBeanPropertyDefinitions.length) { + val fieldBeanPropertyDefinition = fieldBeanPropertyDefinitions(index) + val fieldName = fieldBeanPropertyDefinition.getInternalName + // in many cases we will have the field annotations in the `allAnnotations` Map which + // scans for annotations from the class definition being deserialized, however in some cases + // we are dealing with an annotated field from a static or secondary constructor + // and thus the annotations may not exist in the `allAnnotations` Map so we default to + // any carried bean property definition annotations which includes annotation information + // for any static or secondary constructor. + val annotations = + allAnnotations.getOrElse( + fieldName, + getBeanPropertyDefinitionAnnotations(fieldBeanPropertyDefinition) + ) ++ allMixinAnnotations.getOrElse(fieldName, Array.empty) + if (annotations.nonEmpty) collectedFieldAnnotations.put(fieldName, annotations) + index += 1 + } + + collectedFieldAnnotations + } + + /* exposed for testing */ + private[jackson] val fields: Array[CaseClassField] = + CaseClassField.createFields( + clazz, + caseClazzCreator.executable, + clazzDescriptor, + caseClazzCreator.propertyDefinitions, + fieldAnnotations, + propertyNamingStrategy + ) + + private[this] lazy val numConstructorArgs = fields.length + private[this] lazy val isWrapperClass = classOf[WrappedValue[_]].isAssignableFrom(clazz) + private[this] lazy val firstFieldName = fields.head.name + + /* Public */ + + override def isCachable: Boolean = true + + override def deserialize(jsonParser: JsonParser, context: DeserializationContext): Object = { + if (isWrapperClass) + deserializeWrapperClass(jsonParser, context) + else + deserializeNonWrapperClass(jsonParser, context) + } + + /* Private */ + + private[this] def deserializeWrapperClass( + jsonParser: JsonParser, + context: DeserializationContext + ): Object = { + if (jsonParser.getCurrentToken.isStructStart) { + try { + context.handleUnexpectedToken( + clazz, + jsonParser.currentToken(), + jsonParser, + "Unable to deserialize wrapped value from a json object" + ) + } catch { + case NonFatal(e) => + // wrap in a JsonMappingException + throw JsonMappingException.from(jsonParser, e.getMessage) + } + } + + val jsonNode = context.getNodeFactory.objectNode() + jsonNode.put(firstFieldName, jsonParser.getText) + deserialize(jsonParser, context, jsonNode) + } + + private[this] def deserializeNonWrapperClass( + jsonParser: JsonParser, + context: DeserializationContext + ): Object = { + incrementParserToFirstField(jsonParser, context) + val jsonNode = jsonParser.readValueAsTree[JsonNode] + deserialize(jsonParser, context, jsonNode) + } + + private[this] def deserialize( + jsonParser: JsonParser, + context: DeserializationContext, + jsonNode: JsonNode + ): Object = { + val jsonFieldNames: Seq[String] = jsonNode.fieldNames().asScala.toSeq + val caseClassFieldNames: Seq[String] = fields.map(_.name) + + handleUnknownFields(jsonParser, context, jsonFieldNames, caseClassFieldNames) + val (values, parseErrors) = parseConstructorValues(jsonParser, context, jsonNode) + + // run field validations + val validationErrors = executeFieldValidations( + validator, + caseClazzCreator.executableValidationDescription, + config, + values, + fields) + + val errors = parseErrors ++ validationErrors + if (errors.nonEmpty) throw CaseClassMappingException(errors.toSet) + + newInstance(values) + } + + /** Return all "unknown" properties sent in the incoming JSON */ + private[this] def unknownProperties( + context: DeserializationContext, + jsonFieldNames: Seq[String], + caseClassFieldNames: Seq[String] + ): Seq[String] = { + // if there is a JsonIgnoreProperties annotation on the class, it should prevail + val nonIgnoredFields: Seq[String] = + Annotations.findAnnotation[JsonIgnoreProperties](clazzAnnotations) match { + case Some(annotation) if !annotation.ignoreUnknown() => + // has a JsonIgnoreProperties annotation and is configured to NOT ignore unknown properties + val annotationIgnoredFields: Seq[String] = annotation.value() + // only non-ignored json fields should be considered + jsonFieldNames.diff(annotationIgnoredFields) + case None if context.isEnabled(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) => + // no annotation but feature is configured, thus all json fields should be considered + jsonFieldNames + case _ => + Seq.empty[String] // every field is ignorable + } + + // if we have any non ignored field, return the difference + if (nonIgnoredFields.nonEmpty) { + nonIgnoredFields.diff(caseClassFieldNames) + } else Seq.empty[String] + } + + /** Throws a [[CaseClassMappingException]] with a set of [[CaseClassFieldMappingException]] per unknown field */ + private[this] def handleUnknownFields( + jsonParser: JsonParser, + context: DeserializationContext, + jsonFieldNames: Seq[String], + caseClassFieldNames: Seq[String] + ): Unit = { + // handles more or less incoming fields in the JSON than are defined in the case class + val unknownFields: Seq[String] = unknownProperties(context, jsonFieldNames, caseClassFieldNames) + if (unknownFields.nonEmpty) { + val errors = unknownFields.map { field => + CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.Empty, + CaseClassFieldMappingException.Reason( + message = UnrecognizedPropertyException + .from( + jsonParser, + clazz, + field, + caseClassFieldNames.map(_.asInstanceOf[Object]).asJavaCollection + ).getMessage + ) + ) + } + throw CaseClassMappingException(errors.toSet) + } + } + + private[this] def incrementParserToFirstField( + jsonParser: JsonParser, + context: DeserializationContext + ): Unit = { + if (jsonParser.getCurrentToken == JsonToken.START_OBJECT) { + jsonParser.nextToken() + } + if (jsonParser.getCurrentToken != JsonToken.FIELD_NAME && + jsonParser.getCurrentToken != JsonToken.END_OBJECT) { + try { + context.handleUnexpectedToken(clazz, jsonParser) + } catch { + case NonFatal(e) => + e match { + case j: JsonProcessingException => + // don't include source info since it's often blank. + j.clearLocation() + throw new JsonParseException(jsonParser, j.getMessage) + case _ => + throw new JsonParseException(jsonParser, e.getMessage) + } + } + } + } + + // note: Prefer while loop over Scala for loop for better performance. The scala for loop + // performance is optimized in 2.13.0 if we enable scalac: https://github.com/scala/bug/issues/1338 + private[this] def parseConstructorValues( + jsonParser: JsonParser, + context: DeserializationContext, + jsonNode: JsonNode + ): (Array[Object], Seq[CaseClassFieldMappingException]) = { + /* Mutable Fields */ + var constructorValuesIdx = 0 + val constructorValues = new Array[Object](numConstructorArgs) + val errors = mutable.ListBuffer[CaseClassFieldMappingException]() + + while (constructorValuesIdx < numConstructorArgs) { + val field = fields(constructorValuesIdx) + try { + val value = field.parse(context, jsonParser.getCodec, jsonNode) + constructorValues(constructorValuesIdx) = value //mutation + } catch { + case e: CaseClassFieldMappingException => + if (e.path == null) { + // fill in missing path detail + addException( + field, + e.withPropertyPath(CaseClassFieldMappingException.PropertyPath.leaf(field.name)), + constructorValues, + constructorValuesIdx, + errors + ) + } else { + addException( + field, + e, + constructorValues, + constructorValuesIdx, + errors + ) + } + case e: InvalidFormatException => + addException( + field, + CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.leaf(field.name), + CaseClassFieldMappingException.Reason( + s"'${e.getValue.toString}' is not a " + + s"valid ${Types.wrapperType(e.getTargetType).getSimpleName}${validValuesString(e)}", + CaseClassFieldMappingException.JsonProcessingError(e) + ) + ), + constructorValues, + constructorValuesIdx, + errors + ) + case e: MismatchedInputException => + addException( + field, + CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.leaf(field.name), + CaseClassFieldMappingException.Reason( + s"'${jsonNode.asText("")}' is not a " + + s"valid ${Types.wrapperType(e.getTargetType).getSimpleName}${validValuesString(e)}", + CaseClassFieldMappingException.JsonProcessingError(e) + ) + ), + constructorValues, + constructorValuesIdx, + errors + ) + case e: CaseClassMappingException => + constructorValues(constructorValuesIdx) = field.missingValue //mutation + errors ++= e.errors.map(_.scoped(field.name)) + case e: JsonProcessingException => + // don't include source info since it's often blank. Consider adding e.getCause.getMessage + e.clearLocation() + addException( + field, + CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.leaf(field.name), + CaseClassFieldMappingException.Reason( + e.errorMessage, + CaseClassFieldMappingException.JsonProcessingError(e) + ) + ), + constructorValues, + constructorValuesIdx, + errors + ) + case e: java.util.NoSuchElementException if isScalaEnumerationType(field) => + // Scala enumeration mapping issue + addException( + field, + CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.leaf(field.name), + CaseClassFieldMappingException.Reason(e.getMessage) + ), + constructorValues, + constructorValuesIdx, + errors + ) + case e: InjectableValuesException => + // we rethrow, to prevent leaking internal injection detail in the "errors" array + throw e + case NonFatal(e) => + error("Unexpected exception parsing field: " + field, e) + throw e + } + constructorValuesIdx += 1 + } + + (constructorValues, errors.toSeq) + } + + private[this] def isScalaEnumerationType(field: CaseClassField): Boolean = { + // scala.Enumerations are challenging for class type comparison and thus we simply check + // if the class name "starts with" scala.Enumeration which should indicate if a class + // is a scala.Enumeration type. The added benefit is that this should work even if the + // classes come from different classloaders. + field.javaType.getRawClass.getName.startsWith(classOf[scala.Enumeration].getName) || + (field.isOption && field.javaType + .containedType(0).getRawClass.getName.startsWith(classOf[scala.Enumeration].getName)) + } + + /** Add the given exception to the given array buffer of errors while also adding a missing value field to the given array */ + private[this] def addException( + field: CaseClassField, + e: CaseClassFieldMappingException, + array: Array[Object], + idx: Int, + errors: mutable.ListBuffer[CaseClassFieldMappingException] + ): Unit = { + array(idx) = field.missingValue //mutation + errors += e //mutation + } + + private[this] def validValuesString(e: MismatchedInputException): String = { + if (e.getTargetType != null && e.getTargetType.isEnum) + " with valid values: " + e.getTargetType.getEnumConstants.mkString(", ") + else + "" + } + + private[this] def newInstance(constructorValues: Array[Object]): Object = { + val obj = + try { + instantiate(constructorValues) + } catch { + case e @ (_: InvocationTargetException | _: ExceptionInInitializerError) => + if (e.getCause == null) + throw e // propagate the underlying cause of the failed instantiation if available + else throw e.getCause + } + val methodValidationErrors = + executeMethodValidations( + validator, + caseClazzCreator.executableValidationMethodDescriptions, + config, + obj + ) + if (methodValidationErrors.nonEmpty) + throw CaseClassMappingException(methodValidationErrors.toSet) + + obj + } + + private[this] def instantiate( + constructorValues: Array[Object] + ): Object = { + caseClazzCreator.executable match { + case method: Method => + // if the creator is of type Method, we assume the need to invoke the companion object + method.invoke(clazzDescriptor.companion.get.instance, constructorValues: _*) + case constructor: Constructor[_] => + // otherwise simply invoke the constructor + constructor.newInstance(constructorValues: _*).asInstanceOf[Object] + } + } + + private[this] def propertyNamingStrategy: PropertyNamingStrategy = { + Annotations.findAnnotation[JsonNaming](clazzAnnotations) match { + case Some(jsonNaming) => + jsonNaming.value().getDeclaredConstructor().newInstance() + case _ => + config.getPropertyNamingStrategy + } + } + + private[this] def annotateConstructor( + constructor: Constructor[_], + annotations: Seq[Annotation] + ): AnnotatedConstructor = { + val paramAnnotationMaps: Array[AnnotationMap] = { + constructor.getParameterAnnotations.map { parameterAnnotations => + val parameterAnnotationMap = new AnnotationMap() + parameterAnnotations.map(parameterAnnotationMap.add) + parameterAnnotationMap + } + } + + val annotationMap = new AnnotationMap() + annotations.map(annotationMap.add) + new AnnotatedConstructor( + new TypeResolutionContext.Basic(config.getTypeFactory, javaType.getBindings), + constructor, + annotationMap, + paramAnnotationMaps + ) + } + + /* in order to deal with parameterized types we create a JavaType here and carry it */ + private[this] def getBeanPropertyDefinitions( + parameters: Array[Parameter], + annotatedWithParams: AnnotatedWithParams, + fromCompanion: Boolean = false + ): Array[PropertyDefinition] = { + // need to find the scala description which carries the full type information + val constructorParamDescriptors = + findConstructorDescriptor(parameters) match { + case Some(constructorDescriptor) => + constructorDescriptor.params + case _ => + throw InvalidDefinitionException.from( + CaseClassDeserializer.EmptyJsonParser, + s"Unable to locate suitable constructor for class: ${clazz.getName}", + javaType) + } + + for ((parameter, index) <- parameters.zipWithIndex) yield { + val constructorParamDescriptor = constructorParamDescriptors(index) + val scalaType = constructorParamDescriptor.argType + + val parameterJavaType = + if (!javaType.getBindings.isEmpty && + shouldFullyDefineParameterizedType(scalaType, parameter)) { + // what types are bound to the generic case class parameters + val boundTypeParameters: Array[JavaType] = + ReflectionTypes + .parameterizedTypeNames(parameter.getParameterizedType) + .map(javaType.getBindings.findBoundType) + Types + .javaType( + config.getTypeFactory, + scalaType, + boundTypeParameters + ) + } else { + Types.javaType(config.getTypeFactory, scalaType) + } + + val annotatedParameter = + newAnnotatedParameter( + typeResolutionContext = new TypeResolutionContext.Basic( + config.getTypeFactory, + javaType.getBindings + ), // use the TypeBindings from the top-level JavaType, not the parameter JavaType + owner = annotatedWithParams, + annotations = annotatedWithParams.getParameterAnnotations(index), + javaType = parameterJavaType, + index = index + ) + + PropertyDefinition( + parameterJavaType, + scalaType, + SimpleBeanPropertyDefinition + .construct( + config, + annotatedParameter, + new PropertyName(constructorParamDescriptor.name) + ) + ) + } + } + + // note: Prefer while loop over Scala for loop for better performance. The scala for loop + // performance is optimized in 2.13.0 if we enable scalac: https://github.com/scala/bug/issues/1338 + private[this] def findConstructorDescriptor( + parameters: Array[Parameter] + ): Option[ConstructorDescriptor] = { + val constructors = clazzDescriptor.constructors + var index = 0 + while (index < constructors.length) { + val constructorDescriptor = constructors(index) + val params = constructorDescriptor.params + if (params.length == parameters.length) { + // description has the same number of parameters we're looking for, check each type in order + val checkedParams = params.map { param => + Types + .wrapperType(param.argType.erasure) + .isAssignableFrom(Types.wrapperType(parameters(param.argIndex).getType)) + } + if (checkedParams.forall(_ == true)) return Some(constructorDescriptor) + } + index += 1 + } + None + } + + // if we need to attempt to fully specify the JavaType because it is generically types + private[this] def shouldFullyDefineParameterizedType( + scalaType: ScalaType, + parameter: Parameter + ): Boolean = { + // only need to fully specify if the type is parameterized and it has more than one type arg + // or its typeArg is also parameterized. + def isParameterized(reflectionType: Type): Boolean = reflectionType match { + case _: ParameterizedType | _: TypeVariable[_] => + true + case _ => + false + } + + val parameterizedType = parameter.getParameterizedType + !scalaType.isPrimitive && + (scalaType.typeArgs.isEmpty || + scalaType.typeArgs.head.erasure.isAssignableFrom(classOf[Object])) && + parameterizedType != parameter.getType && + isParameterized(parameterizedType) + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassDeserializerResolver.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassDeserializerResolver.scala new file mode 100644 index 0000000000..72d8295968 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassDeserializerResolver.scala @@ -0,0 +1,22 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.databind.deser.Deserializers +import com.fasterxml.jackson.databind.{BeanDescription, DeserializationConfig, JavaType} +import com.twitter.util.reflect.{Types => ReflectionTypes} +import com.twitter.util.validation.ScalaValidator + +private[jackson] class CaseClassDeserializerResolver( + validator: Option[ScalaValidator]) + extends Deserializers.Base { + + override def findBeanDeserializer( + javaType: JavaType, + deserializationConfig: DeserializationConfig, + beanDescription: BeanDescription + ): CaseClassDeserializer = { + if (ReflectionTypes.isCaseClass(javaType.getRawClass)) + new CaseClassDeserializer(javaType, deserializationConfig, beanDescription, validator) + else + null + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassField.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassField.scala new file mode 100644 index 0000000000..820827f5d3 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassField.scala @@ -0,0 +1,459 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.annotation.{ + JsonIgnore, + JsonIgnoreType, + JsonProperty, + JsonTypeInfo, + JsonView +} +import com.fasterxml.jackson.core.{JsonParser, ObjectCodec} +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import com.fasterxml.jackson.databind.introspect.BeanPropertyDefinition +import com.fasterxml.jackson.databind.node.TreeTraversingParser +import com.fasterxml.jackson.databind.util.ClassUtil +import com.fasterxml.jackson.databind.{BeanProperty => JacksonBeanProperty, _} +import com.twitter.conversions.StringOps._ +import com.twitter.util.jackson.annotation.InjectableValue +import com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException +import com.twitter.util.logging.Logging +import com.twitter.util.reflect.Annotations +import java.lang.annotation.Annotation +import java.lang.reflect.Executable +import org.json4s.reflect.ClassDescriptor +import scala.annotation.tailrec +import scala.reflect.NameTransformer + +/* Keeps a mapping of field "type" as a String and the field name */ +private case class FieldInfo(`type`: String, fieldName: String) + +object CaseClassField { + + // optimized + private[jackson] def createFields( + clazz: Class[_], + constructor: Executable, + clazzDescriptor: ClassDescriptor, + propertyDefinitions: Array[CaseClassDeserializer.PropertyDefinition], + fieldAnnotations: scala.collection.Map[String, Array[Annotation]], + namingStrategy: PropertyNamingStrategy + ): Array[CaseClassField] = { + /* CaseClassFields MUST be returned in constructor/method parameter invocation order */ + // we could choose to use the size of the property definitions here, but since they are + // computed there is the possibility it is incorrect and thus we want to ensure we have + // as many definitions as there are parameters defined for the given constructor. + val parameters = constructor.getParameters + val fields = new Array[CaseClassField](parameters.length) + + var index = 0 + while (index < parameters.length) { + val propertyDefinition = propertyDefinitions(index) + // we look up annotations by the field name as that is how they are keyed + val annotations: Array[Annotation] = + fieldAnnotations.get(propertyDefinition.beanPropertyDefinition.getName) match { + case Some(v) => v + case _ => Array.empty[Annotation] + } + val name = jsonNameForField( + namingStrategy, + annotations, + propertyDefinition.beanPropertyDefinition.getName) + + fields(index) = CaseClassField( + name = name, + index = index, + javaType = propertyDefinition.javaType, + parentClass = clazz, + annotations = annotations, + beanPropertyDefinition = propertyDefinition.beanPropertyDefinition, + defaultFn = defaultMethod(clazzDescriptor, index) + ) + + index += 1 + } + + fields + } + + private[this] def jsonNameForField( + namingStrategy: PropertyNamingStrategy, + annotations: Array[Annotation], + name: String + ): String = { + Annotations.findAnnotation[JsonProperty](annotations) match { + case Some(jsonProperty) if jsonProperty.value.nonEmpty => + jsonProperty.value + case _ => + val decodedName = NameTransformer.decode(name) // decode unicode escaped field names + // apply json naming strategy (e.g. snake_case) + namingStrategy.nameForField( + /* config = */ null, + /* field = */ null, + /* defaultName = */ decodedName) + } + } +} + +/* exposed for testing */ +private[jackson] case class CaseClassField private ( + name: String, + index: Int, + javaType: JavaType, + parentClass: Class[_], + annotations: Array[Annotation], + beanPropertyDefinition: BeanPropertyDefinition, + defaultFn: Option[() => Object]) + extends Logging { + + lazy val missingValue: AnyRef = { + if (javaType.isPrimitive) + ClassUtil.defaultValue(javaType.getRawClass) + else + null + } + + private[caseclass] val isOption = javaType.hasRawClass(classOf[Option[_]]) + private[this] val isString = javaType.hasRawClass(classOf[String]) + + /** Lazy as we may not have a contained type */ + private[this] lazy val firstTypeParam: JavaType = javaType.containedType(0) + + /** Annotations defined on the case class definition */ + private[this] lazy val clazzAnnotations: Array[Annotation] = + if (isOption) { + javaType.containedType(0).getRawClass.getAnnotations + } else { + javaType.getRawClass.getAnnotations + } + + /** If the field has a [[JsonIgnore(true)]] annotation or the type has been annotated with [[JsonIgnoreType(true)]]*/ + private[this] lazy val isIgnored: Boolean = + Annotations + .findAnnotation[JsonIgnore](annotations).exists(_.value) || + Annotations + .findAnnotation[JsonIgnoreType](clazzAnnotations).exists(_.value) + + /** If the field is annotated directly with [[JsonDeserialize]] or if the class is annotated similarly */ + private[this] lazy val jsonDeserializer: Option[JsonDeserialize] = + Annotations + .findAnnotation[JsonDeserialize](annotations) + .orElse(Annotations.findAnnotation[JsonDeserialize](clazzAnnotations)) + + /* Public */ + + /** + * Parse the field from a JsonNode representing a JSON object. NOTE: We would normally return a + * `Try[Object]`, but instead we use exceptions to optimize the non-failure case. + * + * @param context DeserializationContext for deserialization + * @param codec Codec for field + * @param objectJsonNode The JSON object + * + * @return The parsed object for this field + * @note Option fields default to None even if no default is specified + * + * @throws CaseClassFieldMappingException with reason for the parsing error + */ + def parse( + context: DeserializationContext, + codec: ObjectCodec, + objectJsonNode: JsonNode + ): Object = { + val injectableValuesOpt: Option[InjectableValues] = + new DeserializationContextAccessor(context).getInjectableValues + val forProperty: CaseClassBeanProperty = beanProperty(context) + + injectableValuesOpt match { + case Some(_) => + Option( + context.findInjectableValue( + /* valueId = */ forProperty.valueObj.map(_.getId).orNull, + /* forProperty = */ forProperty.property, + /* beanInstance = */ null)) match { + case Some(value) => + value + case _ => + findValue(context, codec, objectJsonNode, forProperty.property) + } + case _ => + findValue(context, codec, objectJsonNode, forProperty.property) + } + } + + /* Private */ + + private[this] def findValue( + context: DeserializationContext, + codec: ObjectCodec, + objectJsonNode: JsonNode, + beanProperty: JacksonBeanProperty + ): Object = { + // current active view (if any) + val activeJsonView: Option[Class[_]] = Option(context.getActiveView) + // current @JsonView annotation (if any) + val fieldJsonViews: Option[Seq[Class[_]]] = + Annotations.findAnnotation[JsonView](annotations).map(_.value.toSeq) + + // context has an active view *and* the field is annotated + if (activeJsonView.isDefined && fieldJsonViews.isDefined) { + if (activeJsonView.exists(fieldJsonViews.get.contains(_))) { + // active view is in the list of views from the annotation + parse(context, codec, objectJsonNode, beanProperty) + } else defaultValueOrException(isIgnored) + } else { + // no active view proceed as normal + parse(context, codec, objectJsonNode, beanProperty) + } + } + + private[this] def beanProperty( + context: DeserializationContext, + optionalJavaType: Option[JavaType] = None + ): CaseClassBeanProperty = + newBeanProperty( + context = context, + javaType = javaType, + optionalJavaType = optionalJavaType, + annotatedParameter = beanPropertyDefinition.getConstructorParameter, + annotations = annotations, + name = name, + index = index + ) + + private[this] def parse( + context: DeserializationContext, + codec: ObjectCodec, + objectJsonNode: JsonNode, + forProperty: JacksonBeanProperty + ): Object = { + val fieldJsonNode = objectJsonNode.get(name) + if (!isIgnored && fieldJsonNode != null) { + // not ignored and a value was passed in the JSON + resolveWithDeserializerAnnotation(context, codec, fieldJsonNode, forProperty) match { + case Some(obj) => obj + case _ => // wasn't handled by another resolved deserializer + parseFieldValue(context, codec, fieldJsonNode, forProperty, None) + } + } else defaultValueOrException(isIgnored) + } + + //optimized + private[this] def resolveWithDeserializerAnnotation( + context: DeserializationContext, + fieldCodec: ObjectCodec, + fieldJsonNode: JsonNode, + forProperty: JacksonBeanProperty + ): Option[Object] = jsonDeserializer match { + case Some(annotation: JsonDeserialize) + if isNotAssignableFrom(annotation.using, classOf[JsonDeserializer.None]) => + // specifies a deserializer to use. we don't want to run any processing of the field + // node value here as the field is specified to be deserialized by some other deserializer + Option( + context + .deserializerInstance(beanPropertyDefinition.getPrimaryMember, annotation.using)) match { + case Some(deserializer) => + val treeTraversingParser = new TreeTraversingParser(fieldJsonNode, fieldCodec) + try { + // advance the parser to the next token for deserialization + treeTraversingParser.nextToken + if (isOption) { + Some( + Option( + deserializer + .deserialize(treeTraversingParser, context))) + } else { + Some(deserializer.deserialize(treeTraversingParser, context)) + } + } finally { + treeTraversingParser.close() + } + case _ => + Some( + context.handleInstantiationProblem( + javaType.getRawClass, + annotation.using.toString, + JsonMappingException.from( + context, + "Unable to locate/create deserializer specified by: " + + s"${annotation.getClass.getName}(using = ${annotation.using()})") + ) + ) + } + case Some(annotation: JsonDeserialize) + if isNotAssignableFrom(annotation.contentAs, classOf[java.lang.Void]) => + // there is a @JsonDeserialize annotation but it merely states to deserialize as a specific type + Some( + parseFieldValue( + context, + fieldCodec, + fieldJsonNode, + forProperty, + Some(annotation.contentAs) + ) + ) + case _ => None + } + + private[this] def parseFieldValue( + context: DeserializationContext, + fieldCodec: ObjectCodec, + fieldJsonNode: JsonNode, + forProperty: JacksonBeanProperty, + subTypeClazz: Option[Class[_]] + ): Object = { + if (fieldJsonNode.isNull) { + // the passed JSON value is a 'null' value + defaultValueOrException(isIgnored) + } else if (isString) { + // if this is a String type, do not try to use a JsonParser and simply return the node as text + if (fieldJsonNode.isValueNode) fieldJsonNode.asText() + else fieldJsonNode.toString + } else { + val treeTraversingParser = new TreeTraversingParser(fieldJsonNode, fieldCodec) + try { + // advance the parser to the next token for deserialization + treeTraversingParser.nextToken + if (isOption) { + Option( + parseFieldValue( + context, + fieldCodec, + treeTraversingParser, + beanProperty(context, Some(firstTypeParam)).property, + subTypeClazz) + ) + } else { + assertNotNull( + context, + fieldJsonNode, + parseFieldValue(context, fieldCodec, treeTraversingParser, forProperty, subTypeClazz) + ) + } + } finally { + treeTraversingParser.close() + } + } + } + + private[this] def parseFieldValue( + context: DeserializationContext, + fieldCodec: ObjectCodec, + jsonParser: JsonParser, + forProperty: JacksonBeanProperty, + subTypeClazz: Option[Class[_]] + ): Object = { + val resolvedType = resolveSubType(context, forProperty.getType, subTypeClazz) + Annotations.findAnnotation[JsonTypeInfo](clazzAnnotations) match { + case Some(_) => + // for polymorphic types we cannot contextualize + // thus we go back to the field codec to read + fieldCodec.readValue(jsonParser, resolvedType) + case _ if resolvedType.isContainerType => + // nor container types -- trying to contextualize on a container type leads to really bad performance + // thus we go back to the field codec to read + fieldCodec.readValue(jsonParser, resolvedType) + case _ => + // contextualization for all others + context.readPropertyValue(jsonParser, forProperty, resolvedType) + } + } + + //optimized + private[this] def assertNotNull( + context: DeserializationContext, + field: JsonNode, + value: Object + ): Object = { + value match { + case null => + throw JsonMappingException.from(context, "error parsing '" + field.asText + "'") + case traversable: Traversable[_] => + assertNotNull(context, traversable) + case array: Array[_] => + assertNotNull(context, array) + case _ => // no-op + } + value + } + + private[this] def assertNotNull( + context: DeserializationContext, + traversable: Traversable[_] + ): Unit = { + if (traversable.exists(_ == null)) { + throw JsonMappingException.from( + context, + "Literal null values are not allowed as json array elements." + ) + } + } + + /** Return the default value of the field */ + private[this] def defaultValue: Option[Object] = { + if (defaultFn.isDefined) + defaultFn.map(_()) + else if (isOption) + Some(None) + else + None + } + + /** Return the [[defaultValue]] or else throw a required field exception */ + private[this] def defaultValueOrException(ignoredField: Boolean): Object = { + defaultValue match { + case Some(value) => value + case _ => throwRequiredFieldException(ignoredField) + } + } + + /** Throw a required field exception */ + private[this] def throwRequiredFieldException(ignorable: Boolean): Nothing = { + val (fieldInfoAttributeType, fieldInfoAttributeName) = getFieldInfo(name, annotations) + + val message = + if (ignorable) s"ignored $fieldInfoAttributeType has no default value specified" + else s"$fieldInfoAttributeType is required" + throw CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.leaf(fieldInfoAttributeName), + CaseClassFieldMappingException.Reason( + message, + CaseClassFieldMappingException.RequiredFieldMissing + ) + ) + } + + @tailrec + private[this] def getFieldInfo( + fieldName: String, + annotations: Seq[Annotation] + ): (String, String) = { + if (annotations.isEmpty) { + ("field", fieldName) + } else { + extractFieldInfo(fieldName, annotations.head) match { + case Some(found) => found + case _ => getFieldInfo(fieldName, annotations.tail) + } + } + } + + // Annotations annotated with `@InjectableValue` represent values that are injected + // into a constructed case class from somewhere other than the incoming JSON, + // e.g., with Jackson InjectableValues. When we are parsing the structure of the + // case class, we want to capture information about these annotations for proper + // error reporting on the fields annotated. The annotation name is used instead + // of the generic classifier of "field", and the annotation#value() is used + // instead of the case class field name being marshalled into. + private[this] def extractFieldInfo( + fieldName: String, + annotation: Annotation + ): Option[(String, String)] = { + if (Annotations.isAnnotationPresent[InjectableValue](annotation)) { + val name = Annotations.getValueIfAnnotatedWith[InjectableValue](annotation) match { + case Some(value) if value != null && value.nonEmpty => value + case _ => fieldName + } + Some((annotation.annotationType.getSimpleName.toCamelCase, name)) + } else None + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassJacksonModule.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassJacksonModule.scala new file mode 100644 index 0000000000..ae77e1da89 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/CaseClassJacksonModule.scala @@ -0,0 +1,14 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.module.scala._ +import com.twitter.util.validation.ScalaValidator + +private[jackson] class CaseClassJacksonModule( + validator: Option[ScalaValidator]) + extends JacksonModule { + override def getModuleName: String = this.getClass.getName + + this += { + _.addDeserializers(new CaseClassDeserializerResolver(validator)) + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/SerdeLogging.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/SerdeLogging.scala new file mode 100644 index 0000000000..fb088d8331 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/SerdeLogging.scala @@ -0,0 +1,24 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties + +/** + * "Serde" (technically "SerDe") is a portmanteau of "Serialization and Deserialization" + * + * Mix this trait into a case class to get helpful logging methods. + * This trait adds a `@JsonIgnoreProperties` for fields which are + * defined in the [[com.twitter.util.logging.Logging]] trait so that + * they are not included in JSON serde operations. + * + */ +@JsonIgnoreProperties( + Array( + "logger_name", + "trace_enabled", + "debug_enabled", + "error_enabled", + "info_enabled", + "warn_enabled" + ) +) +trait SerdeLogging extends com.twitter.util.logging.Logging diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/Types.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/Types.scala new file mode 100644 index 0000000000..907c1984f2 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/Types.scala @@ -0,0 +1,144 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.databind.JavaType +import com.fasterxml.jackson.databind.`type`.{ArrayType, TypeBindings, TypeFactory} +import org.json4s.reflect.ScalaType + +private[twitter] object Types { + + /** + * Primitive class to Java wrapper (boxed) primitive class. + * Similar to [[com.fasterxml.jackson.databind.util.ClassUtil.wrapperType(Class primitiveType)]] + * but accepts non primitives as a pass-thru. + */ + def wrapperType(clazz: Class[_]): Class[_] = { + if (clazz == null) + classOf[Null] + else + clazz match { + case java.lang.Byte.TYPE => classOf[java.lang.Byte] + case java.lang.Short.TYPE => classOf[java.lang.Short] + case java.lang.Character.TYPE => classOf[java.lang.Character] + case java.lang.Integer.TYPE => classOf[java.lang.Integer] + case java.lang.Long.TYPE => classOf[java.lang.Long] + case java.lang.Float.TYPE => classOf[java.lang.Float] + case java.lang.Double.TYPE => classOf[java.lang.Double] + case java.lang.Boolean.TYPE => classOf[java.lang.Boolean] + case java.lang.Void.TYPE => classOf[java.lang.Void] + case _ => clazz + } + } + + /** + * Convert a [[org.json4s.reflect.ScalaType]] to a [[com.fasterxml.jackson.databind.JavaType]]. + * This is used when defining the structure of a case class used in parsing JSON. + * + * @note order matters since Maps can be treated as collections of tuples + */ + def javaType( + typeFactory: TypeFactory, + scalaType: ScalaType + ): JavaType = { + // for backwards compatibility we box primitive types + val erasedClazzType = toScalaBoxedType(scalaType.erasure) + + if (scalaType.typeArgs.isEmpty || scalaType.erasure.isEnum) { + typeFactory.constructType(erasedClazzType) + } else if (scalaType.isCollection) { + if (scalaType.isMap) { + typeFactory.constructMapLikeType( + erasedClazzType, + javaType(typeFactory, scalaType.typeArgs.head), + javaType(typeFactory, scalaType.typeArgs(1)) + ) + } else if (scalaType.isArray) { + // need to special-case array creation + ArrayType.construct( + javaType(typeFactory, scalaType.typeArgs.head), + TypeBindings + .create( + classOf[ + java.util.ArrayList[_] + ], // we hardcode the type to `java.util.ArrayList` to properly support Array creation + javaType(typeFactory, scalaType.typeArgs.head) + ) + ) + } else { + typeFactory.constructCollectionLikeType( + erasedClazzType, + javaType(typeFactory, scalaType.typeArgs.head) + ) + } + } else { + typeFactory.constructParametricType( + erasedClazzType, + javaTypes(typeFactory, scalaType.typeArgs): _* + ) + } + } + + /** + * Convert a [[org.json4s.reflect.ScalaType]] to a [[com.fasterxml.jackson.databind.JavaType]] + * taking into account any parameterized types and type bindings. This is used when parsing + * JSON into a type. + * + * @note Match order matters since Maps can be treated as collections of tuples. + */ + def javaType( + typeFactory: TypeFactory, + scalaType: ScalaType, + typeParameters: Array[JavaType] + ): JavaType = { + + if (scalaType.typeArgs.isEmpty || scalaType.erasure.isEnum) { + typeParameters.head + } else if (scalaType.isCollection) { + if (scalaType.isMap) { + typeFactory.constructMapLikeType( + scalaType.erasure, + typeParameters.head, + typeParameters.last + ) + } else if (scalaType.isArray) { + // need to special-case array creation + // we hardcode the type to `java.util.ArrayList` to properly support Array creation + ArrayType.construct( + javaType(typeFactory, scalaType.typeArgs.head, typeParameters), + TypeBindings.create(classOf[java.util.ArrayList[_]], typeParameters.head) + ) + } else { + typeFactory.constructCollectionLikeType( + scalaType.erasure, + typeParameters.head + ) + } + } else { + typeFactory.constructParametricType(scalaType.erasure, typeParameters: _*) + } + } + + /* Private */ + + /* Primitive class to Scala wrapper (boxed) primitive class */ + private[this] def toScalaBoxedType(clazz: Class[_]): Class[_] = clazz match { + case java.lang.Byte.TYPE => classOf[Byte] + case java.lang.Short.TYPE => classOf[Short] + case java.lang.Character.TYPE => classOf[Char] + case java.lang.Integer.TYPE => classOf[Int] + case java.lang.Long.TYPE => classOf[Long] + case java.lang.Float.TYPE => classOf[Float] + case java.lang.Double.TYPE => classOf[Double] + case java.lang.Boolean.TYPE => classOf[Boolean] + case _ => clazz + } + + /* Create a Seq of JavaTypes from the given ScalaTypes */ + private[this] def javaTypes( + typeFactory: TypeFactory, + scalaTypes: Seq[ScalaType] + ): Seq[JavaType] = { + for (scalaType <- scalaTypes) yield { + javaType(typeFactory, scalaType) + } + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD new file mode 100644 index 0000000000..ff58b07f4e --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD @@ -0,0 +1,20 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-jackson-caseclass-exceptions", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "util/util-validator/src/main/scala/com/twitter/util/validation", + ], + exports = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + ], +) diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/CaseClassFieldMappingException.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/CaseClassFieldMappingException.scala new file mode 100644 index 0000000000..6329853660 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/CaseClassFieldMappingException.scala @@ -0,0 +1,103 @@ +package com.twitter.util.jackson.caseclass.exceptions + +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.JsonMappingException +import jakarta.validation.{ConstraintViolation, Payload} + +object CaseClassFieldMappingException { + + /** Marker trait for CaseClassFieldMappingException detail */ + sealed trait Detail + + /** No specified detail */ + case object Unspecified extends Detail + + /** A case class field specified as required is missing from JSON to deserialize */ + case object RequiredFieldMissing extends Detail + + object ValidationError { + sealed trait Location + case object Field extends Location + case object Method extends Location + } + + /** A violation was raised when performing a field or method validation */ + case class ValidationError( + violation: ConstraintViolation[_], + location: ValidationError.Location, + payload: Option[Payload]) + extends Detail + + /** A JsonProcessingException occurred during deserialization */ + case class JsonProcessingError(cause: JsonProcessingException) extends Detail + + /** Throwable detail which specifies a message */ + case class ThrowableError(message: String, cause: Throwable) extends Detail + + /** + * Represents the reason with message and more details for the field mapping exception. + */ + case class Reason( + message: String, + detail: Detail = Unspecified) + + object PropertyPath { + val Empty: PropertyPath = PropertyPath(Seq.empty) + private val FieldSeparator = "." + def leaf(name: String): PropertyPath = Empty.withParent(name) + } + + /** Represents a path to the case class field/property germane to the exception */ + case class PropertyPath(names: Seq[String]) { + private[jackson] def withParent(name: String): PropertyPath = copy(name +: names) + + def isEmpty: Boolean = names.isEmpty + def prettyString: String = names.mkString(PropertyPath.FieldSeparator) + } +} + +/** + * A subclass of [[JsonMappingException]] which bundles together a failed field location as a + * `CaseClassFieldMappingException.PropertyPath` with the the failure reason represented by a + * `CaseClassFieldMappingException.Reason` to carry the failure reason. + * + * @note this exception is a case class in order to have a useful equals() and hashcode() + * methods since this exception is generally carried in a collection inside of a + * [[CaseClassMappingException]]. + * @param path - a `CaseClassFieldMappingException.PropertyPath` instance to the case class field + * that caused the failure. + * @param reason - an instance of a `CaseClassFieldMappingException.Reason` which carries detail of the + * failure reason. + * + * @see [[com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException.PropertyPath]] + * @see [[com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException.Reason]] + * @see [[com.fasterxml.jackson.databind.JsonMappingException]] + * @see [[CaseClassMappingException]] + */ +case class CaseClassFieldMappingException( + path: CaseClassFieldMappingException.PropertyPath, + reason: CaseClassFieldMappingException.Reason) + extends JsonMappingException(null, reason.message) { + + /* Public */ + + /** + * Render a human readable message. If the error message pertains to + * a specific field it is prefixed with the field's name. + */ + override def getMessage: String = { + if (path == null || path.isEmpty) reason.message + else s"${path.prettyString}: ${reason.message}" + } + + /* Private */ + + /** fill in any missing PropertyPath information */ + private[jackson] def withPropertyPath( + path: CaseClassFieldMappingException.PropertyPath + ): CaseClassFieldMappingException = this.copy(path) + + private[jackson] def scoped(fieldName: String): CaseClassFieldMappingException = { + copy(path = path.withParent(fieldName)) + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/CaseClassMappingException.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/CaseClassMappingException.scala new file mode 100644 index 0000000000..613fce0fd9 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/CaseClassMappingException.scala @@ -0,0 +1,77 @@ +package com.twitter.util.jackson.caseclass.exceptions + +import com.fasterxml.jackson.databind.JsonMappingException + +object CaseClassMappingException { + + /** + * Create a new [[CaseClassMappingException]] with no field exceptions. + * @return a new [[CaseClassMappingException]] + */ + def apply(): CaseClassMappingException = new CaseClassMappingException() + + /** + * Create a new [[CaseClassMappingException]] over the given field exceptions. + * @param fieldMappingExceptions exceptions to enclose in returned [[CaseClassMappingException]] + * @return a new [[CaseClassMappingException]] with the given field exceptions. + */ + def apply( + fieldMappingExceptions: Set[CaseClassFieldMappingException] + ): CaseClassMappingException = + new CaseClassMappingException(fieldMappingExceptions) +} + +/** + * A subclass of [[JsonMappingException]] used to signal fatal problems with mapping of JSON + * content to a Scala case class. + * + * Per-field details (of type [[CaseClassFieldMappingException]]) are carried to provide the + * ability to iterate over all exceptions causing the failure to construct the case class. + * + * This extends [[JsonMappingException]] such that this exception is properly handled + * when deserializing into nested case-classes. + * + * @see [[CaseClassFieldMappingException]] + * @see [[com.fasterxml.jackson.databind.JsonMappingException]] + */ +class CaseClassMappingException private ( + fieldMappingExceptions: Set[CaseClassFieldMappingException] = + Set.empty[CaseClassFieldMappingException]) + extends JsonMappingException(null, "") { + + /** + * The collection of [[CaseClassFieldMappingException]] instances which make up this + * [[CaseClassMappingException]]. This collection is intended to be purposely exhaustive in that + * is specifies all errors encountered in mapping JSON content to a Scala case class. + */ + val errors: Seq[CaseClassFieldMappingException] = + fieldMappingExceptions.toSeq.sortBy(_.getMessage) + + // helpers for formatting the exception message + private[this] val errorsSize = errors.size + private[this] val messagePreambleString = + if (errorsSize == 1) "An error was " else s"${errorsSize} errors " + private[this] val errorsString = + if (errorsSize == 1) "Error: " else "Errors: " + private[this] val errorsSeparatorString = "\n\t " + + /** + * Formats a human-readable message which includes the underlying [[CaseClassMappingException]] messages. + * + * ==Example== + * Multiple errors: + * {{{ + * 2 errors encountered during deserialization. + * Errors: com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException: data: must not be empty + * com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException: number: must be greater than or equal to 5 + * }}} + * Single error: + * {{{ + * An error was encountered during deserialization. + * Error: com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException: data: must not be empty + * }}} + */ + override def getMessage: String = + s"${messagePreambleString}encountered during deserialization.\n\t$errorsString" + + errors.mkString(errorsSeparatorString) +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/InjectableValuesException.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/InjectableValuesException.scala new file mode 100644 index 0000000000..03cbc9dcb7 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/InjectableValuesException.scala @@ -0,0 +1,22 @@ +package com.twitter.util.jackson.caseclass.exceptions + +import scala.util.control.NoStackTrace + +/** + * Represents an exception during deserialization due to the inability to properly inject a value via + * Jackson [[com.fasterxml.jackson.databind.InjectableValues]]. A common cause is an incorrectly + * configured mapper that is not instantiated with an appropriate instance of a + * [[com.fasterxml.jackson.databind.InjectableValues]] that can locate the requested value to inject + * during deserialization. + * + * @note this exception is not handled during case class deserialization and is thus expected to be + * handled by callers. + */ +class InjectableValuesException(message: String, cause: Throwable) + extends Exception(message, cause) + with NoStackTrace { + + def this(message: String) { + this(message, null) + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/package.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/package.scala new file mode 100644 index 0000000000..841f0f5158 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/package.scala @@ -0,0 +1,27 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.JsonMappingException + +package object exceptions { + + implicit class RichJsonProcessingException[E <: JsonProcessingException](val self: E) + extends AnyVal { + + /** + * Converts a [[JsonProcessingException]] to a informative error message that can be returned + * to callers. The goal is to not leak internals of the underlying types. + * @return a useful [[String]] error message + */ + def errorMessage: String = { + self match { + case exc: JsonMappingException => + exc.getOriginalMessage // want to return the original message without and location reference + case _ if self.getCause == null => + self.getOriginalMessage // jackson threw the original error + case _ => + self.getCause.getMessage // custom deserialization code threw the exception (e.g., enum deserialization) + } + } + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/package.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/package.scala new file mode 100644 index 0000000000..8c85c36008 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/package.scala @@ -0,0 +1,148 @@ +package com.twitter.util.jackson + +import com.fasterxml.jackson.annotation.JacksonAnnotation +import com.fasterxml.jackson.databind.deser.impl.ValueInjector +import com.fasterxml.jackson.databind.introspect.{ + AnnotatedMember, + AnnotatedParameter, + AnnotatedWithParams, + AnnotationMap, + TypeResolutionContext +} +import com.fasterxml.jackson.databind.{BeanProperty, DeserializationContext, JavaType, PropertyName} +import java.lang.annotation.Annotation +import org.json4s.reflect.ClassDescriptor + +package object caseclass { + + /** Resolves a subtype from a given optional class */ + private[twitter] def resolveSubType( + context: DeserializationContext, + baseType: JavaType, + subClazz: Option[Class[_]] + ): JavaType = { + subClazz match { + case Some(subTypeClazz) if isNotAssignableFrom(subTypeClazz, baseType.getRawClass) => + context.resolveSubType(baseType, subTypeClazz.getName) + case _ => + baseType + } + } + + /** Returns true if the class is not null and not assignable to "other" */ + private[twitter] def isNotAssignableFrom(clazz: Class[_], other: Class[_]): Boolean = { + clazz != null && !isAssignableFrom(other, clazz) + } + + /** boxes both sides for comparison -- no way to turn the from into the to type */ + private[twitter] def isAssignableFrom(fromType: Class[_], toType: Class[_]): Boolean = + Types.wrapperType(toType).isAssignableFrom(Types.wrapperType(fromType)) + + /** Create a new [[CaseClassBeanProperty]]. Returns the [[CaseClassBeanProperty]] and the JacksonInject.Value */ + private[twitter] def newBeanProperty( + context: DeserializationContext, + javaType: JavaType, + optionalJavaType: Option[JavaType], + annotatedParameter: AnnotatedParameter, + annotations: Iterable[Annotation], + name: String, + index: Int + ): CaseClassBeanProperty = { + // to support deserialization with JsonFormat and other Jackson annotations on fields, + // the Jackson annotations must be carried in the mutator of the ValueInjector. + val jacksonAnnotations = new AnnotationMap() + val contextAnnotationsMap = new AnnotationMap() + annotations.foreach { annotation => + if (annotation.annotationType().isAnnotationPresent(classOf[JacksonAnnotation])) { + jacksonAnnotations.add(annotation) + } else { + contextAnnotationsMap.add(annotation) + } + } + + val mutator: AnnotatedMember = optionalJavaType match { + case Some(optJavaType) => + newAnnotatedParameter( + context, + annotatedParameter, + AnnotationMap.merge(jacksonAnnotations, contextAnnotationsMap), + optJavaType, + index) + case _ => + newAnnotatedParameter( + context, + annotatedParameter, + AnnotationMap.merge(jacksonAnnotations, contextAnnotationsMap), + javaType, + index) + } + + val jacksonInjectValue = context.getAnnotationIntrospector.findInjectableValue(mutator) + val beanProperty: BeanProperty = new ValueInjector( + /* propName = */ new PropertyName(name), + /* type = */ optionalJavaType.getOrElse(javaType), + /* mutator = */ mutator, + /* valueId = */ if (jacksonInjectValue == null) null else jacksonInjectValue.getId + ) { + // ValueInjector no longer supports passing contextAnnotations as an + // argument as of jackson 2.9.x + // https://github.com/FasterXML/jackson-databind/commit/76381c528c9b75265e8b93bf6bb4532e4aa8e957#diff-dbcd29e987f27d95f964a674963a9066R24 + private[this] val contextAnnotations = contextAnnotationsMap + + override def getContextAnnotation[A <: Annotation](clazz: Class[A]): A = { + contextAnnotations.get[A](clazz) + } + } + + CaseClassBeanProperty(valueObj = Option(jacksonInjectValue), property = beanProperty) + } + + /** Create an [[AnnotatedMember]] for use in a [[CaseClassBeanProperty]] */ + private[jackson] def newAnnotatedParameter( + context: DeserializationContext, + base: AnnotatedParameter, + annotations: AnnotationMap, + javaType: JavaType, + index: Int + ): AnnotatedMember = + newAnnotatedParameter( + new TypeResolutionContext.Basic(context.getTypeFactory, javaType.getBindings), + if (base == null) null else base.getOwner, + annotations, + javaType, + index + ) + + /** Create an [[AnnotatedMember]] for use in a [[CaseClassBeanProperty]] */ + private[jackson] def newAnnotatedParameter( + typeResolutionContext: TypeResolutionContext, + owner: AnnotatedWithParams, + annotations: AnnotationMap, + javaType: JavaType, + index: Int + ): AnnotatedMember = { + new AnnotatedParameter( + /* owner = */ owner, + /* type = */ javaType, + /* typeResolutionContext = */ typeResolutionContext, + /* annotations = */ annotations, + /* index = */ index + ) + } + + private[jackson] def defaultMethod( + clazzDescriptor: ClassDescriptor, + constructorParamIdx: Int + ): Option[() => Object] = { + val defaultMethodArgNum = constructorParamIdx + 1 + for { + singletonDescriptor <- clazzDescriptor.companion + companionObjectClass = singletonDescriptor.erasure.erasure + companionObject = singletonDescriptor.instance + method <- companionObjectClass.getMethods.find( + _.getName == "$lessinit$greater$default$" + defaultMethodArgNum) + } yield { () => + method.invoke(companionObject) + } + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/package.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/package.scala new file mode 100644 index 0000000000..938625a6c8 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/package.scala @@ -0,0 +1,8 @@ +package com.twitter.util + +import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper} +import com.fasterxml.jackson.module.scala.{ScalaObjectMapper => JacksonScalaObjectMapper} + +package object jackson { + type JacksonScalaObjectMapperType = JacksonObjectMapper with JacksonScalaObjectMapper +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD new file mode 100644 index 0000000000..4452f36502 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD @@ -0,0 +1,19 @@ +scala_library( + sources = ["*.scala"], + compiler_option_sets = ["fatal_warnings"], + provides = scala_artifact( + org = "com.twitter", + name = "util-jackson-serde", + repo = artifactory, + ), + strict_deps = True, + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + "3rdparty/jvm/org/json4s:json4s-core", + "util/util-core:util-core-util", + "util/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass", + "util/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions", + ], +) diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/DefaultSerdeModule.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/DefaultSerdeModule.scala new file mode 100644 index 0000000000..de1679e51b --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/DefaultSerdeModule.scala @@ -0,0 +1,12 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.databind.module.SimpleModule +import com.twitter.{util => ctu} + +private[jackson] object DefaultSerdeModule extends SimpleModule { + addSerializer(WrappedValueSerializer) + addSerializer(DurationStringSerializer) + addSerializer(TimeStringSerializer()) + addDeserializer(classOf[ctu.Duration], DurationStringDeserializer) + addDeserializer(classOf[ctu.Time], TimeStringDeserializer()) +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/DurationStringDeserializer.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/DurationStringDeserializer.scala new file mode 100644 index 0000000000..7b63d7b689 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/DurationStringDeserializer.scala @@ -0,0 +1,43 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.core.{JsonParser, JsonToken} +import com.fasterxml.jackson.databind.{ + DeserializationContext, + JsonDeserializer, + JsonMappingException +} +import com.twitter.util.Duration +import scala.util.control.NonFatal + +/** + * A Jackson [[JsonDeserializer]] for [[com.twitter.util.Duration]]. + */ +object DurationStringDeserializer extends JsonDeserializer[Duration] { + + override final def deserialize(jp: JsonParser, ctxt: DeserializationContext): Duration = + jp.currentToken match { + case JsonToken.NOT_AVAILABLE | JsonToken.START_OBJECT => + handleToken(jp.nextToken, jp, ctxt) + case _ => + handleToken(jp.currentToken, jp, ctxt) + } + + private[this] def handleToken( + token: JsonToken, + jp: JsonParser, + ctxt: DeserializationContext + ): Duration = { + val value = jp.getText.trim + if (value.isEmpty) { + throw JsonMappingException.from(ctxt, "field cannot be empty") + } else { + try { + Duration.parse(value) + } catch { + case NonFatal(e) => + throw JsonMappingException.from(ctxt, e.getMessage, e) + } + } + } + +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/DurationStringSerializer.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/DurationStringSerializer.scala new file mode 100644 index 0000000000..a6d9517a67 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/DurationStringSerializer.scala @@ -0,0 +1,20 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer +import com.twitter.util.Duration + +/** + * A Jackson [[StdSerializer]] for [[com.twitter.util.Duration]]. + */ +object DurationStringSerializer extends StdSerializer[Duration](classOf[Duration]) { + + override def serialize( + value: Duration, + jgen: JsonGenerator, + provider: SerializerProvider + ): Unit = { + jgen.writeString(value.toString) + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/LongKeyDeserializer.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/LongKeyDeserializer.scala new file mode 100644 index 0000000000..42ac80fd39 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/LongKeyDeserializer.scala @@ -0,0 +1,52 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.databind._ +import com.fasterxml.jackson.databind.deser.KeyDeserializers +import com.fasterxml.jackson.module.scala.JacksonModule +import com.twitter.util.WrappedValue +import org.json4s.reflect.{ClassDescriptor, Reflector, classDescribable} + +private[jackson] class LongKeyDeserializer(clazz: Class[_]) extends KeyDeserializer { + private val constructor = clazz.getConstructor(classOf[Long]) + + override def deserializeKey(key: String, ctxt: DeserializationContext): Object = { + val long = key.toLong.asInstanceOf[Object] + constructor.newInstance(long).asInstanceOf[Object] + } +} + +private[jackson] object LongKeyDeserializers extends JacksonModule { + override def getModuleName = "LongKeyDeserializers" + + private val keyDeserializers = new KeyDeserializers { + override def findKeyDeserializer( + `type`: JavaType, + config: DeserializationConfig, + beanDesc: BeanDescription + ): KeyDeserializer = { + val clazz = beanDesc.getBeanClass + if (isJsonWrappedLong(clazz)) + new LongKeyDeserializer(clazz) + else + null + } + } + + private def isJsonWrappedLong(clazz: Class[_]): Boolean = { + classOf[WrappedValue[_]].isAssignableFrom(clazz) && + isWrappedLong(clazz) + } + + private def isWrappedLong(clazz: Class[_]): Boolean = { + val reflector = + Reflector + .describe(classDescribable(clazz)) + .asInstanceOf[ClassDescriptor] + .constructors + .head + val erased = reflector.params.head.argType.erasure + erased == classOf[Long] || erased.getName == "long" + } + + this += { _.addKeyDeserializers(keyDeserializers) } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/TimeStringDeserializer.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/TimeStringDeserializer.scala new file mode 100644 index 0000000000..cd7b54f393 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/TimeStringDeserializer.scala @@ -0,0 +1,99 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.core.{JsonParser, JsonToken} +import com.fasterxml.jackson.databind.deser.ContextualDeserializer +import com.fasterxml.jackson.databind.deser.std.StdScalarDeserializer +import com.fasterxml.jackson.databind.{ + BeanProperty, + DeserializationContext, + JsonDeserializer, + JsonMappingException +} +import com.twitter.util.{Time, TimeFormat} +import java.util.{Locale, TimeZone} +import scala.util.control.NonFatal + +object TimeStringDeserializer { + private[this] val DefaultTimeFormat: String = "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + + def apply(): TimeStringDeserializer = this(DefaultTimeFormat) + + def apply(pattern: String): TimeStringDeserializer = + this(pattern, None, TimeZone.getTimeZone("UTC")) + + def apply(pattern: String, locale: Option[Locale], timezone: TimeZone): TimeStringDeserializer = + new TimeStringDeserializer(new TimeFormat(pattern, locale, timezone)) +} + +/** + * A Jackson [[JsonDeserializer]] for [[com.twitter.util.Time]]. + * @param timeFormat the configured [[com.twitter.util.TimeFormat]] for this deserializer. + */ +class TimeStringDeserializer(private[this] val timeFormat: TimeFormat) + extends StdScalarDeserializer[Time](classOf[Time]) + with ContextualDeserializer { + + override def deserialize(jp: JsonParser, ctxt: DeserializationContext): Time = + jp.currentToken match { + case JsonToken.NOT_AVAILABLE | JsonToken.START_OBJECT => + handleToken(jp.nextToken, jp, ctxt) + case _ => + handleToken(jp.currentToken, jp, ctxt) + } + + private[this] def handleToken( + token: JsonToken, + jp: JsonParser, + ctxt: DeserializationContext + ): Time = { + try { + val value = jp.getText.trim + if (value.isEmpty) { + throw JsonMappingException.from(ctxt, "field cannot be empty") + } else { + timeFormat.parse(value) + } + } catch { + case NonFatal(e) => + throw JsonMappingException.from( + ctxt, + "error parsing '" + jp.getText + s"' into a ${Time.getClass.getName}", + e) + } + } + + /** + * This method allows extracting the [[com.fasterxml.jackson.annotation.JsonFormat JsonFormat]] + * annotation to create a [[com.twitter.util.TimeFormat TimeFormat]] based on the specifications + * provided in the annotation. The implementation follows the Jackson's java8 & joda-time versions + * + * @param ctxt Deserialization context to access configuration, additional deserializers + * that may be needed by this deserializer + * @param property Method, field or constructor parameter that represents the property (and is + * used to assign deserialized value). Should be available; but there may be + * cases where caller can not provide it and null is passed instead + * (in which case impls usually pass 'this' deserializer as is) + * + * @return Deserializer to use for deserializing values of specified property; may be this + * instance or a new instance. + * + * @see https://github.com/FasterXML/jackson-modules-java8/blob/master/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/deser/JSR310DateTimeDeserializerBase.java#L29 + */ + override def createContextual( + ctxt: DeserializationContext, + property: BeanProperty + ): JsonDeserializer[_] = { + val deserializerOption: Option[TimeStringDeserializer] = for { + jsonFormat <- Option(findFormatOverrides(ctxt, property, handledType())) + deserializer <- Option( + TimeStringDeserializer( + jsonFormat.getPattern, + Option(jsonFormat.getLocale), + Option(jsonFormat.getTimeZone).getOrElse(TimeZone.getTimeZone("UTC")) + )) if jsonFormat.hasPattern + } yield { + deserializer + } + deserializerOption.getOrElse(this) + } +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/TimeStringSerializer.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/TimeStringSerializer.scala new file mode 100644 index 0000000000..e621dec917 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/TimeStringSerializer.scala @@ -0,0 +1,73 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.ser.ContextualSerializer +import com.fasterxml.jackson.databind.ser.std.StdScalarSerializer +import com.fasterxml.jackson.databind.{BeanProperty, JsonSerializer, SerializerProvider} +import com.twitter.util.{Time, TimeFormat} +import java.util.{Locale, TimeZone} + +object TimeStringSerializer { + // TODO: this should be symmetric to the deserializer but we introduced this originally + // without contextual information and thus all strings were serialized with the default + // format of the object mapper ("yyyy-MM-dd HH:mm:ss Z"). This will require a migration. + // private[this] val DefaultTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + private[this] val DefaultTimeFormat = "yyyy-MM-dd HH:mm:ss Z" + + def apply(): TimeStringSerializer = this(DefaultTimeFormat) + + def apply(pattern: String): TimeStringSerializer = + this(pattern, None, TimeZone.getTimeZone("UTC")) + + def apply(pattern: String, locale: Option[Locale], timezone: TimeZone): TimeStringSerializer = + new TimeStringSerializer(new TimeFormat(pattern, locale, timezone)) +} + +/** + * A Jackson [[JsonSerializer]] for [[com.twitter.util.Time]]. + * @param timeFormat the configured [[com.twitter.util.TimeFormat]] for this serializer. + */ +class TimeStringSerializer(private[this] val timeFormat: TimeFormat) + extends StdScalarSerializer[Time](classOf[Time]) + with ContextualSerializer { + + override def serialize(value: Time, jgen: JsonGenerator, provider: SerializerProvider): Unit = { + jgen.writeString(timeFormat.format(value)) + } + + /** + * This method allows extracting the [[com.fasterxml.jackson.annotation.JsonFormat JsonFormat]] + * annotation and create a [[com.twitter.util.TimeFormat TimeFormat]] based on the specifications + * provided in the annotation. The implementation follows the Jackson's java8 & joda-time versions + * + * @param provider Serialization provider to access configuration, additional deserializers + * that may be needed by this deserializer + * @param property Method, field or constructor parameter that represents the property (and is + * used to assign serialized value). Should be available; but there may be + * cases where caller can not provide it and null is passed instead + * (in which case impls usually pass 'this' serializer as is) + * + * @return Serializer to use for serializing values of specified property; may be this + * instance or a new instance. + * + * @see https://github.com/FasterXML/jackson-modules-java8/blob/master/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/ser/JSR310FormattedSerializerBase.java#L114 + */ + override def createContextual( + provider: SerializerProvider, + property: BeanProperty + ): JsonSerializer[_] = { + val serializerOption: Option[TimeStringSerializer] = for { + jsonFormat <- Option(findFormatOverrides(provider, property, handledType())) + serializer <- Option( + TimeStringSerializer( + jsonFormat.getPattern, + Option(jsonFormat.getLocale), + Option(jsonFormat.getTimeZone).getOrElse(TimeZone.getTimeZone("UTC")) + )) if jsonFormat.hasPattern + } yield { + serializer + } + serializerOption.getOrElse(this) + } + +} diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/WrappedValueSerializer.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/WrappedValueSerializer.scala new file mode 100644 index 0000000000..957e7e2fd6 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/WrappedValueSerializer.scala @@ -0,0 +1,18 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.SerializerProvider +import com.fasterxml.jackson.databind.ser.std.StdSerializer +import com.twitter.util.WrappedValue + +private[jackson] object WrappedValueSerializer + extends StdSerializer[WrappedValue[_]](classOf[WrappedValue[_]]) { + + override def serialize( + wrappedValue: WrappedValue[_], + jgen: JsonGenerator, + provider: SerializerProvider + ): Unit = { + jgen.writeObject(wrappedValue.onlyValue) + } +} diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/BUILD new file mode 100644 index 0000000000..d853a7cc0b --- /dev/null +++ b/util-jackson/src/test/java/com/twitter/util/jackson/BUILD @@ -0,0 +1,8 @@ +java_library( + sources = ["*.java"], + compiler_option_sets = ["fatal_warnings"], + strict_deps = "True", + dependencies = [ + "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + ], +) diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/CarMake.java b/util-jackson/src/test/java/com/twitter/util/jackson/CarMake.java new file mode 100644 index 0000000000..6dbc473047 --- /dev/null +++ b/util-jackson/src/test/java/com/twitter/util/jackson/CarMake.java @@ -0,0 +1,8 @@ +package com.twitter.util.jackson; + +/** + * Test Java Enum + */ +public enum CarMake { + Ford, Honda; +} diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/CarMakeEnum.java b/util-jackson/src/test/java/com/twitter/util/jackson/CarMakeEnum.java new file mode 100644 index 0000000000..800d464092 --- /dev/null +++ b/util-jackson/src/test/java/com/twitter/util/jackson/CarMakeEnum.java @@ -0,0 +1,9 @@ +package com.twitter.util.jackson; + +/** + * Test Java Enum + */ +public enum CarMakeEnum { + ford, + vw; +} diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/TestInjectableValue.java b/util-jackson/src/test/java/com/twitter/util/jackson/TestInjectableValue.java new file mode 100644 index 0000000000..adbfb60f47 --- /dev/null +++ b/util-jackson/src/test/java/com/twitter/util/jackson/TestInjectableValue.java @@ -0,0 +1,19 @@ +package com.twitter.util.jackson; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * FOR TESTING ONLY + */ +@Target(ElementType.PARAMETER) +@Retention(RetentionPolicy.RUNTIME) +public @interface TestInjectableValue { + /** + * FOR TESTING ONLY + * @return the value + */ + String value() default ""; +} diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD new file mode 100644 index 0000000000..47cde59828 --- /dev/null +++ b/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD @@ -0,0 +1,9 @@ +junit_tests( + sources = ["**/*.java"], + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson-dataformat-yaml", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/scalatestplus:junit", + "util/util-jackson/src/main/scala/com/twitter/util/jackson", + ], +) diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/tests/JsonDiffJavaTest.java b/util-jackson/src/test/java/com/twitter/util/jackson/tests/JsonDiffJavaTest.java new file mode 100644 index 0000000000..492c36c88b --- /dev/null +++ b/util-jackson/src/test/java/com/twitter/util/jackson/tests/JsonDiffJavaTest.java @@ -0,0 +1,86 @@ +package com.twitter.util.jackson.tests; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import scala.Option; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import org.junit.Assert; +import org.junit.Test; + +import com.twitter.util.jackson.JsonDiff; + +public class JsonDiffJavaTest extends Assert { + + @Test + public void testDiff() { + String a = "{\"a\": 1, \"b\": 2}"; + String b = "{\"b\": 2, \"a\": 1}"; + + Option result = JsonDiff.diff(a, b); + assertFalse(result.isDefined()); + } + + @Test + public void testDiffWithNormalizer() { + String a = "{\"a\": 1, \"b\": 2, \"c\": 5}"; + String b = "{\"b\": 2, \"a\": 1, \"c\": 3}"; + + // normalizerFn will "normalize" the value for "c" into the expected value + Option result = JsonDiff.diff(a, b, (JsonNode jsonNode) -> ((ObjectNode)jsonNode).put("c", 5) ); + assertFalse(result.isDefined()); + } + + @Test + public void testAssertDiff() { + String a = "{\"a\": 1, \"b\": 2}"; + String b = "{\"b\": 2, \"a\": 1}"; + + JsonDiff.assertDiff(a, b); + } + + @Test + public void testAssertDiffWithNormalizer() { + String a = "{\"a\": 1, \"b\": 2, \"c\": 5}"; + String b = "{\"b\": 2, \"a\": 1, \"c\": 3}"; + + // normalizerFn will "normalize" the value for "c" into the expected value + JsonDiff.assertDiff(a, b, (JsonNode jsonNode) -> ((ObjectNode)jsonNode).put("c", 5) ); + } + + @Test + public void testAssertDiffWithPrintStream() throws Exception { + String a = "{\"a\": 1, \"b\": 2}"; + String b = "{\"b\": 2, \"a\": 1}"; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos, true, "utf-8"); + + try { + JsonDiff.assertDiff(a, b, ps); + } finally { + ps.flush(); + ps.close(); + } + } + + @Test + public void testAssertDiffWithNormalizerAndPrintStream() throws Exception { + String a = "{\"a\": 1, \"b\": 2, \"c\": 5}"; + String b = "{\"b\": 2, \"a\": 1, \"c\": 3}"; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos, true, "utf-8"); + + try { + // normalizerFn will "normalize" the value for "c" into the expected value + JsonDiff.assertDiff(a, b, (JsonNode jsonNode) -> ((ObjectNode)jsonNode).put("c", 5) ); + } finally { + ps.flush(); + ps.close(); + } + } +} diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/tests/ScalaObjectMapperJavaTest.java b/util-jackson/src/test/java/com/twitter/util/jackson/tests/ScalaObjectMapperJavaTest.java new file mode 100644 index 0000000000..4a09680646 --- /dev/null +++ b/util-jackson/src/test/java/com/twitter/util/jackson/tests/ScalaObjectMapperJavaTest.java @@ -0,0 +1,47 @@ +package com.twitter.util.jackson.tests; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import org.junit.Assert; +import org.junit.Test; + +import com.twitter.util.jackson.ScalaObjectMapper; + +public class ScalaObjectMapperJavaTest extends Assert { + + /* Class under test */ + private final ScalaObjectMapper mapper = ScalaObjectMapper.apply(); + + @Test + public void testConstructors() { + // new mapper with defaults + Assert.assertNotNull(ScalaObjectMapper.apply()); + + // augments the underlying mapper with the defaults + Assert.assertNotNull(ScalaObjectMapper.apply(mapper.underlying())); + + // copies the underlying Jackson object mapper + Assert.assertNotNull(ScalaObjectMapper.objectMapper(mapper.underlying())); + try { + ScalaObjectMapper.yamlObjectMapper(mapper.underlying()); + fail(); + } catch (IllegalArgumentException e) { + // underlying mapper needs to have a JsonFactory of type YAMLFactory + }; + Assert.assertNotNull(ScalaObjectMapper.yamlObjectMapper(ScalaObjectMapper.builder().yamlObjectMapper().underlying())); + Assert.assertNotNull(ScalaObjectMapper.camelCaseObjectMapper(mapper.underlying())); + Assert.assertNotNull(ScalaObjectMapper.snakeCaseObjectMapper(mapper.underlying())); +} + + @Test + public void testBuilderConstructors() { + Assert.assertNotNull(ScalaObjectMapper.builder().objectMapper()); + Assert.assertNotNull(ScalaObjectMapper.builder().objectMapper(new JsonFactory())); + Assert.assertNotNull(ScalaObjectMapper.builder().objectMapper(new YAMLFactory())); + + Assert.assertNotNull(ScalaObjectMapper.builder().yamlObjectMapper()); + Assert.assertNotNull(ScalaObjectMapper.builder().camelCaseObjectMapper()); + Assert.assertNotNull(ScalaObjectMapper.builder().snakeCaseObjectMapper()); + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD new file mode 100644 index 0000000000..fdeba22589 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD @@ -0,0 +1,41 @@ +junit_tests( + sources = [ + "*.java", + "*.scala", + "caseclass/*.scala", + "serde/*.scala", + ], + dependencies = [ + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", + "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", + "3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson-dataformat-yaml", + "3rdparty/jvm/com/fasterxml/jackson/datatype:jackson-datatype-jsr310", + "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", + "3rdparty/jvm/junit", + "3rdparty/jvm/org/json4s:json4s-core", + "3rdparty/jvm/org/scala-lang/modules:scala-collection-compat", + "3rdparty/jvm/org/scalacheck", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:scalacheck", + "3rdparty/jvm/org/slf4j:slf4j-api", + scoped( + "3rdparty/jvm/org/slf4j:slf4j-simple", + scope = "runtime", + ), + "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io", + "util/util-jackson/src/main/scala/com/fasterxml/jackson/databind", + "util/util-jackson/src/main/scala/com/twitter/util/jackson", + "util/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass", + "util/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions", + "util/util-jackson/src/main/scala/com/twitter/util/jackson/serde", + "util/util-jackson/src/test/java/com/twitter/util/jackson", + "util/util-mock/src/main/scala/com/twitter/util/mock", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator/src/main/scala/com/twitter/util/validation", + ], +) diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala new file mode 100644 index 0000000000..9c0e19953a --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala @@ -0,0 +1,81 @@ +package com.twitter.util.jackson + +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class JSONTest extends AnyFunSuite with Matchers { + + test("JSON#parse 1") { + JSON.parse[Map[String, Int]]("""{"a": 1, "b": 2}""") match { + case Some(map) => + map("a") should equal(1) + map("b") should equal(2) + case _ => fail() + } + } + + test("JSON#parse 2") { + JSON.parse[Seq[String]]("""["a", "b", "c"]""") match { + case Some(seq) => + seq.size should equal(3) + seq.head should be("a") + seq(1) should be("b") + seq.last should be("c") + case _ => fail() + } + } + + test("JSON#parse 3") { + JSON.parse[FooClass]("""{"id": "abcd1234"}""") match { + case Some(foo) => + foo.id should equal("abcd1234") + case _ => fail() + } + } + + test("JSON#parse 4") { + JSON.parse[CaseClassWithNotEmptyValidation]("""{"name": "", "make": "vw"}""") match { + case Some(clazz) => + // performs no validations + clazz.name.isEmpty should be(true) + clazz.make should equal(CarMakeEnum.vw) + case _ => fail() + } + } + + test("JSON#write 1") { + JSON.write(Map("a" -> 1, "b" -> 2)) should equal("""{"a":1,"b":2}""") + } + + test("JSON#write 2") { + JSON.write(Seq("a", "b", "c")) should equal("""["a","b","c"]""") + } + + test("JSON#write 3") { + JSON.write(FooClass("abcd1234")) should equal("""{"id":"abcd1234"}""") + } + + test("JSON#prettyPrint 1") { + JSON.prettyPrint(Map("a" -> 1, "b" -> 2)) should equal("""{ + | "a" : 1, + | "b" : 2 + |}""".stripMargin) + } + + test("JSON#prettyPrint 2") { + JSON.prettyPrint(Seq("a", "b", "c")) should equal("""[ + | "a", + | "b", + | "c" + |]""".stripMargin) + } + + test("JSON#prettyPrint 3") { + JSON.prettyPrint(FooClass("abcd1234")) should equal("""{ + | "id" : "abcd1234" + |}""".stripMargin) + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/JsonDiffTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/JsonDiffTest.scala new file mode 100644 index 0000000000..317a9985b6 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/JsonDiffTest.scala @@ -0,0 +1,256 @@ +package com.twitter.util.jackson + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import java.io.{ByteArrayOutputStream, PrintStream} +import org.junit.runner.RunWith +import org.scalacheck.Gen +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner +import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks +import scala.util.control.NonFatal + +@RunWith(classOf[JUnitRunner]) +class JsonDiffTest + extends AnyFunSuite + with Matchers + with BeforeAndAfterEach + with BeforeAndAfterAll + with JsonGenerator + with ScalaCheckDrivenPropertyChecks { + + private[this] val mapper = ScalaObjectMapper.builder.objectMapper + + private[this] val baos = new ByteArrayOutputStream() + private[this] val ps = new PrintStream(baos, true, "utf-8") + + override protected def beforeEach(): Unit = { + ps.flush() + } + + override protected def afterAll(): Unit = { + try { + ps.close() // closes underlying outputstream + } catch { + case NonFatal(_) => // do nothing + } + super.afterAll() + } + + test("JsonDiff#diff success") { + val a = + """ + { + "a": 1, + "b": 2 + } + """ + + val b = + """ + { + "b": 2, + "a": 1 + } + """ + + val result = JsonDiff.diff(a, b) + result.isDefined should be(false) + } + + test("JsonDiff#diff with normalizer success") { + val expected = + """ + { + "a": 1, + "b": 2, + "c": 5 + } + """ + + // normalizerFn will "normalize" the value for "c" into the expected value + val actual = + """ + { + "b": 2, + "a": 1, + "c": 3 + } + """ + + val result = JsonDiff.diff( + expected, + actual, + normalizeFn = { jsonNode: JsonNode => + jsonNode.asInstanceOf[ObjectNode].put("c", 5) + } + ) + result.isDefined should be(false) + } + + test("JsonDiff#diff failure") { + val expected = + """ + { + "a": 1, + "b": 2 + } + """ + + val actual = + """ + { + "b": 22, + "a": 1 + } + """ + + val result = JsonDiff.diff(expected, actual) + result.isDefined should be(true) + } + + test("JsonDiff#diff with normalizer failure") { + val expected = + """ + { + "a": 1, + "b": 2, + "c": 5 + } + """ + + // normalizerFn only touches the value for "c" and "b" still doesn't match + val actual = + """ + { + "b": 22, + "a": 1, + "c": 3 + } + """ + + val result = JsonDiff.diff( + expected, + actual, + normalizeFn = { jsonNode: JsonNode => + jsonNode.asInstanceOf[ObjectNode].put("c", 5) + } + ) + result.isDefined should be(true) + } + + test("JsonDiff#assertDiff pass") { + val expected = + """ + { + "a": 1, + "b": 2 + } + """ + + val actual = + """ + { + "a": 1, + "b": 2 + } + """ + + JsonDiff.assertDiff(expected, actual, p = ps) + } + + test("JsonDiff#assertDiff with normalizer pass") { + val expected = + """ + { + "a": 1, + "b": 2, + "c": 3 + } + """ + + // normalizerFn will "normalize" the value for "c" into the expected value + val actual = + """ + { + "c": 5, + "a": 1, + "b": 2 + } + """ + + JsonDiff.assertDiff( + expected, + actual, + normalizeFn = { jsonNode: JsonNode => + jsonNode.asInstanceOf[ObjectNode].put("c", 3) + }, + p = ps) + } + + test("JsonDiff#assertDiff with normalizer fail") { + val expected = + """ + { + "a": 1, + "b": 2, + "c": 3 + } + """ + + // normalizerFn only touches the value for "c" and "a" still doesn't match + val actual = + """ + { + "c": 5, + "a": 11, + "b": 2 + } + """ + + intercept[AssertionError] { + JsonDiff.assertDiff( + expected, + actual, + normalizeFn = { jsonNode: JsonNode => + jsonNode.asInstanceOf[ObjectNode].put("c", 3) + }, + p = ps) + } + } + + test("JsonDiff#diff arbitrary pass") { + val expected = for { + depth <- Gen.chooseNum[Int](1, 10) + on <- objectNode(depth) + } yield on + + forAll(expected) { node => + val json = mapper.writeValueAsString(node) + JsonDiff.diff(expected = json, actual = json).isEmpty should be(true) + } + } + + test("JsonDiff#diff arbitrary fail") { + val values = for { + depth <- Gen.chooseNum[Int](1, 5) + expected <- objectNode(depth) + actual <- objectNode(depth) + if expected != actual + } yield (expected, actual) + + forAll(values) { + case (expected, actual) => + val result = JsonDiff.diff(expected = expected, actual = actual) + result.isDefined should be(true) + } + } + + test("JsonDiff#generate sorted") { + val before = mapper.parse[JsonNode]("""{"a":1,"c":3,"b":2}""") + val expected = """{"a":1,"b":2,"c":3}""" + JsonDiff.toSortedString(before) should equal(expected) + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/JsonGenerator.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/JsonGenerator.scala new file mode 100644 index 0000000000..9ab0d334b7 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/JsonGenerator.scala @@ -0,0 +1,72 @@ +package com.twitter.util.jackson + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.{ + ArrayNode, + IntNode, + JsonNodeFactory, + ObjectNode, + TextNode +} +import org.scalacheck.Gen._ +import org.scalacheck._ +import scala.jdk.CollectionConverters._ + +object JsonGenerator extends JsonGenerator + +trait JsonGenerator { + + private[this] val jsonNodeFactory: JsonNodeFactory = new JsonNodeFactory(false) + + /** Generate either a ArrayNode or a ObjectNode */ + def jsonNode(depth: Int): Gen[JsonNode] = oneOf(arrayNode(depth), objectNode(depth)) + + /** Generate a ArrayNode */ + def arrayNode(depth: Int): Gen[ArrayNode] = for { + n <- choose(1, 4) + vals <- values(n, depth) + } yield new ArrayNode(jsonNodeFactory).addAll(vals.asJava) + + /** Generate a ObjectNode */ + def objectNode(depth: Int): Gen[ObjectNode] = for { + n <- choose(1, 4) + ks <- keys(n) + vals <- values(n, depth) + } yield { + val kids: Map[String, JsonNode] = Map(ks.zip(vals): _*) + new ObjectNode(jsonNodeFactory, kids.asJava) + } + + /** Generate a list of keys to be used in the map of a ObjectNode */ + private[this] def keys(n: Int): Gen[List[String]] = { + val key: Gen[String] = for { + str <- alphaStr.filter(_.nonEmpty).map(_.toLowerCase) + } yield { + if (str.length > 4) str.substring(0, 3) + else str + } + listOfN(n, key) + } + + /** + * Generate a list of values to be used in the map of a ObjectNode or in the list of an ArrayNode. + */ + private[this] def values(n: Int, depth: Int): Gen[List[JsonNode]] = listOfN(n, value(depth)) + + /** + * Generate a value to be used in the map of a ObjectNode or in the list of an ArrayNode. + */ + private[this] def value(depth: Int): Gen[JsonNode] = + if (depth == 0) terminalNode + else oneOf(jsonNode(depth - 1), terminalNode) + + /** Generate a terminal node */ + private[this] def terminalNode: Gen[JsonNode] = + oneOf( + new IntNode(4), + new IntNode(2), + new TextNode("b"), + new TextNode("i"), + new TextNode("r"), + new TextNode("d")) +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/NoValidatorImplConstraint.java b/util-jackson/src/test/scala/com/twitter/util/jackson/NoValidatorImplConstraint.java new file mode 100644 index 0000000000..61ae4d3398 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/NoValidatorImplConstraint.java @@ -0,0 +1,38 @@ +package com.twitter.util.jackson; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = {}) +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +public @interface NoValidatorImplConstraint { + + /** Every constraint annotation must define a message element of type String. */ + String message() default ""; + + /** + * Every constraint annotation must define a groups element that specifies the processing groups + * with which the constraint declaration is associated. + */ + Class[] groups() default {}; + + /** + * Constraint annotations must define a payload element that specifies the payload with which + * the constraint declaration is associated. + */ + Class[] payload() default {}; +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperFunctions.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperFunctions.scala new file mode 100644 index 0000000000..d4ed02fb1a --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperFunctions.scala @@ -0,0 +1,95 @@ +package com.twitter.util.jackson + +import com.fasterxml.jackson.core.`type`.TypeReference +import com.fasterxml.jackson.databind.{ + JsonNode, + ObjectMapperCopier, + ObjectMapper => JacksonObjectMapper +} +import com.twitter.util.jackson.caseclass.exceptions.{ + CaseClassFieldMappingException, + CaseClassMappingException +} +import com.twitter.util.logging.Logging +import java.lang.reflect.{ParameterizedType, Type} +import org.scalatest.matchers.should.Matchers + +trait ScalaObjectMapperFunctions extends Matchers with Logging { + /* Abstract */ + protected def mapper: ScalaObjectMapper + + protected def parse[T: Manifest](string: String): T = + mapper.parse[T](string) + + protected def parse[T: Manifest](jsonNode: JsonNode): T = + mapper.parse[T](jsonNode) + + protected def parse[T: Manifest](obj: T): T = + mapper.parse[T](mapper.writeValueAsBytes(obj)) + + protected def generate(any: Any): String = + mapper.writeValueAsString(any) + + protected def assertJson[T: Manifest](obj: T, expected: String): Unit = { + val json = generate(obj) + JsonDiff.assertDiff(expected, json) + parse[T](json) should equal(obj) + } + + protected def copy(underlying: JacksonScalaObjectMapperType): JacksonScalaObjectMapperType = + ObjectMapperCopier.copy(underlying) + + protected def typeFromManifest(m: Manifest[_]): Type = + if (m.typeArguments.isEmpty) { + m.runtimeClass + } else { + new ParameterizedType { + override def getRawType: Class[_] = m.runtimeClass + override def getActualTypeArguments: Array[Type] = + m.typeArguments.map(typeFromManifest).toArray + override def getOwnerType: Null = null + } + } + + protected[this] def typeReference[T: Manifest]: TypeReference[T] = new TypeReference[T] { + override def getType: Type = typeFromManifest(manifest[T]) + } + + protected def deserialize[T: Manifest](mapper: JacksonObjectMapper, value: String): T = + mapper.readValue(value, typeReference[T]) + + protected def clearStackTrace( + exceptions: Seq[CaseClassFieldMappingException] + ): Seq[CaseClassFieldMappingException] = { + exceptions.foreach(_.setStackTrace(Array())) + exceptions + } + + protected def assertJsonParse[T: Manifest](json: String, withErrors: Seq[String]): Unit = { + if (withErrors.nonEmpty) { + val e1 = intercept[CaseClassMappingException] { + val parsed = parse[T](json) + println("Incorrectly parsed: " + ScalaObjectMapper().writePrettyString(parsed)) + } + assertObjectParseException(e1, withErrors) + + // also check that we can parse into an intermediate JsonNode + val e2 = intercept[CaseClassMappingException] { + val jsonNode = parse[JsonNode](json) + parse[T](jsonNode) + } + assertObjectParseException(e2, withErrors) + } else parse[T](json) + } + + private def assertObjectParseException( + e: CaseClassMappingException, + withErrors: Seq[String] + ): Unit = { + trace(e.errors.mkString("\n")) + clearStackTrace(e.errors) + + val actualMessages = e.errors.map(_.getMessage) + JsonDiff.assertDiff(withErrors, actualMessages) + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala new file mode 100644 index 0000000000..fe99859470 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala @@ -0,0 +1,2509 @@ +package com.twitter.util.jackson + +import com.fasterxml.jackson.core.{JsonFactory, JsonParser} +import com.fasterxml.jackson.databind.module.SimpleModule +import com.fasterxml.jackson.databind.node.{IntNode, TreeTraversingParser} +import com.fasterxml.jackson.databind.{ + DeserializationContext, + DeserializationFeature, + JsonDeserializer, + JsonMappingException, + JsonNode, + MapperFeature, + PropertyNamingStrategy, + ObjectMapper => JacksonObjectMapper +} +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.module.scala.{ + DefaultScalaModule, + ScalaObjectMapper => JacksonScalaObjectMapper +} +import com.twitter.io.Buf +import com.twitter.util.Duration +import com.twitter.util.jackson.Obj.{ + NestedCaseClassInObject, + NestedCaseClassInObjectWithNestedCaseClassInObjectParam +} +import com.twitter.util.jackson.TypeAndCompanion.NestedCaseClassInCompanion +import com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException.{ + Unspecified, + ValidationError +} +import com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException +import com.twitter.util.jackson.internal.{ + SimplePersonInPackageObject, + SimplePersonInPackageObjectWithoutConstructorParams +} +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import java.time.format.DateTimeFormatter +import java.time.{LocalDateTime, ZoneId} +import java.util.concurrent.TimeUnit +import java.util.{TimeZone, UUID} +import org.junit.Assert.assertNotNull +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner +import scala.util.Random + +private object ScalaObjectMapperTest { + val UTCZoneId: ZoneId = ZoneId.of("UTC") + val ISO8601DateTimeFormatter: DateTimeFormatter = + DateTimeFormatter.ISO_OFFSET_DATE_TIME // "2014-05-30T03:57:59.302Z" + + class ZeroOrOneDeserializer extends JsonDeserializer[ZeroOrOne] { + override def deserialize( + jsonParser: JsonParser, + deserializationContext: DeserializationContext + ): ZeroOrOne = { + jsonParser.getValueAsString match { + case "zero" => Zero + case "one" => One + case _ => + throw new JsonMappingException(null, "Invalid value") + } + } + } + + object MixInAnnotationsModule extends SimpleModule { + setMixInAnnotation(classOf[Point], classOf[PointMixin]) + setMixInAnnotation(classOf[CaseClassShouldUseKebabCaseFromMixin], classOf[KebabCaseMixin]) + } +} + +@RunWith(classOf[JUnitRunner]) +class ScalaObjectMapperTest + extends AnyFunSuite + with BeforeAndAfterAll + with Matchers + with ScalaObjectMapperFunctions { + + import ScalaObjectMapperTest._ + + /* Class under test */ + protected val mapper: ScalaObjectMapper = ScalaObjectMapper() + + private final val steve: Person = + Person(id = 1, name = "Steve", age = Some(20), age_with_default = Some(20), nickname = "ace") + private final val steveJson: String = + """{ + "id" : 1, + "name" : "Steve", + "age" : 20, + "age_with_default" : 20, + "nickname" : "ace" + } + """ + + override def beforeAll(): Unit = { + super.beforeAll() + mapper.registerModule(ScalaObjectMapperTest.MixInAnnotationsModule) + } + + test("constructors") { + // new mapper with defaults + assertNotNull(ScalaObjectMapper()) + assertNotNull(ScalaObjectMapper.apply()) + + // augments the given mapper with the defaults + assertNotNull(ScalaObjectMapper(mapper.underlying)) + assertNotNull(ScalaObjectMapper.apply(mapper.underlying)) + + // copies the underlying Jackson object mapper + assertNotNull(ScalaObjectMapper.objectMapper(mapper.underlying)) + // underlying mapper needs to have a JsonFactory of type YAMLFactory + assertThrows[IllegalArgumentException](ScalaObjectMapper.yamlObjectMapper(mapper.underlying)) + assertNotNull( + ScalaObjectMapper.yamlObjectMapper(ScalaObjectMapper.builder.yamlObjectMapper.underlying)) + assertNotNull(ScalaObjectMapper.camelCaseObjectMapper(mapper.underlying)) + assertNotNull(ScalaObjectMapper.snakeCaseObjectMapper(mapper.underlying)) + + assertThrows[AssertionError](new ScalaObjectMapper(null)) + } + + test("builder constructors") { + assertNotNull(ScalaObjectMapper.builder.objectMapper) + assertNotNull(ScalaObjectMapper.builder.objectMapper(new JsonFactory)) + assertNotNull(ScalaObjectMapper.builder.objectMapper(new YAMLFactory)) + + assertNotNull(ScalaObjectMapper.builder.yamlObjectMapper) + assertNotNull(ScalaObjectMapper.builder.camelCaseObjectMapper) + assertNotNull(ScalaObjectMapper.builder.snakeCaseObjectMapper) + } + + test("mapper register module") { + val testMapper = ScalaObjectMapper() + + val simpleJacksonModule = new SimpleModule() + simpleJacksonModule.addDeserializer(classOf[ZeroOrOne], new ZeroOrOneDeserializer) + testMapper.registerModule(simpleJacksonModule) + + // regular mapper (without the custom deserializer) -- doesn't parse + intercept[JsonMappingException] { + mapper.parse[CaseClassWithZeroOrOne]("{\"id\" :\"zero\"}") + } + + // mapper with custom deserializer -- parses correctly + testMapper.parse[CaseClassWithZeroOrOne]("{\"id\" :\"zero\"}") should be( + CaseClassWithZeroOrOne(Zero)) + testMapper.parse[CaseClassWithZeroOrOne]("{\"id\" :\"one\"}") should be( + CaseClassWithZeroOrOne(One)) + intercept[JsonMappingException] { + testMapper.parse[CaseClassWithZeroOrOne]("{\"id\" :\"two\"}") + } + } + + test("inject request field fails with a mapping exception") { + // with no InjectableValues we get a CaseClassMappingException since we can't map + // the JSON into the case class. + intercept[CaseClassMappingException] { + parse[ClassWithQueryParamDateTimeInject]("""{}""") + } + } + + test( + "class with an injectable field fails with a mapping exception when it cannot be parsed from JSON") { + // if there is no injector, the default injectable values is not configured, thus the code + // tries to construct the asked for type from the given JSON, which is empty and fails with + // a mapping exception. + intercept[CaseClassMappingException] { + parse[ClassWithFooClassInject]("""{}""") + } + } + + test("class with an injectable field is not constructed from JSON") { + // no field injection configured and parsing proceeds as normal + val result = parse[ClassWithFooClassInject]( + """ + |{ + | "foo_class": { + | "id" : "11" + | } + |}""".stripMargin + ) + result.fooClass should equal(FooClass("11")) + } + + test("class with a defaulted injectable field is constructed from the default") { + // no field injection configured and default value is not used since a value is passed in the JSON. + val result = parse[ClassWithFooClassInjectAndDefault]( + """ + |{ + | "foo_class": { + | "id" : "1" + | } + |}""".stripMargin + ) + + result.fooClass should equal(FooClass("1")) + } + + test("injectable field should use default when field not sent in json") { + parse[CaseClassInjectStringWithDefault]("""{}""") should equal( + CaseClassInjectStringWithDefault("DefaultHello")) + } + + test("injectable field should not use default when field sent in json") { + parse[CaseClassInjectStringWithDefault]("""{"string": "123"}""") should equal( + CaseClassInjectStringWithDefault("123")) + } + + test("injectable field should use None assumed default when field not sent in json") { + parse[CaseClassInjectOptionString]("""{}""") should equal(CaseClassInjectOptionString(None)) + } + + test("injectable field should throw exception when no value is passed in the json") { + intercept[CaseClassMappingException] { + parse[CaseClassInjectString]("""{}""") + } + } + + test("regular mapper handles unknown properties when json provides MORE fields than case class") { + // regular mapper -- doesn't fail + mapper.parse[CaseClass]( + """ + |{ + | "id": 12345, + | "name": "gadget", + | "extra": "fail" + |} + |""".stripMargin + ) + + // mapper = loose, case class = annotated strict --> Fail + intercept[JsonMappingException] { + mapper.parse[StrictCaseClass]( + """ + |{ + | "id": 12345, + | "name": "gadget", + | "extra": "fail" + |} + |""".stripMargin + ) + } + } + + test("regular mapper handles unknown properties when json provides LESS fields than case class") { + // regular mapper -- doesn't fail + mapper.parse[CaseClassWithOption]( + """ + |{ + | "value": 12345, + | "extra": "fail" + |} + |""".stripMargin + ) + + // mapper = loose, case class = annotated strict --> Fail + intercept[JsonMappingException] { + mapper.parse[StrictCaseClassWithOption]( + """ + |{ + | "value": 12345, + | "extra": "fail" + |} + |""".stripMargin + ) + } + } + + test("regular mapper handles unknown properties") { + // regular mapper -- doesn't fail (no field named 'flame') + mapper.parse[CaseClassIdAndOption]( + """ + |{ + | "id": 12345, + | "flame": "gadget" + |} + |""".stripMargin + ) + + // mapper = loose, case class = annotated strict --> Fail (no field named 'flame') + intercept[JsonMappingException] { + mapper.parse[StrictCaseClassIdAndOption]( + """ + |{ + | "id": 12345, + | "flame": "gadget" + |} + |""".stripMargin + ) + } + } + + test("mapper with deserialization config fails on unknown properties") { + val testMapper = + ScalaObjectMapper.builder + .withDeserializationConfig(Map(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES -> true)) + .objectMapper + + // mapper = strict, case class = unannotated --> Fail + intercept[JsonMappingException] { + testMapper.parse[CaseClass]( + """ + |{ + | "id": 12345, + | "name": "gadget", + | "extra": "fail" + |} + |""".stripMargin + ) + } + + // mapper = strict, case class = annotated strict --> Fail + intercept[JsonMappingException] { + testMapper.parse[StrictCaseClass]( + """ + |{ + | "id": 12345, + | "name": "gadget", + | "extra": "fail" + |} + |""".stripMargin + ) + } + + // mapper = strict, case class = annotated loose --> Parse + testMapper.parse[LooseCaseClass]( + """ + |{ + | "id": 12345, + | "name": "gadget", + | "extra": "pass" + |} + |""".stripMargin + ) + } + + test("mapper with additional configuration handles unknown properties") { + // test with additional configuration set on mapper + val testMapper = + ScalaObjectMapper.builder + .withAdditionalMapperConfigurationFn( + _.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true)) + .objectMapper + + // mapper = strict, case class = unannotated --> Fail + intercept[JsonMappingException] { + testMapper.parse[CaseClass]( + """ + |{ + | "id": 12345, + | "name": "gadget", + | "extra": "fail" + |} + |""".stripMargin + ) + } + + // mapper = strict, case class = annotated strict --> Fail + intercept[JsonMappingException] { + testMapper.parse[StrictCaseClass]( + """ + |{ + | "id": 12345, + | "name": "gadget", + | "extra": "fail" + |} + |""".stripMargin + ) + } + + // mapper = strict, case class = annotated loose --> Parse + testMapper.parse[LooseCaseClass]( + """ + |{ + | "id": 12345, + | "name": "gadget", + | "extra": "pass" + |} + |""".stripMargin + ) + } + + test("support wrapped object mapper") { + val person = CamelCaseSimplePersonNoAnnotation(myName = "Bob") + + val jacksonScalaObjectMapper: JacksonScalaObjectMapperType = new JacksonObjectMapper() + with JacksonScalaObjectMapper + jacksonScalaObjectMapper.registerModule(DefaultScalaModule) + jacksonScalaObjectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.LOWER_CAMEL_CASE) + + // default for ScalaObjectMapper#Builder is snake_case which should not get applied here + // since this method only wraps and does not mutate the underlying mapper + val objectMapper: ScalaObjectMapper = ScalaObjectMapper.objectMapper(jacksonScalaObjectMapper) + + val serialized = """{"myName":"Bob"}""" + // serialization -- they should each serialize the same way + jacksonScalaObjectMapper.writeValueAsString(person) should equal(serialized) + objectMapper.writeValueAsString(person) should equal(serialized) + jacksonScalaObjectMapper.writeValueAsString(person) should equal( + objectMapper.writeValueAsString(person)) + // deserialization -- each should be able to deserialize the other's written representation + objectMapper.parse[CamelCaseSimplePersonNoAnnotation]( + jacksonScalaObjectMapper.writeValueAsString(person)) should equal(person) + jacksonScalaObjectMapper.readValue[CamelCaseSimplePersonNoAnnotation]( + objectMapper.writeValueAsString(person)) should equal(person) + objectMapper.parse[CamelCaseSimplePersonNoAnnotation](serialized) should equal( + jacksonScalaObjectMapper.readValue[CamelCaseSimplePersonNoAnnotation](serialized)) + } + + test("support camel case mapper") { + val camelCaseObjectMapper = ScalaObjectMapper.camelCaseObjectMapper(mapper.underlying) + + camelCaseObjectMapper.parse[Map[String, String]]("""{"firstName": "Bob"}""") should equal( + Map("firstName" -> "Bob") + ) + } + + test("support snake case mapper") { + val snakeCaseObjectMapper = ScalaObjectMapper.snakeCaseObjectMapper(mapper.underlying) + + val person = CamelCaseSimplePersonNoAnnotation(myName = "Bob") + + val serialized = snakeCaseObjectMapper.writeValueAsString(person) + serialized should equal("""{"my_name":"Bob"}""") + snakeCaseObjectMapper.parse[CamelCaseSimplePersonNoAnnotation](serialized) should equal(person) + } + + test("support yaml mapper") { + val yamlObjectMapper = ScalaObjectMapper.builder.yamlObjectMapper + + val person = CamelCaseSimplePersonNoAnnotation(myName = "Bob") + + val serialized = yamlObjectMapper.writeValueAsString(person) + // default PropertyNamingStrategy for the generate YAML object mapper is snake_case + serialized should equal("""--- + |my_name: "Bob" + |""".stripMargin) + yamlObjectMapper.parse[CamelCaseSimplePersonNoAnnotation](serialized) should equal(person) + + // but we can also update to a camelCase PropertyNamingStrategy: + val camelCaseObjectMapper = ScalaObjectMapper.camelCaseObjectMapper(yamlObjectMapper.underlying) + val serializedCamelCase = camelCaseObjectMapper.writeValueAsString(person) + serializedCamelCase should equal("""--- + |myName: "Bob" + |""".stripMargin) + camelCaseObjectMapper.parse[CamelCaseSimplePersonNoAnnotation]( + serializedCamelCase) should equal(person) + } + + test("Scala enums") { + val expectedInstance = BasicDate(Month.Feb, 29, 2020, Weekday.Sat) + val expectedStr = """{"month":"Feb","day":29,"year":2020,"weekday":"Sat"}""" + parse[BasicDate](expectedStr) should equal(expectedInstance) // deser + generate(expectedInstance) should equal(expectedStr) // ser + + // test with default scala module + val defaultScalaObjectMapper = new JacksonObjectMapper with JacksonScalaObjectMapper + defaultScalaObjectMapper.registerModule(DefaultScalaModule) + defaultScalaObjectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) + deserialize[BasicDate](defaultScalaObjectMapper, expectedStr) should equal(expectedInstance) + + // case insensitive mapper feature does not pertain to scala enumerations (only Java enums) + val caseInsensitiveEnumMapper = + ScalaObjectMapper.builder + .withAdditionalMapperConfigurationFn( + _.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true)) + .objectMapper + + val withErrors212 = Seq("month: key not found: feb", "weekday: key not found: sat") + val withErrors213 = Seq("month: null", "weekday: null") + + val expectedCaseInsensitiveStr = """{"month":"feb","day":29,"year":2020,"weekday":"sat"}""" + val e = intercept[CaseClassMappingException] { + caseInsensitiveEnumMapper.parse[BasicDate](expectedCaseInsensitiveStr) + } + val actualMessages = e.errors.map(_.getMessage) + assert( + actualMessages.mkString(",") == withErrors212.mkString(",") || + actualMessages.mkString(",") == withErrors213.mkString(",")) + + // non-existent values + val withErrors2212 = Seq("month: key not found: Nnn", "weekday: key not found: Flub") + val withErrors2213 = Seq("month: null", "weekday: null") + + val expectedStr2 = """{"month":"Nnn","day":29,"year":2020,"weekday":"Flub"}""" + val e2 = intercept[CaseClassMappingException] { + parse[BasicDate](expectedStr2) + } + val actualMessages2 = e2.errors.map(_.getMessage) + assert( + actualMessages2.mkString(",") == withErrors2212.mkString(",") || + actualMessages2.mkString(",") == withErrors2213.mkString(",")) + + val invalidWithOptionalScalaEnumerationValue = """{"month":"Nnn"}""" + val e3 = intercept[CaseClassMappingException] { + parse[WithOptionalScalaEnumeration](invalidWithOptionalScalaEnumerationValue) + } + e3.errors.size shouldEqual 1 + e3.errors.head.getMessage match { + case "month: key not found: Nnn" => // Scala 2.12.x (https://github.com/scala/scala/blob/2.12.x/src/library/scala/collection/MapLike.scala#L236) + case "month: null" => // Scala 2.13.1 (https://github.com/scala/scala/blob/2.13.x/src/library/scala/collection/immutable/HashMap.scala#L635) + case _ => fail() + } + e3.errors.head.reason.message match { + case "key not found: Nnn" => // Scala 2.12.x (https://github.com/scala/scala/blob/2.12.x/src/library/scala/collection/MapLike.scala#L236_ + case null => // Scala 2.13.1 (https://github.com/scala/scala/blob/2.13.x/src/library/scala/collection/immutable/HashMap.scala#L635) + case _ => fail() + } + e3.errors.head.reason.detail shouldEqual Unspecified + + val validWithOptionalScalaEnumerationValue = """{"month":"Apr"}""" + val validWithOptionalScalaEnumeration = + parse[WithOptionalScalaEnumeration](validWithOptionalScalaEnumerationValue) + validWithOptionalScalaEnumeration.month.isDefined shouldBe true + validWithOptionalScalaEnumeration.month.get shouldEqual Month.Apr + } + + // based on Jackson test to ensure compatibility: + // https://github.com/FasterXML/jackson-module-scala/blob/fa7cf702e0f61467d726384af88de9ea1f798b97/src/test/scala/com/fasterxml/jackson/module/scala/deser/CaseClassDeserializerTest.scala#L78-L81 + test("deserialization#generic types") { + parse[GenericTestCaseClass[Int]]("""{"data" : 3}""") should equal(GenericTestCaseClass(3)) + parse[GenericTestCaseClass[CaseClass]]("""{"data" : {"id" : 123, "name" : "foo"}}""") should + equal(GenericTestCaseClass(CaseClass(123, "foo"))) + parse[CaseClassWithGeneric[CaseClass]]( + """{"inside" : { "data" : {"id" : 123, "name" : "foo"}}}""") should + equal(CaseClassWithGeneric(GenericTestCaseClass(CaseClass(123, "foo")))) + + parse[GenericTestCaseClass[String]]("""{"data" : "Hello, World"}""") should equal( + GenericTestCaseClass("Hello, World")) + parse[GenericTestCaseClass[Double]]("""{"data" : 3.14}""") should equal( + GenericTestCaseClass(3.14d)) + + parse[CaseClassWithOptionalGeneric[Int]]("""{"inside": {"data": 3}}""") should equal( + CaseClassWithOptionalGeneric[Int](Some(GenericTestCaseClass[Int](3)))) + + parse[CaseClassWithTypes[String, Int]]("""{"first": "Bob", "second" : 42}""") should equal( + CaseClassWithTypes("Bob", 42)) + parse[CaseClassWithTypes[Int, Float]]("""{"first": 127, "second" : 39.0}""") should equal( + CaseClassWithTypes(127, 39.0f)) + + parse[CaseClassWithMapTypes[String, Float]]( + """{"data": {"pi": 3.14, "inverse fine structure constant": 137.035}}""") should equal( + CaseClassWithMapTypes( + Map[String, Float]("pi" -> 3.14f, "inverse fine structure constant" -> 137.035f)) + ) + parse[CaseClassWithManyTypes[Int, Float, String]]( + """{"one": 1, "two": 3.1, "three": "Hello, World!"}""") should equal( + CaseClassWithManyTypes(1, 3.1f, "Hello, World!")) + + val result = Page( + List( + Person(1, "Bob Marley", None, Some(32), Some(32), "Music Master"), + Person(2, "Jimi Hendrix", None, Some(27), None, "Melody Man")), + 5, + None, + None) + val input = + """ + |{ + | "data": [ + | {"id": 1, "name": "Bob Marley", "age": 32, "age_with_default": 32, "nickname": "Music Master"}, + | {"id": 2, "name": "Jimi Hendrix", "age": 27, "nickname": "Melody Man"} + | ], + | "page_size": 5 + |} + """.stripMargin + + // test with default scala module + val defaultScalaObjectMapper = new JacksonObjectMapper with JacksonScalaObjectMapper + defaultScalaObjectMapper.registerModule(DefaultScalaModule) + defaultScalaObjectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) + deserialize[Page[Person]](defaultScalaObjectMapper, input) should equal(result) + + // test with ScalaObjectMapper + deserialize[Page[Person]](mapper.underlying, input) should equal(result) + } + + // tests for GH #547 + test("deserialization#generic types 2") { + val aGeneric = mapper.parse[AGeneric[B]]("""{"b":{"value":"string"}}""") + aGeneric.b.map(_.value) should equal(Some("string")) + + val c = mapper.parse[C]("""{"a":{"b":{"value":"string"}}}""") + c.a.b.map(_.value) should equal(Some("string")) + + val aaGeneric = mapper.parse[AAGeneric[B, Int]]("""{"b":{"value":"string"},"c":42}""") + aaGeneric.b.map(_.value) should equal(Some("string")) + aaGeneric.c.get should equal(42) + + val aaGeneric2 = mapper.parse[AAGeneric[B, D]]("""{"b":{"value":"string"},"c":{"value":42}}""") + aaGeneric2.b.map(_.value) should equal(Some("string")) + aaGeneric2.c.map(_.value) should equal(Some(42)) + + val eGeneric = + mapper.parse[E[B, D, Int]]("""{"a": {"b":{"value":"string"},"c":{"value":42}}, "b":42}""") + eGeneric.a.b.map(_.value) should equal(Some("string")) + eGeneric.a.c.map(_.value) should equal(Some(42)) + eGeneric.b should equal(Some(42)) + + val fGeneric = + mapper.parse[F[B, Int, D, AGeneric[B], Int, String]](""" + |{ + | "a":{"value":"string"}, + | "b":[1,2,3], + | "c":{"value":42}, + | "d":{"b":{"value":"string"}}, + | "e":{"r":"forty-two"} + |} + |""".stripMargin) + fGeneric.a.map(_.value) should equal(Some("string")) + fGeneric.b should equal(Seq(1, 2, 3)) + fGeneric.c.map(_.value) should equal(Some(42)) + fGeneric.d.b.map(_.value) should equal(Some("string")) + fGeneric.e should equal(Right("forty-two")) + + val fGeneric2 = + mapper.parse[F[B, Int, D, AGeneric[B], Int, String]](""" + |{ + | "a":{"value":"string"}, + | "b":[1,2,3], + | "c":{"value":42}, + | "d":{"b":{"value":"string"}}, + | "e":{"l":42} + |} + |""".stripMargin) + fGeneric2.a.map(_.value) should equal(Some("string")) + fGeneric2.b should equal(Seq(1, 2, 3)) + fGeneric2.c.map(_.value) should equal(Some(42)) + fGeneric2.d.b.map(_.value) should equal(Some("string")) + fGeneric2.e should equal(Left(42)) + + // requires polymorphic handling with @JsonTypeInfo + val withTypeBounds = + mapper.parse[WithTypeBounds[BoundaryA]](""" + |{ + | "a": {"type":"a", "value":"Guineafowl"} + |} + |""".stripMargin) + withTypeBounds.a.map(_.value) should equal(Some("Guineafowl")) + + // uses a specific json deserializer + val someNumberType = mapper.parse[SomeNumberType[java.lang.Integer]]( + """ + |{ + | "n": 42 + |} + |""".stripMargin + ) + someNumberType.n should equal(Some(42)) + + // multi-parameter non-collection type + val gGeneric = + mapper.parse[G[B, Int, D, AGeneric[B], Int, String]](""" + |{ + | "gg": { + | "t":{"value":"string"}, + | "u":42, + | "v":{"value":42}, + | "x":{"b":{"value":"string"}}, + | "y":137, + | "z":"string" + | } + |} + |""".stripMargin) + gGeneric.gg.t.value should be("string") + gGeneric.gg.u should equal(42) + gGeneric.gg.v.value should equal(42) + gGeneric.gg.x.b.map(_.value) should equal(Some("string")) + gGeneric.gg.y should equal(137) + gGeneric.gg.z should be("string") + } + + test("JsonProperty#annotation inheritance") { + val aumJson = """{"i":1,"j":"J"}""" + val aum = parse[Aum](aumJson) + aum should equal(Aum(1, "J")) + mapper.writeValueAsString(Aum(1, "J")) should equal(aumJson) + + val testCaseClassJson = """{"fedoras":["felt","straw"],"oldness":27}""" + val testCaseClass = parse[CaseClassTraitImpl](testCaseClassJson) + testCaseClass should equal(CaseClassTraitImpl(Seq("felt", "straw"), 27)) + mapper.writeValueAsString(CaseClassTraitImpl(Seq("felt", "straw"), 27)) should equal( + testCaseClassJson) + } + + test("simple tests#parse super simple") { + val foo = parse[SimplePerson]("""{"name": "Steve"}""") + foo should equal(SimplePerson("Steve")) + } + + test("simple tests#parse super simple -- YAML") { + val yamlMapper = ScalaObjectMapper.builder.yamlObjectMapper + val foo = yamlMapper.parse[SimplePerson]("""name: Steve""") + foo should equal(SimplePerson("Steve")) + + yamlMapper.writeValueAsString(foo) should equal("""--- + |name: "Steve" + |""".stripMargin) + } + + test("get PropertyNamingStrategy") { + val namingStrategy = mapper.propertyNamingStrategy + namingStrategy should not be null + } + + test("simple tests#parse simple") { + val foo = parse[SimplePerson]("""{"name": "Steve"}""") + foo should equal(SimplePerson("Steve")) + } + + test("simple tests#parse CamelCase simple person") { + val foo = parse[CamelCaseSimplePerson]("""{"myName": "Steve"}""") + foo should equal(CamelCaseSimplePerson("Steve")) + } + + test("simple tests#parse json") { + val person = parse[Person](steveJson) + person should equal(steve) + } + + test("simple tests#parse json list of objects") { + val json = Seq(steveJson, steveJson).mkString("[", ", ", "]") + val persons = parse[Seq[Person]](json) + persons should equal(Seq(steve, steve)) + } + + test("simple tests#parse json list of ints") { + val nums = parse[Seq[Int]]("""[1,2,3]""") + nums should equal(Seq(1, 2, 3)) + } + + test("simple tests#serialize case class with logging") { + val steveWithLogging = PersonWithLogging( + id = 1, + name = "Steve", + age = Some(20), + age_with_default = Some(20), + nickname = "ace" + ) + + assertJson(steveWithLogging, steveJson) + } + + test("simple tests#parse json with extra field at end") { + val person = parse[Person](""" + { + "id" : 1, + "name" : "Steve", + "age" : 20, + "age_with_default" : 20, + "nickname" : "ace", + "extra" : "extra" + } + """) + person should equal(steve) + } + + test("simple tests#parse json with extra field in middle") { + val person = parse[Person](""" + { + "id" : 1, + "name" : "Steve", + "age" : 20, + "extra" : "extra", + "age_with_default" : 20, + "nickname" : "ace" + } + """) + person should equal(steve) + } + + test("simple tests#parse json with extra field name with dot") { + val person = parse[PersonWithDottedName](""" + { + "id" : 1, + "name.last" : "Cosenza" + } + """) + + person should equal( + PersonWithDottedName( + id = 1, + lastName = "Cosenza" + ) + ) + } + + test("simple tests#parse json with missing 'id' and 'name' field and invalid age field") { + assertJsonParse[Person]( + """ { + "age" : "foo", + "age_with_default" : 20, + "nickname" : "ace" + }""", + withErrors = + Seq("age: 'foo' is not a valid Integer", "id: field is required", "name: field is required") + ) + } + + test("simple tests#parse nested json with missing fields") { + assertJsonParse[Car]( + """ + { + "id" : 0, + "make": "Foo", + "year": 2000, + "passengers" : [ { "id": "-1", "age": "blah" } ] + } + """, + withErrors = Seq( + "make: 'Foo' is not a valid CarMake with valid values: Ford, Honda", + "model: field is required", + "owners: field is required", + "passengers.age: 'blah' is not a valid Integer", + "passengers.name: field is required" + ) + ) + } + + test("simple test#parse char") { + assertJsonParse[CaseClassCharacter]( + """ + |{ + |"c" : -1 + |} + """.stripMargin, + withErrors = Seq( + "c: '' is not a valid Character" + ) + ) + } + + test("simple tests#parse json with missing 'nickname' field that has a string default") { + val person = parse[Person](""" + { + "id" : 1, + "name" : "Steve", + "age" : 20, + "age_with_default" : 20 + }""") + person should equal(steve.copy(nickname = "unknown")) + } + + test( + "simple tests#parse json with missing 'age' field that is an Option without a default should succeed" + ) { + parse[Person](""" + { + "id" : 1, + "name" : "Steve", + "age_with_default" : 20, + "nickname" : "bob" + } + """) + } + + test("simple tests#parse json into JsonNode") { + parse[JsonNode](steveJson) + } + + test("simple tests#generate json") { + assertJson(steve, steveJson) + } + + test("simple tests#generate then parse") { + val json = generate(steve) + val person = parse[Person](json) + person should equal(steve) + } + + test("simple tests#generate then parse Either type") { + type T = Either[String, Int] + val l: T = Left("Q?") + val r: T = Right(42) + assertJson(l, """{"l":"Q?"}""") + assertJson(r, """{"r":42}""") + } + + test("simple tests#generate then parse nested case class") { + val origCar = Car(1, CarMake.Ford, "Explorer", 2001, Seq(steve, steve)) + val carJson = generate(origCar) + val car = parse[Car](carJson) + car should equal(origCar) + } + + test("simple tests#Prevent overwriting val in case class") { + parse[CaseClassWithVal]("""{ + "name" : "Bob", + "type" : "dog" + }""") should equal(CaseClassWithVal("Bob")) + } + + test("simple tests#parse WithEmptyJsonProperty then write and see if it equals original") { + val withEmptyJsonProperty = + """{ + | "foo" : "abc" + |}""".stripMargin + val obj = parse[WithEmptyJsonProperty](withEmptyJsonProperty) + val json = mapper.writePrettyString(obj) + json should equal(withEmptyJsonProperty) + } + + test("simple tests#parse WithNonemptyJsonProperty then write and see if it equals original") { + val withNonemptyJsonProperty = + """{ + | "bar" : "abc" + |}""".stripMargin + val obj = parse[WithNonemptyJsonProperty](withNonemptyJsonProperty) + val json = mapper.writePrettyString(obj) + json should equal(withNonemptyJsonProperty) + } + + test( + "simple tests#parse WithoutJsonPropertyAnnotation then write and see if it equals original") { + val withoutJsonPropertyAnnotation = + """{ + | "foo" : "abc" + |}""".stripMargin + val obj = parse[WithoutJsonPropertyAnnotation](withoutJsonPropertyAnnotation) + val json = mapper.writePrettyString(obj) + json should equal(withoutJsonPropertyAnnotation) + } + + test( + "simple tests#use default Jackson mapper without setting naming strategy to see if it remains camelCase to verify default Jackson behavior" + ) { + val objMapper = new JacksonObjectMapper with JacksonScalaObjectMapper + objMapper.registerModule(DefaultScalaModule) + + val response = objMapper.writeValueAsString(NamingStrategyJsonProperty("abc")) + response should equal("""{"longFieldName":"abc"}""") + } + + test( + "simple tests#use default Jackson mapper after setting naming strategy and see if it changes to verify default Jackson behavior" + ) { + val objMapper = new JacksonObjectMapper with JacksonScalaObjectMapper + objMapper.registerModule(DefaultScalaModule) + objMapper.setPropertyNamingStrategy(new PropertyNamingStrategy.SnakeCaseStrategy) + + val response = objMapper.writeValueAsString(NamingStrategyJsonProperty("abc")) + response should equal("""{"long_field_name":"abc"}""") + } + + test("enums#simple") { + parse[CaseClassWithEnum]("""{ + "name" : "Bob", + "make" : "ford" + }""") should equal(CaseClassWithEnum("Bob", CarMakeEnum.ford)) + } + + test("enums#complex") { + JsonDiff.assertDiff( + CaseClassWithComplexEnums( + "Bob", + CarMakeEnum.vw, + Some(CarMakeEnum.ford), + Seq(CarMakeEnum.vw, CarMakeEnum.ford), + Set(CarMakeEnum.ford, CarMakeEnum.vw) + ), + parse[CaseClassWithComplexEnums]("""{ + "name" : "Bob", + "make" : "vw", + "make_opt" : "ford", + "make_seq" : ["vw", "ford"], + "make_set" : ["ford", "vw"] + }""") + ) + } + + test("enums#complex case insensitive") { + val caseInsensitiveEnumMapper = + ScalaObjectMapper.builder + .withAdditionalMapperConfigurationFn( + _.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true)) + .objectMapper + + JsonDiff.assertDiff( + CaseClassWithComplexEnums( + "Bob", + CarMakeEnum.vw, + Some(CarMakeEnum.ford), + Seq(CarMakeEnum.vw, CarMakeEnum.ford), + Set(CarMakeEnum.ford, CarMakeEnum.vw) + ), + caseInsensitiveEnumMapper.parse[CaseClassWithComplexEnums]("""{ + "name" : "Bob", + "make" : "VW", + "make_opt" : "Ford", + "make_seq" : ["vW", "foRD"], + "make_set" : ["fORd", "Vw"] + }""") + ) + } + + test("enums#invalid enum entry") { + val e = intercept[CaseClassMappingException] { + parse[CaseClassWithEnum]("""{ + "name" : "Bob", + "make" : "foo" + }""") + } + e.errors.map { _.getMessage } should equal( + Seq("""make: 'foo' is not a valid CarMakeEnum with valid values: ford, vw""") + ) + } + + test("LocalDateTime support") { + parse[CaseClassWithDateTime]("""{ + "date_time" : "2014-05-30T03:57:59.302Z" + }""") should equal( + CaseClassWithDateTime( + LocalDateTime.parse("2014-05-30T03:57:59.302Z", ISO8601DateTimeFormatter)) + ) + } + + test("invalid LocalDateTime 1") { + assertJsonParse[CaseClassWithDateTime]( + """{ + "date_time" : "" + }""", + withErrors = Seq("date_time: error parsing ''")) + } + + test("invalid LocalDateTime 2") { + assertJsonParse[CaseClassWithIntAndDateTime]( + """{ + "name" : "Bob", + "age" : "old", + "age2" : "1", + "age3" : "", + "date_time" : "today", + "date_time2" : "1", + "date_time3" : -1, + "date_time4" : "" + }""", + withErrors = Seq( + "age3: '' is not a valid Integer", + "age: 'old' is not a valid Integer", + "date_time2: '1' is not a valid LocalDateTime", + "date_time3: '' is not a valid LocalDateTime", + "date_time4: error parsing ''", + "date_time: 'today' is not a valid LocalDateTime" + ) + ) + } + + test("TwitterUtilDuration serialize") { + val serialized = mapper.writeValueAsString( + CaseClassWithTwitterUtilDuration(Duration.fromTimeUnit(3L, TimeUnit.HOURS)) + ) + serialized should equal("""{"duration":"3.hours"}""") + } + + test("TwitterUtilDuration deserialize") { + parse[CaseClassWithTwitterUtilDuration]("""{ + "duration": "3.hours" + }""") should equal( + CaseClassWithTwitterUtilDuration(Duration.fromTimeUnit(3L, TimeUnit.HOURS)) + ) + } + + test("TwitterUtilDuration deserialize invalid") { + assertJsonParse[CaseClassWithTwitterUtilDuration]( + """{"duration": "3.unknowns"}""", + withErrors = Seq("duration: Invalid unit: unknowns") + ) + } + + test("escaped fields#long") { + parse[CaseClassWithEscapedLong]("""{ + "1-5" : 10 + }""") should equal(CaseClassWithEscapedLong(`1-5` = 10)) + } + + test("escaped fields#string") { + parse[CaseClassWithEscapedString]("""{ + "1-5" : "10" + }""") should equal(CaseClassWithEscapedString(`1-5` = "10")) + } + + test("escaped fields#non-unicode escaped") { + parse[CaseClassWithEscapedNormalString]("""{ + "a" : "foo" + }""") should equal(CaseClassWithEscapedNormalString("foo")) + } + + test("escaped fields#unicode and non-unicode fields") { + parse[UnicodeNameCaseClass]("""{"winning-id":23,"name":"the name of this"}""") should equal( + UnicodeNameCaseClass(23, "the name of this") + ) + } + + test("wrapped values#support value class") { + val orig = WrapperValueClass(1) + val obj = parse[WrapperValueClass](generate(orig)) + orig should equal(obj) + } + + test("wrapped values#direct WrappedValue for Int") { + val origObj = WrappedValueInt(1) + val obj = parse[WrappedValueInt](generate(origObj)) + origObj should equal(obj) + } + + test("wrapped values#direct WrappedValue for String") { + val origObj = WrappedValueString("1") + val obj = parse[WrappedValueString](generate(origObj)) + origObj should equal(obj) + } + + test( + "wrapped values#direct WrappedValue for String when asked to parse wrapped json object should throw exception" + ) { + intercept[JsonMappingException] { + parse[WrappedValueString]("""{"value": "1"}""") + } + } + + test("wrapped values#direct WrappedValue for Long") { + val origObj = WrappedValueLong(1) + val obj = parse[WrappedValueLong](generate(origObj)) + origObj should equal(obj) + } + + test("wrapped values#WrappedValue for Int") { + val origObj = WrappedValueIntInObj(WrappedValueInt(1)) + val json = mapper.writeValueAsString(origObj) + val obj = parse[WrappedValueIntInObj](json) + origObj should equal(obj) + } + + test("wrapped values#WrappedValue for String") { + val origObj = WrappedValueStringInObj(WrappedValueString("1")) + val json = mapper.writeValueAsString(origObj) + val obj = parse[WrappedValueStringInObj](json) + origObj should equal(obj) + } + + test("wrapped values#WrappedValue for Long") { + val origObj = WrappedValueLongInObj(WrappedValueLong(11111111)) + val json = mapper.writeValueAsString(origObj) + val obj = parse[WrappedValueLongInObj](json) + origObj should equal(obj) + } + + test("wrapped values#Seq[WrappedValue]") { + generate(Seq(WrappedValueLong(11111111))) should be("""[11111111]""") + } + + test("wrapped values#Map[WrappedValueString, String]") { + val obj = Map(WrappedValueString("11111111") -> "asdf") + val json = generate(obj) + json should be("""{"11111111":"asdf"}""") + parse[Map[WrappedValueString, String]](json) should be(obj) + } + + test("wrapped values#Map[WrappedValueLong, String]") { + assertJson(Map(WrappedValueLong(11111111) -> "asdf"), """{"11111111":"asdf"}""") + } + + test("wrapped values#deser Map[Long, String]") { + val obj = parse[Map[Long, String]]("""{"11111111":"asdf"}""") + val expected = Map(11111111L -> "asdf") + obj should equal(expected) + } + + test("wrapped values#deser Map[String, String]") { + parse[Map[String, String]]("""{"11111111":"asdf"}""") should + be(Map("11111111" -> "asdf")) + } + + test("wrapped values#Map[String, WrappedValueLong]") { + generate(Map("asdf" -> WrappedValueLong(11111111))) should be("""{"asdf":11111111}""") + } + + test("wrapped values#Map[String, WrappedValueString]") { + generate(Map("asdf" -> WrappedValueString("11111111"))) should be("""{"asdf":"11111111"}""") + } + + test("wrapped values#object with Map[WrappedValueString, String]") { + assertJson( + obj = WrappedValueStringMapObject(Map(WrappedValueString("11111111") -> "asdf")), + expected = """{ + "map" : { + "11111111":"asdf" + } + }""" + ) + } + + test("fail when CaseClassWithSeqLongs with null array element") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqOfLongs]("""{"seq": [null]}""") + } + } + + test("fail when CaseClassWithSeqWrappedValueLong with null array element") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqWrappedValueLong]("""{"seq": [null]}""") + } + } + + test("fail when CaseClassWithArrayWrappedValueLong with null array element") { + intercept[CaseClassMappingException] { + parse[CaseClassWithArrayWrappedValueLong]("""{"array": [null]}""") + } + } + + test("fail when CaseClassWithSeqWrappedValueLongWithValidation with null array element") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqWrappedValueLongWithValidation]("""{"seq": [null]}""") + } + } + + test("fail when CaseClassWithSeqWrappedValueLongWithValidation with invalid field") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqWrappedValueLongWithValidation]("""{"seq": [{"value": 0}]}""") + } + } + + test("fail when CaseClassWithSeqOfCaseClassWithValidation with null array element") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [null]}""") + } + } + + test("fail when CaseClassWithSeqOfCaseClassWithValidation with invalid array element") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [0]}""") + } + } + + test("fail when CaseClassWithSeqOfCaseClassWithValidation with null field in object") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [{"value": null}]}""") + } + } + + test("fail when CaseClassWithSeqOfCaseClassWithValidation with invalid field in object") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [{"value": 0}]}""") + } + } + + test("fail when CaseClassWithArrayLong with null field in object") { + intercept[CaseClassMappingException] { + parse[CaseClassWithArrayLong]("""{"array": [null]}""") + } + } + + test("fail when CaseClassWithArrayBoolean with null field in object") { + intercept[CaseClassMappingException] { + parse[CaseClassWithArrayBoolean]("""{"array": [null]}""") + } + } + + // ======================================================== + // Jerkson Inspired/Copied Tests Below + test("A basic case class generates a JSON object with matching field values") { + generate(CaseClass(1, "Coda")) should be("""{"id":1,"name":"Coda"}""") + } + + test("A basic case class is parsable from a JSON object with corresponding fields") { + parse[CaseClass]("""{"id":111,"name":"Coda"}""") should be(CaseClass(111L, "Coda")) + } + + test("A basic case class is parsable from a JSON object with extra fields") { + parse[CaseClass]("""{"id":1,"name":"Coda","derp":100}""") should be(CaseClass(1, "Coda")) + } + + test("A basic case class is not parsable from an incomplete JSON object") { + intercept[Exception] { + parse[CaseClass]("""{"id":1}""") + } + } + + test("A case class with lazy fields generates a JSON object with those fields evaluated") { + generate(CaseClassWithLazyVal(1)) should be("""{"id":1,"woo":"yeah"}""") + } + + test("A case class with lazy fields is parsable from a JSON object without those fields") { + parse[CaseClassWithLazyVal]("""{"id":1}""") should be(CaseClassWithLazyVal(1)) + } + + test("A case class with lazy fields is not parsable from an incomplete JSON object") { + intercept[Exception] { + parse[CaseClassWithLazyVal]("""{}""") + } + } + + test("A case class with ignored members generates a JSON object without those fields") { + generate(CaseClassWithIgnoredField(1)) should be("""{"id":1}""") + generate(CaseClassWithIgnoredFieldsExactMatch(1)) should be("""{"id":1}""") + generate(CaseClassWithIgnoredFieldsMatchAfterToSnakeCase(1)) should be("""{"id":1}""") + } + + test("A case class with some ignored members is not parsable from an incomplete JSON object") { + intercept[Exception] { + parse[CaseClassWithIgnoredField]("""{}""") + } + intercept[Exception] { + parse[CaseClassWithIgnoredFieldsMatchAfterToSnakeCase]("""{}""") + } + } + + test("A case class with transient members generates a JSON object without those fields") { + generate(CaseClassWithTransientField(1)) should be("""{"id":1}""") + } + + test("A case class with transient members is parsable from a JSON object without those fields") { + parse[CaseClassWithTransientField]("""{"id":1}""") should be(CaseClassWithTransientField(1)) + } + + test("A case class with some transient members is not parsable from an incomplete JSON object") { + intercept[Exception] { + parse[CaseClassWithTransientField]("""{}""") + } + } + + test("A case class with lazy vals generates a JSON object without those fields") { + // jackson-module-scala serializes lazy vals and the only work around is to use @JsonIgnore on the field + // see: https://github.com/FasterXML/jackson-module-scala/issues/113 + generate(CaseClassWithLazyField(1)) should be("""{"id":1}""") + } + + test("A case class with lazy vals is parsable from a JSON object without those fields") { + parse[CaseClassWithLazyField]("""{"id":1}""") should be(CaseClassWithLazyField(1)) + } + + test( + "A case class with an overloaded field generates a JSON object with the nullary version of that field" + ) { + generate(CaseClassWithOverloadedField(1)) should be("""{"id":1}""") + } + + test("A case class with an Option[String] member generates a field if the member is Some") { + generate(CaseClassWithOption(Some("what"))) should be("""{"value":"what"}""") + } + + test( + "A case class with an Option[String] member is parsable from a JSON object with that field") { + parse[CaseClassWithOption]("""{"value":"what"}""") should be(CaseClassWithOption(Some("what"))) + } + + test( + "A case class with an Option[String] member doesn't generate a field if the member is None") { + generate(CaseClassWithOption(None)) should be("""{}""") + } + + test( + "A case class with an Option[String] member is parsable from a JSON object without that field" + ) { + parse[CaseClassWithOption]("""{}""") should be(CaseClassWithOption(None)) + } + + test( + "A case class with an Option[String] member is parsable from a JSON object with a null value for that field" + ) { + parse[CaseClassWithOption]("""{"value":null}""") should be(CaseClassWithOption(None)) + } + + test("A case class with a JsonNode member generates a field of the given type") { + generate(CaseClassWithJsonNode(new IntNode(2))) should be("""{"value":2}""") + } + + test("issues#standalone map") { + val map = parse[Map[String, String]](""" + { + "one": "two" + } + """) + map should equal(Map("one" -> "two")) + } + + test("issues#case class with map") { + val obj = parse[CaseClassWithMap](""" + { + "map": {"one": "two"} + } + """) + obj.map should equal(Map("one" -> "two")) + } + + test("issues#case class with multiple constructors") { + intercept[AssertionError] { + parse[CaseClassWithTwoConstructors]("""{"id":42,"name":"TwoConstructorsNoDefault"}"""") + } + } + + test("issues#case class with multiple (3) constructors") { + intercept[AssertionError] { + parse[CaseClassWithThreeConstructors]("""{"id":42,"name":"ThreeConstructorsNoDefault"}"""") + } + } + + test("issues#case class nested within an object") { + parse[NestedCaseClassInObject](""" + { + "id": "foo" + } + """) should equal(NestedCaseClassInObject(id = "foo")) + } + + test( + "issues#case class nested within an object with member that is also a case class in an object" + ) { + parse[NestedCaseClassInObjectWithNestedCaseClassInObjectParam]( + """ + { + "nested": { + "id": "foo" + } + } + """ + ) should equal( + NestedCaseClassInObjectWithNestedCaseClassInObjectParam( + nested = NestedCaseClassInObject(id = "foo") + ) + ) + } + + test("deserialization#case class nested within a companion object works") { + parse[NestedCaseClassInCompanion]( + """ + { + "id": "foo" + } + """ + ) should equal(NestedCaseClassInCompanion(id = "foo")) + } + + case class NestedCaseClassInClass(id: String) + test("deserialization#case class nested within a class fails") { + intercept[AssertionError] { + parse[NestedCaseClassInClass](""" + { + "id": "foo" + } + """) should equal(NestedCaseClassInClass(id = "foo")) + } + } + + test("deserialization#case class with set of longs") { + val obj = parse[CaseClassWithSetOfLongs](""" + { + "set": [5000000, 1, 2, 3, 1000] + } + """) + obj.set.toSeq.sorted should equal(Seq(1L, 2L, 3L, 1000L, 5000000L)) + } + + test("deserialization#case class with seq of longs") { + val obj = parse[CaseClassWithSeqOfLongs](""" + { + "seq": [%s] + } + """.format(1004 to 1500 mkString ",")) + obj.seq.sorted should equal(1004 to 1500) + } + + test("deserialization#nested case class with collection of longs") { + val idsStr = 1004 to 1500 mkString "," + val obj = parse[CaseClassWithNestedSeqLong](""" + { + "seq_class" : {"seq": [%s]}, + "set_class" : {"set": [%s]} + } + """.format(idsStr, idsStr)) + obj.seqClass.seq.sorted should equal(1004 to 1500) + obj.setClass.set.toSeq.sorted should equal(1004 to 1500) + } + + test("deserialization#complex without companion class") { + val json = + """{ + "entity_ids" : [ 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1441, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1496, 1497, 1498, 1499, 1500 ], + "previous_cursor" : "$", + "next_cursor" : "2892e7ab37d44c6a15b438f78e8d76ed$" + }""" + val entityIdsResponse = parse[TestEntityIdsResponse](json) + entityIdsResponse.entityIds.sorted.size should be > 0 + } + + test("deserialization#complex with companion class") { + val json = + """{ + "entity_ids" : [ 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1027, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1075, 1076, 1077, 1078, 1079, 1080, 1081, 1082, 1083, 1084, 1085, 1086, 1087, 1088, 1089, 1090, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111, 1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1241, 1242, 1243, 1244, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1278, 1279, 1280, 1281, 1282, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, 1422, 1423, 1424, 1425, 1426, 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437, 1438, 1439, 1440, 1441, 1442, 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1496, 1497, 1498, 1499, 1500 ], + "previous_cursor" : "$", + "next_cursor" : "2892e7ab37d44c6a15b438f78e8d76ed$" + }""" + val entityIdsResponse = parse[TestEntityIdsResponseWithCompanion](json) + entityIdsResponse.entityIds.sorted.size should be > 0 + } + + val json = """ + { + "map": { + "one": "two" + }, + "set": [1, 2, 3], + "string": "woo", + "list": [4, 5, 6], + "seq": [7, 8, 9], + "sequence": [10, 11, 12], + "collection": [13, 14, 15], + "indexed_seq": [16, 17, 18], + "random_access_seq": [19, 20, 21], + "vector": [22, 23, 24], + "big_decimal": 12.0, + "big_int": 13, + "int": 1, + "long": 2, + "char": "x", + "bool": false, + "short": 14, + "byte": 15, + "float": 34.5, + "double": 44.9, + "any": true, + "any_ref": "wah", + "int_map": { + "1": "1" + }, + "long_map": { + "2": 2 + } + }""" + + test( + "deserialization#case class with members of all ScalaSig types " + + "is parsable from a JSON object with those fields" + ) { + val got = parse[CaseClassWithAllTypes](json) + val expected = CaseClassWithAllTypes( + map = Map("one" -> "two"), + set = Set(1, 2, 3), + string = "woo", + list = List(4, 5, 6), + seq = Seq(7, 8, 9), + indexedSeq = IndexedSeq(16, 17, 18), + vector = Vector(22, 23, 24), + bigDecimal = BigDecimal("12.0"), + bigInt = 13, + int = 1, + long = 2L, + char = 'x', + bool = false, + short = 14, + byte = 15, + float = 34.5f, + double = 44.9d, + any = true, + anyRef = "wah", + intMap = Map(1 -> 1), + longMap = Map(2L -> 2L) + ) + + got should be(expected) + } + + test("deserialization# case class that throws an exception is not parsable from a JSON object") { + intercept[JsonMappingException] { + parse[CaseClassWithException]("""{}""") + } + } + + test("deserialization#case class nested inside of an object is parsable from a JSON object") { + parse[OuterObject.NestedCaseClass]("""{"id": 1}""") should be(OuterObject.NestedCaseClass(1)) + } + + test( + "deserialization#case class nested inside of an object nested inside of an object is parsable from a JSON object" + ) { + parse[OuterObject.InnerObject.SuperNestedCaseClass]("""{"id": 1}""") should be( + OuterObject.InnerObject.SuperNestedCaseClass(1) + ) + } + + test("A case class with array members is parsable from a JSON object") { + val jsonStr = """ + { + "one":"1", + "two":["a","b","c"], + "three":[1,2,3], + "four":[4, 5], + "five":["x", "y"], + "bools":["true", false], + "bytes":[1,2], + "doubles":[1,5.0], + "floats":[1.1, 22] + } + """ + + val c = parse[CaseClassWithArrays](jsonStr) + c.one should be("1") + c.two should be(Array("a", "b", "c")) + c.three should be(Array(1, 2, 3)) + c.four should be(Array(4L, 5L)) + c.five should be(Array('x', 'y')) + + JsonDiff.assertDiff( + """{"bools":[true,false],"bytes":"AQI=","doubles":[1.0,5.0],"five":"xy","floats":[1.1,22.0],"four":[4,5],"one":"1","three":[1,2,3],"two":["a","b","c"]}""", + generate(c) + ) + } + + test("deserialization#case class with collection of Longs array of longs") { + val c = parse[CaseClassWithArrayLong]("""{"array":[3,1,2]}""") + c.array.sorted should equal(Array(1, 2, 3)) + } + + test("deserialization#case class with collection of Longs seq of longs") { + val c = parse[CaseClassWithSeqLong]("""{"seq":[3,1,2]}""") + c.seq.sorted should equal(Seq(1, 2, 3)) + } + + test("deserialization#case class with an ArrayList of Integers") { + val c = parse[CaseClassWithArrayListOfIntegers]("""{"arraylist":[3,1,2]}""") + val l = new java.util.ArrayList[Integer](3) + l.add(3) + l.add(1) + l.add(2) + c.arraylist should equal(l) + } + + test("serde#case class with a SortedMap[String, Int]") { + val origCaseClass = CaseClassWithSortedMap(scala.collection.SortedMap("aggregate" -> 20)) + val caseClassJson = generate(origCaseClass) + val caseClass = parse[CaseClassWithSortedMap](caseClassJson) + caseClass should equal(origCaseClass) + } + + test("serde#case class with a Seq of Longs") { + val origCaseClass = CaseClassWithSeqOfLongs(Seq(10, 20, 30)) + val caseClassJson = generate(origCaseClass) + val caseClass = parse[CaseClassWithSeqOfLongs](caseClassJson) + caseClass should equal(origCaseClass) + } + + test("deserialization#seq of longs") { + val seq = parse[Seq[Long]]("""[3,1,2]""") + seq.sorted should equal(Seq(1L, 2L, 3L)) + } + + test("deserialization#parse seq of longs") { + val ids = parse[Seq[Long]]("[3,1,2]") + ids.sorted should equal(Seq(1L, 2L, 3L)) + } + + test("deserialization#handle options and defaults in case class") { + val bob = parse[Person](""" + { + "id" :1, + "name" : "Bob", + "age" : 21 + } + """) + bob should equal(Person(1, "Bob", None, Some(21), None)) + } + + test("deserialization#missing required field") { + intercept[CaseClassMappingException] { + parse[Person](""" + { + } + """) + } + } + + test("deserialization#incorrectly specified required field") { + intercept[CaseClassMappingException] { + parse[PersonWithThings](""" + { + "id" :1, + "name" : "Bob", + "age" : 21, + "things" : { + "foo" : [ + "IhaveNoKey" + ] + } + } + """) + } + } + + test("serialization#nulls will not render") { + generate(Person(1, null, null, null)) should equal("""{"id":1,"nickname":"unknown"}""") + } + + test("deserialization#string wrapper deserialization") { + val parsedValue = parse[ObjWithTestId](""" + { + "id": "5" + } + """) + val expectedValue = ObjWithTestId(TestIdStringWrapper("5")) + + parsedValue should equal(expectedValue) + parsedValue.id.onlyValue should equal(expectedValue.id.onlyValue) + parsedValue.id.asString should equal(expectedValue.id.asString) + parsedValue.id.toString should equal(expectedValue.id.toString) + } + + test("deserialization#parse input stream") { + val is = new ByteArrayInputStream("""{"foo": "bar"}""".getBytes) + mapper.parse[Blah](is) should equal(Blah("bar")) + } + + test("serialization#ogging Trait fields should be ignored") { + generate(Group3("123")) should be("""{"id":"123"}""") + } + + test("deserialization#class with no constructor") { + parse[NoConstructorArgs]("""{}""") + } + + //Jackson parses numbers into boolean type without error. see https://jira.codehaus.org/browse/JACKSON-78 + test("deserialization#case class with boolean as number") { + parse[CaseClassWithBoolean](""" { + "foo": 100 + }""") should equal(CaseClassWithBoolean(true)) + } + + //Jackson parses numbers into boolean type without error. see https://jira.codehaus.org/browse/JACKSON-78 + test("deserialization#case class with Seq[Boolean]") { + parse[CaseClassWithSeqBooleans](""" { + "foos": [100, 5, 0, 9] + }""") should equal(CaseClassWithSeqBooleans(Seq(true, true, false, true))) + } + + //Jackson parses numbers into boolean type without error. see https://jira.codehaus.org/browse/JACKSON-78 + test("Seq[Boolean]") { + parse[Seq[Boolean]]("""[100, 5, 0, 9]""") should equal(Seq(true, true, false, true)) + } + + test("case class with boolean as number 0") { + parse[CaseClassWithBoolean](""" { + "foo": 0 + }""") should equal(CaseClassWithBoolean(false)) + } + + test("case class with boolean as string") { + assertJsonParse[CaseClassWithBoolean]( + """ { + "foo": "bar" + }""", + withErrors = Seq("foo: 'bar' is not a valid Boolean")) + } + + test("case class with boolean number as string") { + assertJsonParse[CaseClassWithBoolean]( + """ { + "foo": "1" + }""", + withErrors = Seq("foo: '1' is not a valid Boolean")) + } + + val msgHiJsonStr = """{"msg":"hi"}""" + + test("parse jsonParser") { + val jsonNode = mapper.parse[JsonNode]("{}") + val jsonParser = new TreeTraversingParser(jsonNode) + mapper.parse[JsonNode](jsonParser) should equal(jsonNode) + } + + test("writeValue") { + val os = new ByteArrayOutputStream() + mapper.writeValue(Map("msg" -> "hi"), os) + os.close() + new String(os.toByteArray) should equal(msgHiJsonStr) + } + + test("writeValueAsBuf") { + val buf = mapper.writeValueAsBuf(Map("msg" -> "hi")) + val Buf.Utf8(str) = buf + str should equal(msgHiJsonStr) + } + + test("writeStringMapAsBuf") { + val buf = mapper.writeStringMapAsBuf(Map("msg" -> "hi")) + val Buf.Utf8(str) = buf + str should equal(msgHiJsonStr) + } + + test("writePrettyString") { + val jsonStr = mapper.writePrettyString("""{"msg": "hi"}""") + mapper.parse[JsonNode](jsonStr).get("msg").textValue() should equal("hi") + } + + test("reader") { + assert(mapper.reader[JsonNode] != null) + } + + test( + "deserialization#jackson JsonDeserialize annotations deserializes json to case class with 2 decimal places for mandatory field" + ) { + parse[CaseClassWithCustomDecimalFormat](""" { + "my_big_decimal": 23.1201 + }""") should equal(CaseClassWithCustomDecimalFormat(BigDecimal(23.12), None)) + } + + test("deserialization#jackson JsonDeserialize annotations long with JsonDeserialize") { + val result = parse[CaseClassWithLongAndDeserializer](""" { + "long": 12345 + }""") + result should equal(CaseClassWithLongAndDeserializer(12345)) + result.long should equal(12345L) // promotes the returned integer into a long + } + + test( + "deserialization#jackson JsonDeserialize annotations deserializes json to case class with 2 decimal places for option field" + ) { + parse[CaseClassWithCustomDecimalFormat](""" { + "my_big_decimal": 23.1201, + "opt_my_big_decimal": 23.1201 + }""") should equal( + CaseClassWithCustomDecimalFormat(BigDecimal(23.12), Some(BigDecimal(23.12))) + ) + } + + test("deserialization#jackson JsonDeserialize annotations opt long with JsonDeserialize") { + parse[CaseClassWithOptionLongAndDeserializer](""" + { + "opt_long": 12345 + } + """) should equal(CaseClassWithOptionLongAndDeserializer(Some(12345))) + } + + test("deserialization#case class in package object") { + parse[SimplePersonInPackageObject]("""{"name": "Steve"}""") should equal( + SimplePersonInPackageObject("Steve") + ) + } + + test("deserialization#case class in package object uses default when name not specified") { + parse[SimplePersonInPackageObject]("""{}""") should equal( + SimplePersonInPackageObject() + ) + } + + test( + "deserialization#case class in package object without constructor params and parsing an empty json object") { + parse[SimplePersonInPackageObjectWithoutConstructorParams]("""{}""") should equal( + SimplePersonInPackageObjectWithoutConstructorParams() + ) + } + + test( + "deserialization#case class in package object without constructor params and parsing a json object with extra fields" + ) { + parse[SimplePersonInPackageObjectWithoutConstructorParams]( + """{"name": "Steve"}""") should equal( + SimplePersonInPackageObjectWithoutConstructorParams() + ) + } + + test("serialization#upport sealed traits and case objects#json serialization") { + val vin = Random.alphanumeric.take(17).mkString + val vehicle = Vehicle(vin, Audi) + + mapper.writeValueAsString(vehicle) should equal(s"""{"vin":"$vin","type":"audi"}""") + } + + test("deserialization#json creator") { + parse[TestJsonCreator]("""{"s" : "1234"}""") should equal(TestJsonCreator("1234")) + } + + test("deserialization#json creator with parameterized type") { + parse[TestJsonCreator2]("""{"strings" : ["1", "2", "3", "4"]}""") should equal( + TestJsonCreator2(Seq(1, 2, 3, 4))) + } + + test("deserialization#multiple case class constructors#no annotation") { + intercept[AssertionError] { + parse[CaseClassWithMultipleConstructors]( + """ + |{ + | "number1" : 12345, + | "number2" : 65789, + | "number3" : 99999 + |} + |""".stripMargin + ) + } + } + + test("deserialization#multiple case class constructors#annotated") { + parse[CaseClassWithMultipleConstructorsAnnotated]( + """ + |{ + | "number_as_string1" : "12345", + | "number_as_string2" : "65789", + | "number_as_string3" : "99999" + |} + |""".stripMargin + ) should equal( + CaseClassWithMultipleConstructorsAnnotated(12345L, 65789L, 99999L) + ) + } + + test("deserialization#JsonCreator with Validation") { + val e = intercept[CaseClassMappingException] { + // fails validation + parse[TestJsonCreatorWithValidation]("""{ "s": "" }""") + } + e.errors.size should equal(1) + e.errors.head.getMessage should be("s: must not be empty") + } + + test("deserialization#JsonCreator with Validation 1") { + // works with multiple validations + var e = intercept[CaseClassMappingException] { + // fails validation + parse[TestJsonCreatorWithValidations]("""{ "s": "" }""") + } + e.errors.size should equal(2) + // errors are alpha-sorted by message to be stable for testing + e.errors.head.getMessage should be("s: not one of [42, 137]") + e.errors.last.getMessage should be("s: must not be empty") + + e = intercept[CaseClassMappingException] { + // fails validation + parse[TestJsonCreatorWithValidations]("""{ "s": "99" }""") + } + e.errors.size should equal(1) + e.errors.head.getMessage should be("s: 99 not one of [42, 137]") + + parse[TestJsonCreatorWithValidations]("""{ "s": "42" }""") should equal( + TestJsonCreatorWithValidations("42")) + + parse[TestJsonCreatorWithValidations]("""{ "s": "137" }""") should equal( + TestJsonCreatorWithValidations(137)) + } + + test("deserialization#JsonCreator with Validation 2") { + val uuid = UUID.randomUUID().toString + + var e = intercept[CaseClassMappingException] { + // fails validation + parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( + s""" + |{ + | "number_as_string1" : "", + | "number_as_string2" : "20002", + | "third_argument" : "$uuid" + |} + |""".stripMargin + ) + } + e.errors.size should equal(1) + e.errors.head.getMessage should be("number_as_string1: must not be empty") + + e = intercept[CaseClassMappingException] { + // fails validation + parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( + s""" + |{ + | "number_as_string1" : "", + | "number_as_string2" : "65789", + | "third_argument" : "$uuid" + |} + |""".stripMargin + ) + } + e.errors.size should equal(2) + // errors are alpha-sorted by message to be stable for testing + e.errors.head.getMessage should be("number_as_string1: must not be empty") + e.errors.last.getMessage should be("number_as_string2: 65789 not one of [10001, 20002, 30003]") + + e = intercept[CaseClassMappingException] { + // fails validation + parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( + s""" + |{ + | "number_as_string1" : "", + | "number_as_string2" : "65789", + | "third_argument" : "foobar" + |} + |""".stripMargin + ) + } + e.errors.size should equal(3) + // errors are alpha-sorted by message to be stable for testing + e.errors.head.getMessage should be("number_as_string1: must not be empty") + e.errors(1).getMessage should be("number_as_string2: 65789 not one of [10001, 20002, 30003]") + e.errors(2).getMessage should be("third_argument: must be a valid UUID") + + parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( + s""" + |{ + | "number_as_string1" : "12345", + | "number_as_string2" : "20002", + | "third_argument" : "$uuid" + |} + |""".stripMargin + ) should equal( + CaseClassWithMultipleConstructorsAnnotatedAndValidations(12345L, 20002L, uuid) + ) + } + + test("deserialization#JsonFormat with c.t.util.Time") { + val timeString = "2018-09-14T23:20:08.000-07:00" + val format = new com.twitter.util.TimeFormat( + "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", + None, + TimeZone.getTimeZone("UTC")) + val time: com.twitter.util.Time = format.parse(timeString) + + parse[TimeWithFormat]( + s""" + |{ + | "when": "$timeString" + |} + |""".stripMargin + ) should equal(TimeWithFormat(time)) + } + + test("deserialization#complex object") { + val profiles = Seq( + LimiterProfile(1, "up"), + LimiterProfile(2, "down"), + LimiterProfile(3, "left"), + LimiterProfile(4, "right")) + val expected = LimiterProfiles(profiles) + + parse[LimiterProfiles](""" + |{ + | "profiles" : [ + | { + | "id" : "1", + | "name" : "up" + | }, + | { + | "id" : "2", + | "name" : "down" + | }, + | { + | "id" : "3", + | "name" : "left" + | }, + | { + | "id" : "4", + | "name" : "right" + | } + | ] + |} + |""".stripMargin) should equal(expected) + } + + test("deserialization#simple class with boxed primitive constructor args") { + parse[CaseClassWithBoxedPrimitives](""" + |{ + | "events" : 2, + | "errors" : 0 + |} + |""".stripMargin) should equal(CaseClassWithBoxedPrimitives(2, 0)) + } + + test("deserialization#complex object with primitives") { + val json = """ + |{ + | "cluster_name" : "cluster1", + | "zone" : "abcd1", + | "environment" : "devel", + | "job" : "devel-1", + | "owners" : "owning-group", + | "dtab" : "/s#/devel/abcd1/space/devel-1", + | "address" : "local-test.foo.bar.com:1111/owning-group/space/devel-1", + | "enabled" : "true" + |} + |""".stripMargin + + parse[AddClusterRequest](json) should equal( + AddClusterRequest( + clusterName = "cluster1", + zone = "abcd1", + environment = "devel", + job = "devel-1", + dtab = "/s#/devel/abcd1/space/devel-1", + address = "local-test.foo.bar.com:1111/owning-group/space/devel-1", + owners = "owning-group" + )) + } + + test("deserialization#inject request field fails with mapping exception") { + // with an injector and default injectable values but the field cannot be parsed + // thus we get a case class mapping exception + // without an injector, the incoming JSON is used to populate the case class, which + // is empty and thus we also get a case class mapping exception + intercept[CaseClassMappingException] { + parse[ClassWithQueryParamDateTimeInject]("""{}""") + } + } + + test("deserialization#ignored constructor field - no default value") { + val json = + """ + |{ + | "name" : "Widget", + | "description" : "This is a thing." + |} + |""".stripMargin + + val e = intercept[CaseClassMappingException] { + parse[CaseClassIgnoredFieldInConstructorNoDefault](json) + } + e.errors.head.getMessage should be("id: ignored field has no default value specified") + } + + test("ignored constructor field - with default value") { + val json = + """ + |{ + | "name" : "Widget", + | "description" : "This is a thing." + |} + |""".stripMargin + + val result = parse[CaseClassIgnoredFieldInConstructorWithDefault](json) + result.id should equal(42L) + result.name should equal("Widget") + result.description should equal("This is a thing.") + } + + test("deserialization#mixin annotations") { + val points = Points(first = Point(1, 1), second = Point(4, 5)) + val json = + """ + |{ + | "first": { "x": 1, "y": 1 }, + | "second": { "x": 4, "y": 5 } + |} + |""".stripMargin + + parse[Points](json) should equal(points) + generate(points) should be("""{"first":{"x":1,"y":1},"second":{"x":4,"y":5}}""".stripMargin) + } + + test("deserialization#mixin annotations with validations") { + val json = + """ + |{ + | "first": { "x": -1, "y": 120 }, + | "second": { "x": 4, "y": 5 } + |} + |""".stripMargin + + val e = intercept[CaseClassMappingException] { + parse[Points](json) + } + e.errors.size should equal(2) + e.errors.head.getMessage should be("first.x: must be greater than or equal to 0") + e.errors.head.reason.detail match { + case ValidationError(violation, ValidationError.Field, None) => + violation.getPropertyPath.toString should equal("Point.x") + violation.getMessage should equal("must be greater than or equal to 0") + violation.getInvalidValue should equal(-1) + violation.getRootBeanClass should equal(classOf[Point]) + violation.getRootBean == null should be(true) + case _ => fail() + } + e.errors.last.getMessage should be("first.y: must be less than or equal to 100") + e.errors.last.reason.detail match { + case ValidationError(violation, ValidationError.Field, None) => + violation.getPropertyPath.toString should equal("Point.y") + violation.getMessage should equal("must be less than or equal to 100") + violation.getInvalidValue should equal(120) + violation.getRootBeanClass should equal(classOf[Point]) + violation.getRootBean == null should be(true) + case _ => fail() + } + } + + test("deserialization#ignore type with no default fails") { + val json = + """ + |{ + | "name" : "Widget", + | "description" : "This is a thing." + |} + |""".stripMargin + val e = intercept[CaseClassMappingException] { + parse[ContainsAnIgnoreTypeNoDefault](json) + } + e.errors.head.getMessage should be("ignored: ignored field has no default value specified") + } + + test("deserialization#ignore type with default passes") { + val json = + """ + |{ + | "name" : "Widget", + | "description" : "This is a thing." + |} + |""".stripMargin + val result = parse[ContainsAnIgnoreTypeWithDefault](json) + result.ignored should equal(IgnoreMe(42L)) + result.name should equal("Widget") + result.description should equal("This is a thing.") + } + + test("deserialization#parse SimpleClassWithInjection fails") { + // no field injection configured, parsing happens normally + val json = + """{ + | "hello" : "Mom" + |}""".stripMargin + + val result = parse[SimpleClassWithInjection](json) + result.hello should equal("Mom") + } + + test("deserialization#support JsonView") { + /* using [[Item]] which has @JsonView annotation on all fields: + Public -> id + Public -> name + Internal -> owner + */ + val json = + """ + |{ + | "id": 42, + | "name": "My Item" + |} + |""".stripMargin + + val item = + mapper.underlying.readerWithView[Views.Public].forType(classOf[Item]).readValue[Item](json) + item.id should equal(42L) + item.name should equal("My Item") + item.owner should equal("") + } + + test("deserialization#support JsonView 2") { + /* using [[Item]] which has @JsonView annotation on all fields: + Public -> id + Public -> name + Internal -> owner + */ + val json = + """ + |{ + | "owner": "Mister Magoo" + |} + |""".stripMargin + + val item = + mapper.underlying.readerWithView[Views.Internal].forType(classOf[Item]).readValue[Item](json) + item.id should equal(1L) + item.name should equal("") + item.owner should equal("Mister Magoo") + } + + test("deserialization#support JsonView 3") { + /* using [[ItemSomeViews]] which has no @JsonView annotation on owner field: + Public -> id + Public -> name + -> owner + */ + val json = + """ + |{ + | "id": 42, + | "name": "My Item", + | "owner": "Mister Magoo" + |} + |""".stripMargin + + val item = + mapper.underlying + .readerWithView[Views.Public] + .forType(classOf[ItemSomeViews]) + .readValue[ItemSomeViews](json) + item.id should equal(42L) + item.name should equal("My Item") + item.owner should equal("Mister Magoo") + } + + test("deserialization#support JsonView 4") { + /* using [[ItemSomeViews]] which has no @JsonView annotation on owner field: + Public -> id + Public -> name + -> owner + */ + val json = + """ + |{ + | "owner": "Mister Magoo" + |} + |""".stripMargin + + val item = + mapper.underlying + .readerWithView[Views.Internal] + .forType(classOf[ItemSomeViews]) + .readValue[ItemSomeViews](json) + item.id should equal(1L) + item.name should equal("") + item.owner should equal("Mister Magoo") + } + + test("deserialization#support JsonView 5") { + /* using [[ItemNoDefaultForView]] which has @JsonView annotation on non-defaulted field: + Public -> name (no default value) + */ + val json = + """ + |{ + | "name": "My Item" + |} + |""".stripMargin + + val item = + mapper.underlying + .readerWithView[Views.Public] + .forType(classOf[ItemNoDefaultForView]) + .readValue[ItemNoDefaultForView](json) + item.name should equal("My Item") + } + + test("deserialization#support JsonView 6") { + /* using [[ItemNoDefaultForView]] which has @JsonView annotation on non-defaulted field: + Public -> name (no default value) + */ + + intercept[CaseClassMappingException] { + mapper.underlying + .readerWithView[Views.Public] + .forType(classOf[ItemNoDefaultForView]) + .readValue[ItemNoDefaultForView]("{}") + } + } + + test("deserialization#test @JsonNaming") { + val json = + """ + |{ + | "please-use-kebab-case": true + |} + |""".stripMargin + + parse[CaseClassWithKebabCase](json) should equal(CaseClassWithKebabCase(true)) + + val snakeCaseMapper = ScalaObjectMapper.snakeCaseObjectMapper(mapper.underlying) + snakeCaseMapper.parse[CaseClassWithKebabCase](json) should equal(CaseClassWithKebabCase(true)) + + val camelCaseMapper = ScalaObjectMapper.camelCaseObjectMapper(mapper.underlying) + camelCaseMapper.parse[CaseClassWithKebabCase](json) should equal(CaseClassWithKebabCase(true)) + } + + test("deserialization#test @JsonNaming 2") { + val snakeCaseMapper = ScalaObjectMapper.snakeCaseObjectMapper(mapper.underlying) + + // case class is marked with @JsonNaming and the default naming strategy is LOWER_CAMEL_CASE + // which overrides the mapper's configured naming strategy + val json = + """ + |{ + | "thisFieldShouldUseDefaultPropertyNamingStrategy": true + |} + |""".stripMargin + + snakeCaseMapper.parse[UseDefaultNamingStrategy](json) should equal( + UseDefaultNamingStrategy(true)) + + // default for mapper is snake_case, but the case class is annotated with @JsonNaming which + // tells the mapper what property naming strategy to expect. + parse[UseDefaultNamingStrategy](json) should equal(UseDefaultNamingStrategy(true)) + } + + test("deserialization#test @JsonNaming 3") { + val camelCaseMapper = ScalaObjectMapper.camelCaseObjectMapper(mapper.underlying) + + val json = + """ + |{ + | "thisFieldShouldUseDefaultPropertyNamingStrategy": true + |} + |""".stripMargin + + camelCaseMapper.parse[UseDefaultNamingStrategy](json) should equal( + UseDefaultNamingStrategy(true)) + + // default for mapper is snake_case, but the case class is annotated with @JsonNaming which + // tells the mapper what property naming strategy to expect. + parse[UseDefaultNamingStrategy](json) should equal(UseDefaultNamingStrategy(true)) + } + + test("deserialization#test @JsonNaming with mixin") { + val json = + """ + |{ + | "will-this-get-the-right-casing": true + |} + |""".stripMargin + + parse[CaseClassShouldUseKebabCaseFromMixin](json) should equal( + CaseClassShouldUseKebabCaseFromMixin(true)) + + val snakeCaseMapper = ScalaObjectMapper.snakeCaseObjectMapper(mapper.underlying) + snakeCaseMapper.parse[CaseClassShouldUseKebabCaseFromMixin](json) should equal( + CaseClassShouldUseKebabCaseFromMixin(true)) + + val camelCaseMapper = ScalaObjectMapper.camelCaseObjectMapper(mapper.underlying) + camelCaseMapper.parse[CaseClassShouldUseKebabCaseFromMixin](json) should equal( + CaseClassShouldUseKebabCaseFromMixin(true)) + } + + test("serde#can serialize and deserialize polymorphic types") { + val view = View(shapes = Seq(Circle(10), Rectangle(5, 5), Circle(3), Rectangle(10, 5))) + val result = generate(view) + result should be( + """{"shapes":[{"type":"circle","radius":10},{"type":"rectangle","width":5,"height":5},{"type":"circle","radius":3},{"type":"rectangle","width":10,"height":5}]}""") + + parse[View](result) should equal(view) + } + + test("serde#can serialize and deserialize optional polymorphic types") { + val view = + OptionalView(shapes = Seq(Circle(10), Rectangle(5, 5)), optional = Some(Rectangle(10, 5))) + val result = generate(view) + result should be( + """{"shapes":[{"type":"circle","radius":10},{"type":"rectangle","width":5,"height":5}],"optional":{"type":"rectangle","width":10,"height":5}}""") + + parse[OptionalView](result) should equal(view) + } + + test("serde#can deserialize case class with option when missing") { + val view = OptionalView(shapes = Seq(Circle(10), Rectangle(5, 5)), optional = None) + val result = generate(view) + result should equal( + """{"shapes":[{"type":"circle","radius":10},{"type":"rectangle","width":5,"height":5}]}""") + + parse[OptionalView](result) should equal(view) + } + + test("nullable deserializer - allows parsing a JSON null into a null type") { + parse[WithNullableCarMake]( + """{"non_null_car_make": "vw", "nullable_car_make": null}""") should be( + WithNullableCarMake(nonNullCarMake = CarMakeEnum.vw, nullableCarMake = null)) + } + + test("nullable deserializer - fail on missing value even when null allowed") { + val e = intercept[CaseClassMappingException] { + parse[WithNullableCarMake]("""{}""") + } + e.errors.size should equal(2) + e.errors.head.getMessage should equal("non_null_car_make: field is required") + e.errors.last.getMessage should equal("nullable_car_make: field is required") + } + + test("nullable deserializer - allow null values when default is provided") { + parse[WithDefaultNullableCarMake]("""{"nullable_car_make":null}""") should be( + WithDefaultNullableCarMake(null)) + } + + test("nullable deserializer - use default when null is allowed, but not provided") { + parse[WithDefaultNullableCarMake]("""{}""") should be( + WithDefaultNullableCarMake(CarMakeEnum.ford)) + } + + test("deserialization of json array into a string field") { + val json = + """ + |{ + | "value": [{ + | "languages": "es", + | "reviewable": "tweet", + | "baz": "too", + | "foo": "bar", + | "coordinates": "polar" + | }] + |} + |""".stripMargin + + val expected = WithJsonStringType( + """[{"languages":"es","reviewable":"tweet","baz":"too","foo":"bar","coordinates":"polar"}]""".stripMargin) + parse[WithJsonStringType](json) should equal(expected) + } + + test("deserialization of json int into a string field") { + val json = + """ + |{ + | "value": 1 + |} + |""".stripMargin + + val expected = WithJsonStringType("1") + parse[WithJsonStringType](json) should equal(expected) + } + + test("deserialization of json object into a string field") { + val json = + """ + |{ + | "value": {"foo": "bar"} + |} + |""".stripMargin + + val expected = WithJsonStringType("""{"foo":"bar"}""") + parse[WithJsonStringType](json) should equal(expected) + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ThrowsRuntimeExceptionConstraint.java b/util-jackson/src/test/scala/com/twitter/util/jackson/ThrowsRuntimeExceptionConstraint.java new file mode 100644 index 0000000000..0e8b538e22 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ThrowsRuntimeExceptionConstraint.java @@ -0,0 +1,38 @@ +package com.twitter.util.jackson; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import jakarta.validation.Constraint; +import jakarta.validation.Payload; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.CONSTRUCTOR; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Documented +@Constraint(validatedBy = {ThrowsRuntimeExceptionConstraintValidator.class}) +@Target({METHOD, FIELD, ANNOTATION_TYPE, CONSTRUCTOR, PARAMETER, TYPE_USE}) +@Retention(RUNTIME) +public @interface ThrowsRuntimeExceptionConstraint { + + /** Every constraint annotation must define a message element of type String. */ + String message() default ""; + + /** + * Every constraint annotation must define a groups element that specifies the processing groups + * with which the constraint declaration is associated. + */ + Class[] groups() default {}; + + /** + * Constraint annotations must define a payload element that specifies the payload with which + * the constraint declaration is associated. + */ + Class[] payload() default {}; +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ThrowsRuntimeExceptionConstraintValidator.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ThrowsRuntimeExceptionConstraintValidator.scala new file mode 100644 index 0000000000..741697bcaa --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ThrowsRuntimeExceptionConstraintValidator.scala @@ -0,0 +1,14 @@ +package com.twitter.util.jackson + +import jakarta.validation.{ConstraintValidator, ConstraintValidatorContext} +import scala.util.control.NoStackTrace + +class ThrowsRuntimeExceptionConstraintValidator + extends ConstraintValidator[ThrowsRuntimeExceptionConstraint, Any] { + override def isValid( + obj: Any, + constraintValidatorContext: ConstraintValidatorContext + ): Boolean = { + throw new RuntimeException("validator foo error") with NoStackTrace + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/CaseClassFieldTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/CaseClassFieldTest.scala new file mode 100644 index 0000000000..9eb811b457 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/CaseClassFieldTest.scala @@ -0,0 +1,226 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.annotation.{JsonIgnore, JsonProperty} +import com.fasterxml.jackson.databind.JavaType +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import com.fasterxml.jackson.databind.deser.std.NumberDeserializers.BigDecimalDeserializer +import com.twitter.util.jackson.{ + JacksonScalaObjectMapperType, + ScalaObjectMapper, + TestInjectableValue, + _ +} +import com.twitter.util.validation.ScalaValidator +import jakarta.validation.constraints.NotEmpty +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class CaseClassFieldTest extends AnyFunSuite with Matchers { + private[this] val mapper: JacksonScalaObjectMapperType = + ScalaObjectMapper().underlying + + test("CaseClassField.createFields have field name foo") { + val fields = deserializerFor(classOf[WithEmptyJsonProperty]).fields + + fields.length should equal(1) + fields.head.name should equal("foo") + } + + test("CaseClassField.createFields also have field name foo") { + val fields = deserializerFor(classOf[WithoutJsonPropertyAnnotation]).fields + + fields.length should equal(1) + fields.head.name should equal("foo") + } + + test("CaseClassField.createFields have field name bar") { + val fields = deserializerFor(classOf[WithNonemptyJsonProperty]).fields + + fields.length should equal(1) + fields.head.name should equal("bar") + } + + test("CaseClassField.createFields sees inherited JsonProperty annotation") { + val fields = deserializerFor(classOf[Aum]).fields + + fields.length should equal(2) + + val iField = fields.head + iField.name should equal("i") + iField.annotations.length should equal(1) + iField.annotations.head.annotationType() should be(classOf[JsonProperty]) + + val jField = fields.last + jField.name should equal("j") + jField.annotations.length should equal(1) + jField.annotations.head.annotationType() should be(classOf[JsonProperty]) + } + + test("CaseClassField.createFields sees inherited JsonProperty annotation 2") { + val fields = deserializerFor(classOf[FooBar]).fields + + fields.length should equal(1) + + val helloField = fields.head + helloField.name should equal("helloWorld") + helloField.annotations.length should equal(2) + helloField.annotations.head.annotationType() should be(classOf[JsonProperty]) + helloField.annotations.exists(_.annotationType() == classOf[TestInjectableValue]) should be( + true) + helloField.annotations.last + .asInstanceOf[TestInjectableValue].value() should be("accept") // from Bar + } + + test("CaseClassField.createFields sees inherited JsonProperty annotation 3") { + val fields = deserializerFor(classOf[TestTraitImpl]).fields + + /* + in trait: + --------- + @JsonProperty("oldness") + def age: Int + @NotEmpty + def name: String + + case class constructor: + ----------------------- + @JsonProperty("ageness") age: Int, // should override inherited annotation from trait + @TestInjectableValue name: String, // should have two annotations, one from trait and one here + @TestInjectableValue dateTime: LocalDate, + @JsonProperty foo: String, // empty JsonProperty should default to field name + @JsonDeserialize(contentAs = classOf[BigDecimal], using = classOf[BigDecimalDeserializer]) + double: BigDecimal, + @JsonIgnore ignoreMe: String + */ + fields.length should equal(6) + + val fieldMap: Map[String, CaseClassField] = + fields.map(field => field.name -> field).toMap + + val ageField = fieldMap("ageness") + ageField.annotations.length should equal(1) + ageField.annotations.exists(_.annotationType() == classOf[JsonProperty]) should be(true) + ageField.annotations.head.asInstanceOf[JsonProperty].value() should equal("ageness") + + val nameField = fieldMap("name") + nameField.annotations.length should equal(2) + nameField.annotations.exists(_.annotationType() == classOf[NotEmpty]) should be(true) + nameField.annotations.exists(_.annotationType() == classOf[TestInjectableValue]) should be(true) + + val dateTimeField = fieldMap("dateTime") + dateTimeField.annotations.length should equal(1) + dateTimeField.annotations.exists(_.annotationType() == classOf[TestInjectableValue]) should be( + true) + + val fooField = fieldMap("foo") + fooField.annotations.length should equal(1) + fooField.annotations.exists(_.annotationType() == classOf[JsonProperty]) should be(true) + fooField.annotations.head.asInstanceOf[JsonProperty].value() should equal("") + + val doubleField = fieldMap("double") + doubleField.annotations.length should equal(1) + doubleField.annotations.exists(_.annotationType() == classOf[JsonDeserialize]) should be(true) + doubleField.annotations.head.asInstanceOf[JsonDeserialize].contentAs() should be( + classOf[BigDecimal]) + doubleField.annotations.head.asInstanceOf[JsonDeserialize].using() should be( + classOf[BigDecimalDeserializer]) + + val ignoreMeField = fieldMap("ignoreMe") + ignoreMeField.annotations.length should equal(1) + ignoreMeField.annotations.exists(_.annotationType() == classOf[JsonIgnore]) should be(true) + } + + test("CaseClassField.createFields sees inherited JsonProperty annotation 4") { + val fields: Seq[CaseClassField] = deserializerFor(classOf[FooBaz]).fields + + fields.length should equal(1) + val helloField: CaseClassField = fields.head + helloField.annotations.length should equal(2) + helloField.annotations.exists(_.annotationType() == classOf[JsonProperty]) should be(true) + helloField.annotations.head + .asInstanceOf[JsonProperty].value() should equal("goodbyeWorld") // from Baz + + helloField.annotations.exists(_.annotationType() == classOf[TestInjectableValue]) should be( + true) + helloField.annotations.last + .asInstanceOf[TestInjectableValue].value() should be("accept") // from Bar + } + + test("CaseClassField.createFields sees inherited JsonProperty annotation 5") { + val fields = deserializerFor(classOf[FooBarBaz]).fields + + fields.length should equal(1) + val helloField: CaseClassField = fields.head + helloField.annotations.length should equal(2) + helloField.annotations.exists(_.annotationType() == classOf[JsonProperty]) should be(true) + helloField.annotations.head + .asInstanceOf[JsonProperty].value() should equal("goodbye") // from BarBaz + + helloField.annotations.exists(_.annotationType() == classOf[TestInjectableValue]) should be( + true) + helloField.annotations.last + .asInstanceOf[TestInjectableValue].value() should be("accept") // from Bar + } + + test("CaseClassField.createFields sees inherited JsonProperty annotation 6") { + val fields = deserializerFor(classOf[File]).fields + + fields.length should equal(1) + val uriField: CaseClassField = fields.head + uriField.annotations.length should equal(1) + uriField.annotations.exists(_.annotationType() == classOf[JsonProperty]) should be(true) + uriField.annotations.head.asInstanceOf[JsonProperty].value() should equal("file") + } + + test("CaseClassField.createFields sees inherited JsonProperty annotation 7") { + val fields = deserializerFor(classOf[Folder]).fields + + fields.length should equal(1) + val uriField: CaseClassField = fields.head + uriField.annotations.length should equal(1) + uriField.annotations.exists(_.annotationType() == classOf[JsonProperty]) should be(true) + uriField.annotations.head.asInstanceOf[JsonProperty].value() should equal("folder") + } + + test("CaseClassField.createFields sees inherited JsonProperty annotation 8") { + val fields = deserializerFor(classOf[LoadableFile]).fields + + fields.length should equal(1) + val uriField: CaseClassField = fields.head + uriField.annotations.length should equal(1) + uriField.annotations.exists(_.annotationType() == classOf[JsonProperty]) should be(true) + uriField.annotations.head.asInstanceOf[JsonProperty].value() should equal("file") + } + + test("CaseClassField.createFields sees inherited JsonProperty annotation 9") { + val fields = deserializerFor(classOf[LoadableFolder]).fields + + fields.length should equal(1) + val uriField: CaseClassField = fields.head + uriField.annotations.length should equal(1) + uriField.annotations.exists(_.annotationType() == classOf[JsonProperty]) should be(true) + uriField.annotations.head.asInstanceOf[JsonProperty].value() should equal("folder") + } + + test("Seq[Long]") { + val fields = deserializerFor(classOf[CaseClassWithArrayLong]).fields + + fields.length should equal(1) + val arrayField: CaseClassField = fields.head + arrayField.javaType.getTypeName should be( + "[array type, component type: [simple type, class long]]") + } + + private[this] def deserializerFor(clazz: Class[_]): CaseClassDeserializer = { + val javaType: JavaType = mapper.constructType(clazz) + new CaseClassDeserializer( + javaType = javaType, + mapper.getDeserializationConfig, + mapper.getDeserializationConfig.introspect(javaType), + Some(ScalaValidator()) + ) + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/CaseClassValidationsTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/CaseClassValidationsTest.scala new file mode 100644 index 0000000000..592567c887 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/CaseClassValidationsTest.scala @@ -0,0 +1,501 @@ +package com.twitter.util.jackson.caseclass + +import com.twitter.util.jackson._ +import com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException.ValidationError +import com.twitter.util.jackson.caseclass.exceptions.{ + CaseClassFieldMappingException, + CaseClassMappingException +} +import com.twitter.util.validation.conversions.PathOps._ +import jakarta.validation.Path.ParameterNode +import jakarta.validation.{ElementKind, UnexpectedTypeException} +import java.time.format.DateTimeFormatter +import java.time.{LocalDate, LocalDateTime} +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +private object CaseClassValidationsTest { + val now: LocalDateTime = + LocalDateTime.parse("2015-04-09T05:17:15Z", DateTimeFormatter.ISO_OFFSET_DATE_TIME) + val BaseCar: Car = { + Car( + id = 1, + make = CarMake.Ford, + model = "Model-T", + year = 2000, + owners = Seq(), + numDoors = 2, + manual = true, + ownershipStart = now, + ownershipEnd = now.plusYears(15), + warrantyStart = Some(now), + warrantyEnd = Some(now.plusYears(5)), + passengers = Seq.empty[Person] + ) + } +} + +@RunWith(classOf[JUnitRunner]) +class CaseClassValidationsTest + extends AnyFunSuite + with BeforeAndAfterAll + with Matchers + with ScalaObjectMapperFunctions { + import CaseClassValidationsTest._ + + /* Class under test */ + override protected val mapper: ScalaObjectMapper = ScalaObjectMapper() + + test("deserialization#generic types") { + // passes validation + parse[GenericTestCaseClassWithValidation[String]]("""{"data" : "widget"}""") should equal( + GenericTestCaseClassWithValidation("widget")) + + val e1 = intercept[CaseClassMappingException] { + parse[GenericTestCaseClassWithValidationAndMultipleArgs[String]]( + """{"data" : "", "number" : 3}""") + } + e1.errors.size should equal(2) + val e2 = intercept[CaseClassMappingException] { + parse[GenericTestCaseClassWithValidationAndMultipleArgs[String]]( + """{"data" : "widget", "number" : 3}""") + } + e2.errors.size should equal(1) + // passes validation + parse[GenericTestCaseClassWithValidationAndMultipleArgs[String]]( + """{"data" : "widget", "number" : 5}""") + } + + test("enums#default validation") { + val e = intercept[CaseClassMappingException] { + parse[CaseClassWithNotEmptyValidation]("""{ + "name" : "", + "make" : "foo" + }""") + } + e.errors.map { _.getMessage } should equal( + Seq( + """make: 'foo' is not a valid CarMakeEnum with valid values: ford, vw""", + "name: must not be empty") + ) + } + + test("enums#invalid validation") { + // the name field is annotated with @ThrowsRuntimeExceptionConstraint which is implemented + // by the ThrowsRuntimeExceptionConstraintValidator...that simply throws a RuntimeException + val e = intercept[RuntimeException] { + parse[CaseClassWithInvalidValidation]("""{ + "name" : "Bob", + "make" : "foo" + }""") + } + + e.getMessage should be("validator foo error") + } + + test("enums#invalid validation 1") { + // the name field is annotated with @NoValidatorImplConstraint which has no constraint validator + // specified nor registered and thus errors when applied. + val e = intercept[UnexpectedTypeException] { + parse[CaseClassWithNoValidatorImplConstraint]("""{ + "name" : "Bob", + "make" : "foo" + }""") + } + + e.getMessage.contains( + "No validator could be found for constraint 'interface com.twitter.util.jackson.NoValidatorImplConstraint' validating type 'java.lang.String'. Check configuration for 'CaseClassWithNoValidatorImplConstraint.name'") should be( + true) + } + + test("method validation exceptions") { + val e = intercept[RuntimeException] { + parse[CaseClassWithMethodValidationException]( + """{ + | "name" : "foo barr", + | "passphrase" : "the quick brown fox jumps over the lazy dog" + |}""".stripMargin) + } + e.getMessage.contains("oh noes") + } + + /* + Notes: + + ValidationError.Field types will not have a root bean instance and the violation property path + will include the class name, e.g., for a violation on the `id` parameter of the `RentalStation` case + class, the property path would be: `RentalStation.id`. + + ValidationError.Method types will have a root bean instance (since they require an instance in order for + the method to be invoked) and the violation property path will not include the class name since this + is understood to be known since the method validations typically have to be manually invoked, e.g., + for the method `validateState` that is annotated with field `state`, the property path of a violation would be: + `validateState.state`. + */ + + test("class and field level validations#success") { + parse[Car](BaseCar) + } + + test("class and field level validations#top-level failed validations") { + val value = BaseCar.copy(id = 2, year = 1910) + val e = intercept[CaseClassMappingException] { + parse[Car](value) + } + + e.errors.size should equal(1) + + val error = e.errors.head + error.path should equal(CaseClassFieldMappingException.PropertyPath.leaf("year")) + error.reason.message should equal("must be greater than or equal to 2000") + error.reason.detail match { + case ValidationError(violation, ValidationError.Field, None) => + violation.getPropertyPath.toString should equal("Car.year") + violation.getMessage should equal("must be greater than or equal to 2000") + violation.getInvalidValue should equal(1910) + violation.getRootBeanClass should equal(classOf[Car]) + violation.getRootBean == null should be( + true + ) // ValidationError.Field types won't have a root bean instance + case _ => fail() + } + } + + test("method validation#top-level") { + val address = Address( + street = Some("123 Main St."), + city = "Tampa", + state = "FL" // invalid + ) + + val e = intercept[CaseClassMappingException] { + parse[Address](address) + } + + e.errors.size should equal(1) + val errors = e.errors + + errors.head.path should equal(CaseClassFieldMappingException.PropertyPath.Empty) + errors.head.reason.message should equal("state must be one of [CA, MD, WI]") + errors.head.reason.detail.getClass should equal(classOf[ValidationError]) + errors.head.reason.detail match { + case ValidationError(violation, ValidationError.Method, None) => + violation.getMessage should equal("state must be one of [CA, MD, WI]") + violation.getInvalidValue should equal(address) + violation.getRootBeanClass should equal(classOf[Address]) + violation.getRootBean should equal( + address + ) // ValidationError.Method types have a root bean instance + violation.getPropertyPath.toString should equal( + "validateState" + ) // the @MethodValidation annotation does not specify a field. + val leafNode = violation.getPropertyPath.getLeafNode + leafNode.getKind should equal(ElementKind.PROPERTY) + leafNode.getName should equal("validateState") + case _ => + fail() + } + } + + test("class and field level validations#nested failed validations") { + val invalidAddress = + Address( + street = Some(""), // invalid + city = "", // invalid + state = "FL" // invalid + ) + + val owners = Seq( + Person( + id = 1, + name = "joe smith", + dob = Some(LocalDate.now), + age = None, + address = Some(invalidAddress) + ) + ) + val car = BaseCar.copy(owners = owners) + + val e = intercept[CaseClassMappingException] { + parse[Car](car) + } + + e.errors.size should equal(2) + val errors = e.errors + + errors.head.path should equal( + CaseClassFieldMappingException.PropertyPath + .leaf("city").withParent("address").withParent("owners")) + errors.head.reason.message should equal("must not be empty") + errors.head.reason.detail match { + case ValidationError(violation, ValidationError.Field, None) => + violation.getPropertyPath.toString should equal("Address.city") + violation.getMessage should equal("must not be empty") + violation.getInvalidValue should equal("") + violation.getRootBeanClass should equal(classOf[Address]) + violation.getRootBean == null should be( + true + ) // ValidationError.Field types won't have a root bean instance + case _ => + fail() + } + + errors(1).path should equal( + CaseClassFieldMappingException.PropertyPath + .leaf("street").withParent("address").withParent("owners") + ) + errors(1).reason.message should equal("must not be empty") + errors(1).reason.detail match { + case ValidationError(violation, ValidationError.Field, None) => + violation.getPropertyPath.toString should equal("Address.street") + violation.getMessage should equal("must not be empty") + violation.getInvalidValue should equal("") + violation.getRootBeanClass should equal(classOf[Address]) + violation.getRootBean == null should be( + true + ) // ValidationError.Field types won't have a root bean instance + case _ => + fail() + } + } + + test("class and field level validations#nested method validations") { + val owners = Seq( + Person( + id = 2, + name = "joe smith", + dob = Some(LocalDate.now), + age = None, + address = Some(Address(city = "pyongyang", state = "KP" /* invalid */ )) + ) + ) + val car: Car = BaseCar.copy(owners = owners) + + val e = intercept[CaseClassMappingException] { + parse[Car](car) + } + + e.errors.size should equal(1) + val errors = e.errors + + errors.head.path should equal( + CaseClassFieldMappingException.PropertyPath + .leaf("address").withParent("owners") + ) + errors.head.reason.message should equal("state must be one of [CA, MD, WI]") + errors.head.reason.detail.getClass should equal(classOf[ValidationError]) + errors.head.reason.detail match { + case ValidationError(violation, ValidationError.Method, None) => + violation.getMessage should equal("state must be one of [CA, MD, WI]") + violation.getInvalidValue should equal(Address(city = "pyongyang", state = "KP")) + violation.getRootBeanClass should equal(classOf[Address]) + violation.getRootBean should equal( + Address(city = "pyongyang", state = "KP") + ) // ValidationError.Method types have a root bean instance + violation.getPropertyPath.toString should equal( + "validateState" + ) // the @MethodValidation annotation does not specify a field. + val leafNode = violation.getPropertyPath.getLeafNode + leafNode.getKind should equal(ElementKind.PROPERTY) + leafNode.getName should equal("validateState") + case _ => + fail() + } + + errors.map(_.getMessage) should equal(Seq("owners.address: state must be one of [CA, MD, WI]")) + } + + test("class and field level validations#end before start") { + val value: Car = + BaseCar.copy(ownershipStart = BaseCar.ownershipEnd, ownershipEnd = BaseCar.ownershipStart) + val e = intercept[CaseClassMappingException] { + parse[Car](value) + } + + e.errors.size should equal(1) + val errors = e.errors + + errors.head.path should equal(CaseClassFieldMappingException.PropertyPath.leaf("ownership_end")) + errors.head.reason.message should equal( + "ownershipEnd [2015-04-09T05:17:15] must be after ownershipStart [2030-04-09T05:17:15]") + errors.head.reason.detail.getClass should equal(classOf[ValidationError]) + errors.head.reason.detail match { + case ValidationError(violation, ValidationError.Method, None) => + violation.getMessage should equal( + "ownershipEnd [2015-04-09T05:17:15] must be after ownershipStart [2030-04-09T05:17:15]") + violation.getInvalidValue should equal(value) + violation.getRootBeanClass should equal(classOf[Car]) + violation.getRootBean should equal( + value + ) // ValidationError.Method types have a root bean instance + violation.getPropertyPath.toString should equal( + "ownershipTimesValid.ownershipEnd" + ) // the @MethodValidation annotation specifies a single field. + val leafNode = violation.getPropertyPath.getLeafNode + leafNode.getKind should equal(ElementKind.PARAMETER) + leafNode.getName should equal("ownershipEnd") + leafNode.as(classOf[ParameterNode]) // shouldn't blow up + case _ => + fail() + } + } + + test("class and field level validations#optional end before start") { + val value: Car = + BaseCar.copy(warrantyStart = BaseCar.warrantyEnd, warrantyEnd = BaseCar.warrantyStart) + val e = intercept[CaseClassMappingException] { + parse[Car](value) + } + + e.errors.size should equal(2) + val errors = e.errors + + errors.head.path should equal(CaseClassFieldMappingException.PropertyPath.leaf("warranty_end")) + errors.head.reason.message should equal( + "warrantyEnd [2015-04-09T05:17:15] must be after warrantyStart [2020-04-09T05:17:15]" + ) + errors.head.reason.detail.getClass should equal(classOf[ValidationError]) + errors.head.reason.detail match { + case ValidationError(violation, ValidationError.Method, None) => + violation.getMessage should equal( + "warrantyEnd [2015-04-09T05:17:15] must be after warrantyStart [2020-04-09T05:17:15]") + violation.getInvalidValue should equal(value) + violation.getRootBeanClass should equal(classOf[Car]) + violation.getRootBean should equal( + value + ) // ValidationError.Method types have a root bean instance + violation.getPropertyPath.toString should equal( + "warrantyTimeValid.warrantyEnd" + ) // the @MethodValidation annotation specifies two fields. + val leafNode = violation.getPropertyPath.getLeafNode + leafNode.getKind should equal(ElementKind.PARAMETER) + leafNode.getName should equal("warrantyEnd") + leafNode.as(classOf[ParameterNode]) // shouldn't blow up + case _ => + fail() + } + + errors(1).path should equal(CaseClassFieldMappingException.PropertyPath.leaf("warranty_start")) + errors(1).reason.message should equal( + "warrantyEnd [2015-04-09T05:17:15] must be after warrantyStart [2020-04-09T05:17:15]" + ) + errors(1).reason.detail.getClass should equal(classOf[ValidationError]) + errors(1).reason.detail match { + case ValidationError(violation, ValidationError.Method, None) => + violation.getMessage should equal( + "warrantyEnd [2015-04-09T05:17:15] must be after warrantyStart [2020-04-09T05:17:15]") + violation.getInvalidValue should equal(value) + violation.getRootBeanClass should equal(classOf[Car]) + violation.getRootBean should equal( + value + ) // ValidationError.Method types have a root bean instance + violation.getPropertyPath.toString should equal( + "warrantyTimeValid.warrantyStart" + ) // the @MethodValidation annotation specifies two fields. + val leafNode = violation.getPropertyPath.getLeafNode + leafNode.getKind should equal(ElementKind.PARAMETER) + leafNode.getName should equal("warrantyStart") + leafNode.as(classOf[ParameterNode]) // shouldn't blow up + case _ => + fail() + } + } + + test("class and field level validations#no start with end") { + val value: Car = BaseCar.copy(warrantyStart = None, warrantyEnd = BaseCar.warrantyEnd) + val e = intercept[CaseClassMappingException] { + parse[Car](value) + } + + e.errors.size should equal(2) + val errors = e.errors + + errors.head.path should equal(CaseClassFieldMappingException.PropertyPath.leaf("warranty_end")) + errors.head.reason.message should equal( + "both warrantyStart and warrantyEnd are required for a valid range" + ) + errors.head.reason.detail.getClass should equal(classOf[ValidationError]) + errors.head.reason.detail match { + case ValidationError(violation, ValidationError.Method, None) => + violation.getMessage should equal( + "both warrantyStart and warrantyEnd are required for a valid range") + violation.getInvalidValue should equal(value) + violation.getRootBeanClass should equal(classOf[Car]) + violation.getRootBean should equal( + value + ) // ValidationError.Method types have a root bean instance + violation.getPropertyPath.toString should equal( + "warrantyTimeValid.warrantyEnd" + ) // the @MethodValidation annotation specifies two fields. + val leafNode = violation.getPropertyPath.getLeafNode + leafNode.getKind should equal(ElementKind.PARAMETER) + leafNode.getName should equal("warrantyEnd") + leafNode.as(classOf[ParameterNode]) // shouldn't blow up + case _ => + fail() + } + + errors(1).path should equal(CaseClassFieldMappingException.PropertyPath.leaf("warranty_start")) + errors(1).reason.message should equal( + "both warrantyStart and warrantyEnd are required for a valid range" + ) + errors(1).reason.detail.getClass should equal(classOf[ValidationError]) + errors(1).reason.detail match { + case ValidationError(violation, ValidationError.Method, None) => + violation.getMessage should equal( + "both warrantyStart and warrantyEnd are required for a valid range") + violation.getInvalidValue should equal(value) + violation.getRootBeanClass should equal(classOf[Car]) + violation.getRootBean should equal( + value + ) // ValidationError.Method types have a root bean instance + violation.getPropertyPath.toString should equal( + "warrantyTimeValid.warrantyStart" + ) // the @MethodValidation annotation specifies two fields. + val leafNode = violation.getPropertyPath.getLeafNode + leafNode.getKind should equal(ElementKind.PARAMETER) + leafNode.getName should equal("warrantyStart") + leafNode.as(classOf[ParameterNode]) // shouldn't blow up + case _ => + fail() + } + } + + test("class and field level validations#errors sorted by message") { + val first = + CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.Empty, + CaseClassFieldMappingException.Reason("123")) + val second = + CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.Empty, + CaseClassFieldMappingException.Reason("aaa")) + val third = CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.leaf("bla"), + CaseClassFieldMappingException.Reason("zzz")) + val fourth = + CaseClassFieldMappingException( + CaseClassFieldMappingException.PropertyPath.Empty, + CaseClassFieldMappingException.Reason("xxx")) + + val unsorted = Set(third, second, fourth, first) + val expectedSorted = Seq(first, second, third, fourth) + + CaseClassMappingException(unsorted).errors should equal((expectedSorted)) + } + + test("class and field level validations#option[string] validation") { + val address = Address( + street = Some(""), // invalid + city = "New Orleans", + state = "LA" + ) + + intercept[CaseClassMappingException] { + mapper.parse[Address](mapper.writeValueAsBytes(address)) + } + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/OptionalValidationTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/OptionalValidationTest.scala new file mode 100644 index 0000000000..aaf134cb43 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/OptionalValidationTest.scala @@ -0,0 +1,87 @@ +package com.twitter.util.jackson.caseclass + +import com.twitter.util.WrappedValue +import com.twitter.util.jackson.ScalaObjectMapper +import com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException +import com.twitter.util.validation.MethodValidation +import com.twitter.util.validation.constraints.OneOf +import com.twitter.util.validation.engine.MethodValidationResult +import jakarta.validation.constraints.{Min, NotEmpty} +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +private object OptionalValidationTest { + case class State( + @OneOf(Array("active", "inactive")) + state: String) + extends WrappedValue[String] + + case class Threshold( + @NotEmpty id: Option[String], + @Min(0) lowerBound: Int, + @Min(0) upperBound: Int, + state: State) { + @MethodValidation + def method: MethodValidationResult = MethodValidationResult.validIfTrue( + lowerBound <= upperBound, + "Lower Bound cannot be greater than Upper Bound" + ) + } +} + +@RunWith(classOf[JUnitRunner]) +class OptionalValidationTest extends AnyFunSuite with Matchers { + import OptionalValidationTest._ + + private val defaultMapper = + ScalaObjectMapper.builder.objectMapper + + private val nullValidationMapper = + ScalaObjectMapper.builder.withNoValidation.objectMapper + + test("default mapper will trigger NotEmpty validation") { + val invalid = Threshold(Some(""), 1, 4, State("active")) + intercept[CaseClassMappingException] { + check(defaultMapper, invalid) + } + } + + test("default mapper will trigger lower and upper min validation") { + val invalid = Threshold(Some(""), -1, -1, State("active")) + intercept[CaseClassMappingException] { + check(defaultMapper, invalid) + } + } + + test("default mapper will trigger invalid state") { + val invalid = Threshold(Some("id"), 4, 6, State("other")) + intercept[CaseClassMappingException] { + check(defaultMapper, invalid) + } + } + + test("default mapper will trigger method validation") { + val invalid = Threshold(None, 8, 4, State("active")) + intercept[CaseClassMappingException] { + check(defaultMapper, invalid) + } + } + + test("no-op mapper will allow values that are considered invalid") { + val invalid = Threshold(Some(""), -1, -3, State("other")) + check(nullValidationMapper, invalid) + } + + test("no-op mapper will never call method validation") { + val invalid = Threshold(None, 8, 4, State("active")) + check(nullValidationMapper, invalid) + } + + def check[T: Manifest](mapper: ScalaObjectMapper, check: T): Unit = { + val str = mapper.writeValueAsString(check) + val result = mapper.parse[T](str) + check should equal(result) + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/RichJsonProcessingExceptionTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/RichJsonProcessingExceptionTest.scala new file mode 100644 index 0000000000..7afade0e5c --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/RichJsonProcessingExceptionTest.scala @@ -0,0 +1,92 @@ +package com.twitter.util.jackson.caseclass + +import com.fasterxml.jackson.core.{JsonGenerator, JsonParseException, JsonParser} +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.`type`.SimpleType +import com.fasterxml.jackson.databind.exc.{ + IgnoredPropertyException, + InvalidDefinitionException, + MismatchedInputException +} +import com.twitter.util.jackson.CaseClassWithAllTypes +import com.twitter.util.jackson.caseclass.exceptions._ +import com.twitter.util.mock.Mockito +import java.util +import java.util.Collections +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class RichJsonProcessingExceptionTest extends AnyFunSuite with Matchers with Mockito { + private[this] val genericType = SimpleType.constructUnsafe(classOf[String]) + private val message = "this is a test" + + test("JsonMappingException with cause") { + val cause = new IllegalArgumentException(message) + val e = new JsonMappingException(null, message, cause) + + // should be the mapping exception message + e.errorMessage should equal(message) + } + + test("JsonMappingException with no cause") { + val e = new JsonMappingException(null, message) + + // should be the mapping exception message + e.errorMessage should equal(message) + } + + test("MismatchedInputException with cause") { + val mockJsonParser = mock[JsonParser] + + val e = MismatchedInputException.from( + mockJsonParser, + genericType, + message + ) + + // should be from the cause + e.errorMessage should equal(message) + } + + test("InvalidDefinitionException with no cause") { + val mockJsonGenerator = mock[JsonGenerator] + + val e = InvalidDefinitionException.from(mockJsonGenerator, message, genericType) + + // should be from the invalid definition exception message via `getOriginalMessage` + e.errorMessage should equal(message) + } + + test("IgnoredPropertyException with no cause") { + val mockJsonParser = mock[JsonParser] + + val e = IgnoredPropertyException.from( + mockJsonParser, + CaseClassWithAllTypes.getClass, + "noname", + Collections.emptySet().asInstanceOf[util.Collection[Object]]) + + // should be from the ignored property exception message via `getOriginalMessage` + e.errorMessage should equal( + """Ignored field "noname" (class com.twitter.util.jackson.CaseClassWithAllTypes$) encountered; mapper configured not to allow this""") + } + + test("JsonParseException with cause") { + val cause = new IllegalArgumentException(message) + val e = new JsonParseException(null, "this is not the message", cause) + + // should be from the cause + e.errorMessage should equal(message) + } + + test("JsonParseException with no cause") { + val e = new JsonParseException(null, message) + + // should be from the parse exception message via `getOriginalMessage` + e.errorMessage should equal(message) + } + +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/TypesTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/TypesTest.scala new file mode 100644 index 0000000000..efa1656de8 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclass/TypesTest.scala @@ -0,0 +1,34 @@ +package com.twitter.util.jackson.caseclass + +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class TypesTest extends AnyFunSuite with Matchers { + + test("types#handles null") { + val clazz = Types.wrapperType(null) + clazz != null should be(true) + clazz.getSimpleName should equal(classOf[Null].getSimpleName) + } + + test("types#all types") { + classOf[Null].isAssignableFrom(Types.wrapperType(null)) should be(true) + + classOf[java.lang.Byte].isAssignableFrom(Types.wrapperType(java.lang.Byte.TYPE)) should be(true) + classOf[java.lang.Short] + .isAssignableFrom(Types.wrapperType(java.lang.Short.TYPE)) should be(true) + classOf[java.lang.Character] + .isAssignableFrom(Types.wrapperType(java.lang.Character.TYPE)) should be(true) + classOf[java.lang.Integer] + .isAssignableFrom(Types.wrapperType(java.lang.Integer.TYPE)) should be(true) + classOf[java.lang.Long].isAssignableFrom(Types.wrapperType(java.lang.Long.TYPE)) should be(true) + classOf[java.lang.Double] + .isAssignableFrom(Types.wrapperType(java.lang.Double.TYPE)) should be(true) + classOf[java.lang.Boolean] + .isAssignableFrom(Types.wrapperType(java.lang.Boolean.TYPE)) should be(true) + classOf[java.lang.Void].isAssignableFrom(Types.wrapperType(java.lang.Void.TYPE)) should be(true) + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala new file mode 100644 index 0000000000..0c1f19436d --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala @@ -0,0 +1,841 @@ +package com.twitter.util.jackson + +import com.fasterxml.jackson.annotation.JsonSubTypes.Type +import com.fasterxml.jackson.annotation._ +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.`type`.TypeReference +import com.fasterxml.jackson.databind._ +import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonNaming} +import com.fasterxml.jackson.databind.deser.std.NumberDeserializers.{ + BigDecimalDeserializer, + NumberDeserializer +} +import com.fasterxml.jackson.databind.deser.std.StdDeserializer +import com.fasterxml.jackson.databind.node.ValueNode +import com.fasterxml.jackson.module.scala.JsonScalaEnumeration +import com.twitter.util.{Time, WrappedValue} +import com.twitter.util.jackson.caseclass.SerdeLogging +import com.twitter.util.validation.MethodValidation +import com.twitter.util.validation.constraints.{OneOf, UUID} +import com.twitter.util.validation.engine.MethodValidationResult +import com.twitter.{util => ctu} +import jakarta.validation.Payload +import jakarta.validation.constraints._ +import java.time.format.DateTimeFormatter +import java.time.temporal.Temporal +import java.time.{LocalDate, LocalDateTime} +import scala.math.BigDecimal.RoundingMode + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes( + Array( + new Type(value = classOf[BoundaryA], name = "a"), + new Type(value = classOf[BoundaryB], name = "b"), + new Type(value = classOf[BoundaryC], name = "c") + ) +) +trait Boundary { + def value: String +} +case class BoundaryA(value: String) extends Boundary +case class BoundaryB(value: String) extends Boundary +case class BoundaryC(value: String) extends Boundary + +case class GG[T, U, V, X, Y, Z](t: T, u: U, v: V, x: X, y: Y, z: Z) +case class B(value: String) +case class D(value: Int) +case class AGeneric[T](b: Option[T]) +case class AAGeneric[T, U](b: Option[T], c: Option[U]) +case class C(a: AGeneric[B]) +case class E[T, U, V](a: AAGeneric[T, U], b: Option[V]) +case class F[T, U, V, X, Y, Z](a: Option[T], b: Seq[U], c: Option[V], d: X, e: Either[Y, Z]) +case class WithTypeBounds[A <: Boundary](a: Option[A]) +case class G[T, U, V, X, Y, Z](gg: GG[T, U, V, X, Y, Z]) + +case class SomeNumberType[N <: Number]( + @JsonDeserialize(contentAs = classOf[Number], using = classOf[NumberDeserializer]) n: Option[N]) + +object Weekday extends Enumeration { + type Weekday = Value + val Mon, Tue, Wed, Thu, Fri, Sat, Sun = Value +} +class WeekdayType extends TypeReference[Weekday.type] +object Month extends Enumeration { + type Month = Value + val Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec = Value +} +class MonthType extends TypeReference[Month.type] +case class BasicDate( + @JsonScalaEnumeration(classOf[MonthType]) month: Month.Month, + day: Int, + year: Int, + @JsonScalaEnumeration(classOf[WeekdayType]) weekday: Weekday.Weekday) + +case class WithOptionalScalaEnumeration( + @JsonScalaEnumeration(classOf[MonthType]) month: Option[Month.Month]) + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes( + Array( + new Type(value = classOf[Rectangle], name = "rectangle"), + new Type(value = classOf[Circle], name = "circle") + ) +) +sealed trait Shape +case class Rectangle(@Min(0) width: Int, @Min(0) height: Int) extends Shape +case class Circle(@Min(0) radius: Int) extends Shape +case class View(shapes: Seq[Shape]) +case class OptionalView(shapes: Seq[Shape], optional: Option[Shape]) + +object TestJsonCreator { + @JsonCreator + def apply(s: String): TestJsonCreator = TestJsonCreator(s.toInt) +} +case class TestJsonCreator(int: Int) + +object TestJsonCreator2 { + @JsonCreator + def apply(strings: Seq[String]): TestJsonCreator2 = TestJsonCreator2(strings.map(_.toInt)) +} +case class TestJsonCreator2(ints: Seq[Int], default: String = "Hello, World") + +object TestJsonCreatorWithValidation { + @JsonCreator + def apply(@NotEmpty s: String): TestJsonCreatorWithValidation = + TestJsonCreatorWithValidation(s.toInt) +} +case class TestJsonCreatorWithValidation(int: Int) + +object TestJsonCreatorWithValidations { + @JsonCreator + def apply(@NotEmpty @OneOf(Array("42", "137")) s: String): TestJsonCreatorWithValidations = + TestJsonCreatorWithValidations(s.toInt) +} +case class TestJsonCreatorWithValidations(int: Int) + +case class CaseClassWithMultipleConstructors(number1: Long, number2: Long, number3: Long) { + def this(numberAsString1: String, numberAsString2: String, numberAsString3: String) { + this(numberAsString1.toLong, numberAsString2.toLong, numberAsString3.toLong) + } +} + +case class CaseClassWithMultipleConstructorsAnnotated(number1: Long, number2: Long, number3: Long) { + @JsonCreator + def this(numberAsString1: String, numberAsString2: String, numberAsString3: String) { + this(numberAsString1.toLong, numberAsString2.toLong, numberAsString3.toLong) + } +} + +case class CaseClassWithMultipleConstructorsAnnotatedAndValidations( + number1: Long, + number2: Long, + uuid: String) { + @JsonCreator + def this( + @NotEmpty numberAsString1: String, + @OneOf(Array("10001", "20002", "30003")) numberAsString2: String, + @UUID thirdArgument: String + ) { + this(numberAsString1.toLong, numberAsString2.toLong, thirdArgument) + } +} + +case class TimeWithFormat(@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX") when: Time) + +/* Note: the decoder automatically changes "_i" to "i" for de/serialization: + * See CaseClassField#jsonNameForField */ +trait Aumly { @JsonProperty("i") def _i: Int; @JsonProperty("j") def _j: String } +case class Aum(_i: Int, _j: String) extends Aumly + +trait Bar { + @JsonProperty("helloWorld") @TestInjectableValue(value = "accept") + def hello: String +} +case class FooBar(hello: String) extends Bar + +trait Baz extends Bar { + @JsonProperty("goodbyeWorld") + def hello: String +} +case class FooBaz(hello: String) extends Baz + +trait BarBaz { + @JsonProperty("goodbye") + def hello: String +} +case class FooBarBaz(hello: String) + extends BarBaz + with Bar // will end up with BarBaz @JsonProperty value as trait linearization is "right-to-left" + +trait Loadable { + @JsonProperty("url") + def uri: String +} +abstract class Resource { + @JsonProperty("resource") + def uri: String +} +case class File(@JsonProperty("file") uri: String) extends Resource +case class Folder(@JsonProperty("folder") uri: String) extends Resource + +abstract class LoadableResource extends Loadable { + @JsonProperty("resource") + override def uri: String +} +case class LoadableFile(@JsonProperty("file") uri: String) extends LoadableResource +case class LoadableFolder(@JsonProperty("folder") uri: String) extends LoadableResource + +trait TestTrait { + @JsonProperty("oldness") + def age: Int + @NotEmpty + def name: String +} +@JsonNaming +case class TestTraitImpl( + @JsonProperty("ageness") age: Int, // should override inherited annotation from trait + @TestInjectableValue name: String, // should have two annotations, one from trait and one here + @TestInjectableValue dateTime: LocalDate, + @JsonProperty foo: String, + @JsonDeserialize(contentAs = classOf[BigDecimal], using = classOf[BigDecimalDeserializer]) + double: BigDecimal, + @JsonIgnore ignoreMe: String) + extends TestTrait { + + lazy val testFoo: String = "foo" + lazy val testBar: String = "bar" +} + +sealed trait CarType { + @JsonValue + def toJson: String +} +object Volvo extends CarType { + override def toJson: String = "volvo" +} +object Audi extends CarType { + override def toJson: String = "audi" +} +object Volkswagen extends CarType { + override def toJson: String = "vw" +} + +case class Vehicle(vin: String, `type`: CarType) + +sealed trait ZeroOrOne +object Zero extends ZeroOrOne +object One extends ZeroOrOne +object Two extends ZeroOrOne + +case class CaseClassWithZeroOrOne(id: ZeroOrOne) + +case class CaseClass(id: Long, name: String) +case class CaseClassIdAndOption(id: Long, name: Option[String]) +@JsonIgnoreProperties(ignoreUnknown = false) +case class StrictCaseClassIdAndOption(id: Long, name: Option[String]) + +case class SimpleClassWithInjection(@TestInjectableValue(value = "accept") hello: String) + +@JsonIgnoreProperties(ignoreUnknown = false) +case class StrictCaseClass(id: Long, name: String) + +@JsonIgnoreProperties(ignoreUnknown = false) +case class StrictCaseClassWithOption(value: Option[String] = None) + +@JsonIgnoreProperties(ignoreUnknown = true) +case class LooseCaseClass(id: Long, name: String) + +case class CaseClassWithLazyVal(id: Long) { + lazy val woo = "yeah" +} + +case class GenericTestCaseClass[T](data: T) +case class GenericTestCaseClassWithValidation[T](@NotEmpty data: T) +case class GenericTestCaseClassWithValidationAndMultipleArgs[T]( + @NotEmpty data: T, + @Min(5) number: Int) + +case class Page[T](data: List[T], pageSize: Int, next: Option[Long], previous: Option[Long]) + +case class CaseClassWithGeneric[T](inside: GenericTestCaseClass[T]) + +case class CaseClassWithOptionalGeneric[T](inside: Option[GenericTestCaseClass[T]]) + +case class CaseClassWithTypes[T, U](first: T, second: U) + +case class CaseClassWithMapTypes[T, U](data: Map[T, U]) + +case class CaseClassWithManyTypes[R, S, T](one: R, two: S, three: T) + +case class CaseClassIgnoredFieldInConstructorNoDefault( + @JsonIgnore id: Long, + name: String, + description: String) + +case class CaseClassIgnoredFieldInConstructorWithDefault( + @JsonIgnore id: Long = 42L, + name: String, + description: String) + +case class CaseClassWithIgnoredField(id: Long) { + @JsonIgnore + val ignoreMe = "Foo" +} + +@JsonIgnoreProperties(Array("ignore_me", "feh")) +case class CaseClassWithIgnoredFieldsMatchAfterToSnakeCase(id: Long) { + val ignoreMe = "Foo" + val feh = "blah" +} + +@JsonIgnoreProperties(Array("ignore_me", "feh")) +case class CaseClassWithIgnoredFieldsExactMatch(id: Long) { + val ignore_me = "Foo" + val feh = "blah" +} + +case class CaseClassWithTransientField(id: Long) { + @transient + val lol = "asdf" +} + +case class CaseClassWithLazyField(id: Long) { + @JsonIgnore lazy val lol = "asdf" +} + +case class CaseClassWithOverloadedField(id: Long) { + def id(prefix: String): String = prefix + id +} + +case class CaseClassWithOption(value: Option[String] = None) + +case class CaseClassWithJsonNode(value: JsonNode) + +case class CaseClassWithAllTypes( + map: Map[String, String], + set: Set[Int], + string: String, + list: List[Int], + seq: Seq[Int], + indexedSeq: IndexedSeq[Int], + vector: Vector[Int], + bigDecimal: BigDecimal, + bigInt: Int, //TODO: BigInt, + int: Int, + long: Long, + char: Char, + bool: Boolean, + short: Short, + byte: Byte, + float: Float, + double: Double, + any: Any, + anyRef: AnyRef, + intMap: Map[Int, Int] = Map(), + longMap: Map[Long, Long] = Map()) + +case class CaseClassWithException() { + throw JsonMappingException.from(null.asInstanceOf[JsonParser], "Oops!!!") +} + +object OuterObject { + + case class NestedCaseClass(id: Long) + + object InnerObject { + + case class SuperNestedCaseClass(id: Long) + } +} + +case class CaseClassWithSnakeCase(oneThing: String, twoThing: String) + +case class CaseClassWithArrays( + one: String, + two: Array[String], + three: Array[Int], + four: Array[Long], + five: Array[Char], + bools: Array[Boolean], + bytes: Array[Byte], + doubles: Array[Double], + floats: Array[Float]) + +case class CaseClassWithArrayLong(array: Array[Long]) + +case class CaseClassWithArrayListOfIntegers(arraylist: java.util.ArrayList[java.lang.Integer]) + +case class CaseClassWithArrayBoolean(array: Array[Boolean]) + +case class CaseClassWithArrayWrappedValueLong(array: Array[WrappedValueLong]) + +case class CaseClassWithSeqLong(seq: Seq[Long]) + +case class CaseClassWithSeqWrappedValueLong(seq: Seq[WrappedValueLong]) + +case class CaseClassWithValidation(@Min(1) value: Long) + +case class CaseClassWithSeqOfCaseClassWithValidation(seq: Seq[CaseClassWithValidation]) + +case class WrappedValueLongWithValidation(@Min(1) value: Long) extends WrappedValue[Long] + +case class CaseClassWithSeqWrappedValueLongWithValidation(seq: Seq[WrappedValueLongWithValidation]) + +case class Foo(name: String) + +case class CaseClassCharacter(c: Char) + +case class Car( + id: Long, + make: CarMake, + model: String, + @Min(2000) year: Int, + owners: Seq[Person], + @Min(0) numDoors: Int = 4, + manual: Boolean = false, + ownershipStart: LocalDateTime = LocalDateTime.now, + ownershipEnd: LocalDateTime = LocalDateTime.now.plusYears(1), + warrantyStart: Option[LocalDateTime] = None, + warrantyEnd: Option[LocalDateTime] = None, + passengers: Seq[Person] = Seq()) { + + @MethodValidation + def validateId: MethodValidationResult = { + MethodValidationResult.validIfTrue(id % 2 == 1, "id may not be even") + } + + @MethodValidation + def validateYearBeforeNow: MethodValidationResult = { + val thisYear = LocalDate.now.getYear + val yearMoreThanOneYearInFuture: Boolean = + if (year > thisYear) { (year - thisYear) > 1 } + else false + MethodValidationResult.validIfFalse( + yearMoreThanOneYearInFuture, + "Model year can be at most one year newer." + ) + } + + @MethodValidation(fields = Array("ownershipEnd")) + def ownershipTimesValid: MethodValidationResult = + validateTimeRange( + ownershipStart, + ownershipEnd, + "ownershipStart", + "ownershipEnd" + ) + + @MethodValidation(fields = Array("warrantyStart", "warrantyEnd")) + def warrantyTimeValid: MethodValidationResult = + validateTimeRange( + warrantyStart, + warrantyEnd, + "warrantyStart", + "warrantyEnd" + ) + + private[this] def validateTimeRange( + start: Option[LocalDateTime], + end: Option[LocalDateTime], + startProperty: String, + endProperty: String + ): MethodValidationResult = { + + val rangeDefined = start.isDefined && end.isDefined + val partialRange = !rangeDefined && (start.isDefined || end.isDefined) + + if (rangeDefined) + validateTimeRange(start.get, end.get, startProperty, endProperty) + else if (partialRange) + MethodValidationResult.Invalid( + "both %s and %s are required for a valid range".format(startProperty, endProperty) + ) + else + MethodValidationResult.Valid + } + + private[this] def validateTimeRange( + start: LocalDateTime, + end: LocalDateTime, + startProperty: String, + endProperty: String + ): MethodValidationResult = + MethodValidationResult.validIfTrue( + start.isBefore(end), + "%s [%s] must be after %s [%s]" + .format( + endProperty, + DateTimeFormatter.ISO_DATE_TIME.format(end), + startProperty, + DateTimeFormatter.ISO_DATE_TIME.format(start)) + ) +} + +case class PersonWithLogging( + id: Int, + name: String, + age: Option[Int], + age_with_default: Option[Int] = None, + nickname: String = "unknown") + extends SerdeLogging + +case class PersonWithDottedName(id: Int, @JsonProperty("name.last") lastName: String) + +case class SimplePerson(name: String) + +case class PersonWithThings( + id: Int, + name: String, + age: Option[Int], + @Size(min = 1, max = 10) things: Map[String, Things]) + +case class Things(@Size(min = 1, max = 2) names: Seq[String]) + +@JsonNaming +case class CamelCaseSimplePerson(myName: String) + +case class CamelCaseSimplePersonNoAnnotation(myName: String) + +case class CaseClassWithMap(map: Map[String, String]) + +case class CaseClassWithSortedMap(sortedMap: scala.collection.SortedMap[String, Int]) + +case class CaseClassWithSetOfLongs(set: Set[Long]) + +case class CaseClassWithSeqOfLongs(seq: Seq[Long]) + +case class CaseClassWithNestedSeqLong( + seqClass: CaseClassWithSeqLong, + setClass: CaseClassWithSetOfLongs) + +case class Blah(foo: String) + +case class TestIdStringWrapper(id: String) extends WrappedValue[String] + +case class ObjWithTestId(id: TestIdStringWrapper) + +object Obj { + + case class NestedCaseClassInObject(id: String) + + case class NestedCaseClassInObjectWithNestedCaseClassInObjectParam( + nested: NestedCaseClassInObject) + +} + +class TypeAndCompanion +object TypeAndCompanion { + case class NestedCaseClassInCompanion(id: String) +} + +case class WrapperValueClass(underlying: Int) extends AnyVal with WrappedValue[Int] + +case class WrappedValueInt(value: Int) extends WrappedValue[Int] + +case class WrappedValueLong(value: Long) extends WrappedValue[Long] + +case class WrappedValueString(value: String) extends WrappedValue[String] + +case class WrappedValueIntInObj(foo: WrappedValueInt) + +case class WrappedValueStringInObj(foo: WrappedValueString) + +case class WrappedValueLongInObj(foo: WrappedValueLong) + +case class CaseClassWithVal(name: String) { + val `type`: String = "person" +} + +case class CaseClassWithEnum(name: String, make: CarMakeEnum) + +case class CaseClassWithComplexEnums( + name: String, + make: CarMakeEnum, + makeOpt: Option[CarMakeEnum], + makeSeq: Seq[CarMakeEnum], + makeSet: Set[CarMakeEnum]) + +case class CaseClassWithSeqEnum(enumSeq: Seq[CarMakeEnum]) + +case class CaseClassWithOptionEnum(enumOpt: Option[CarMakeEnum]) + +case class CaseClassWithDateTime(dateTime: LocalDateTime) + +case class CaseClassWithIntAndDateTime( + @NotEmpty name: String, + age: Int, + age2: Int, + age3: Int, + dateTime: LocalDateTime, + dateTime2: LocalDateTime, + dateTime3: LocalDateTime, + dateTime4: LocalDateTime, + @NotEmpty dateTime5: Option[LocalDateTime]) + +case class CaseClassWithTwitterUtilDuration(duration: ctu.Duration) + +case class ClassWithFooClassInject(@JacksonInject fooClass: FooClass) + +case class ClassWithFooClassInjectAndDefault(@JacksonInject fooClass: FooClass = FooClass("12345")) + +case class ClassWithQueryParamDateTimeInject(@TestInjectableValue dateTime: Temporal) + +case class CaseClassWithEscapedLong(`1-5`: Long) + +case class CaseClassWithEscapedLongAndAnnotation(@Max(25) `1-5`: Long) + +case class CaseClassWithEscapedString(`1-5`: String) + +case class CaseClassWithEscapedNormalString(`a`: String) + +case class UnicodeNameCaseClass(`winning-id`: Int, name: String) + +case class TestEntityIdsResponse(entityIds: Seq[Long], previousCursor: String, nextCursor: String) + +object TestEntityIdsResponseWithCompanion { + val msg = "im the companion" +} + +case class TestEntityIdsResponseWithCompanion( + entityIds: Seq[Long], + previousCursor: String, + nextCursor: String) + +case class WrappedValueStringMapObject(map: Map[WrappedValueString, String]) + +case class FooClass(id: String) + +case class Group3(id: String) extends SerdeLogging + +case class CaseClassWithNotEmptyValidation(@NotEmpty name: String, make: CarMakeEnum) + +case class CaseClassWithInvalidValidation( + @ThrowsRuntimeExceptionConstraint name: String, + make: CarMakeEnum) + +// a constraint with no provided or registered validator class implementation +case class CaseClassWithNoValidatorImplConstraint( + @NoValidatorImplConstraint name: String, + make: CarMakeEnum) + +case class CaseClassWithMethodValidationException( + @NotEmpty name: String, + passphrase: String) { + @MethodValidation(fields = Array("passphrase")) + def checkPassphrase: MethodValidationResult = { + throw new RuntimeException("oh noes") + } +} + +case class NoConstructorArgs() + +case class CaseClassWithBoolean(foo: Boolean) + +case class CaseClassWithSeqBooleans(foos: Seq[Boolean]) + +case class CaseClassInjectStringWithDefault(@JacksonInject string: String = "DefaultHello") + +case class CaseClassInjectInt(@JacksonInject age: Int) + +case class CaseClassInjectOptionInt(@JacksonInject age: Option[Int]) + +case class CaseClassInjectOptionString(@JacksonInject string: Option[String]) + +case class CaseClassInjectString(@JacksonInject string: String) + +case class CaseClassWithCustomDecimalFormat( + @JsonDeserialize(using = classOf[MyBigDecimalDeserializer]) myBigDecimal: BigDecimal, + @JsonDeserialize(using = classOf[MyBigDecimalDeserializer]) optMyBigDecimal: Option[BigDecimal]) + +case class CaseClassWithLongAndDeserializer( + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + long: Number) + +case class CaseClassWithOptionLongAndDeserializer( + @JsonDeserialize(contentAs = classOf[java.lang.Long]) + optLong: Option[Long]) + +case class CaseClassWithTwoConstructors(id: Long, @NotEmpty name: String) { + def this(id: Long) = this(id, "New User") +} + +case class CaseClassWithThreeConstructors(id: Long, @NotEmpty name: String) { + def this(id: Long) = this(id, "New User") + + def this(name: String) = this(42, name) +} + +case class Person( + id: Int, + @NotEmpty name: String, + dob: Option[LocalDate] = None, + age: Option[Int], + age_with_default: Option[Int] = None, + nickname: String = "unknown", + address: Option[Address] = None) + +class MyBigDecimalDeserializer extends StdDeserializer[BigDecimal](classOf[BigDecimal]) { + override def deserialize(jp: JsonParser, ctxt: DeserializationContext): BigDecimal = { + val jsonNode: ValueNode = jp.getCodec.readTree(jp) + BigDecimal(jsonNode.asText).setScale(2, RoundingMode.HALF_UP) + } + + override def getEmptyValue: BigDecimal = BigDecimal(0) +} + +// allows parsing a JSON null into a null type for this object +class NullableCarMakeDeserializer extends StdDeserializer[CarMakeEnum](classOf[CarMakeEnum]) { + override def deserialize(jp: JsonParser, ctxt: DeserializationContext): CarMakeEnum = { + val jsonNode: ValueNode = jp.getCodec.readTree(jp) + if (jsonNode.isNull) { + null + } else { + CarMakeEnum.valueOf(jsonNode.asText()) + } + } +} + +case class WithNullableCarMake( + nonNullCarMake: CarMakeEnum, + @JsonDeserialize(using = classOf[NullableCarMakeDeserializer]) nullableCarMake: CarMakeEnum) + +case class WithDefaultNullableCarMake( + @JsonDeserialize(using = classOf[NullableCarMakeDeserializer]) nullableCarMake: CarMakeEnum = + CarMakeEnum.ford) + +// is a string type but json types are passed as the value +case class WithJsonStringType(value: String) + +case class WithEmptyJsonProperty(@JsonProperty foo: String) + +case class WithNonemptyJsonProperty(@JsonProperty("bar") foo: String) + +case class WithoutJsonPropertyAnnotation(foo: String) + +case class NamingStrategyJsonProperty(@JsonProperty longFieldName: String) + +case class Address( + @NotEmpty street: Option[String] = None, + @NotEmpty city: String, + @NotEmpty state: String) { + + @MethodValidation + def validateState: MethodValidationResult = + MethodValidationResult.validIfTrue( + state == "CA" || state == "MD" || state == "WI", + "state must be one of [CA, MD, WI]" + ) +} + +trait CaseClassTrait { + @JsonProperty("fedoras") + @Size(min = 1, max = 2) + def names: Seq[String] + + @Min(1L) + def age: Int +} +case class CaseClassTraitImpl(names: Seq[String], @JsonProperty("oldness") age: Int) + extends CaseClassTrait + +package object internal { + case class SimplePersonInPackageObject( // not recommended but used here for testing use case + name: String = "default-name") + + case class SimplePersonInPackageObjectWithoutConstructorParams( + ) // not recommended but used here for testing use case +} + +case class LimiterProfile(id: Long, name: String) +case class LimiterProfiles(profiles: Option[Seq[LimiterProfile]] = None) + +object LimiterProfiles { + def apply(profiles: Seq[LimiterProfile]): LimiterProfiles = { + if (profiles.isEmpty) { + LimiterProfiles() + } else { + LimiterProfiles(Some(profiles)) + } + } +} + +case class CaseClassWithBoxedPrimitives(events: Integer, errors: Integer) + +sealed trait ClusterRequest +case class AddClusterRequest( + @Size(min = 0, max = 30) clusterName: String, + @NotEmpty job: String, + zone: String, + environment: String, + dtab: String, + address: String, + owners: String = "", + dedicated: Boolean = true, + enabled: Boolean = true, + description: String = "") + extends ClusterRequest { + + @MethodValidation(fields = Array("clusterName"), payload = Array(classOf[PatternNotMatched])) + def validateClusterName: MethodValidationResult = { + validateName(clusterName) + } + + private def validateName(name: String): MethodValidationResult = { + val regex = "[0-9a-zA-Z_\\-\\.>]+" + MethodValidationResult.validIfTrue( + name.matches(regex), + s"$name is invalid. Only alphanumeric and special characters from (_,-,.,>) are allowed.", + Some(PatternNotMatched(name, regex))) + } +} + +case class PatternNotMatched(pattern: String, regex: String) extends Payload + +case class Point(abscissa: Int, ordinate: Int) { + def area: Int = abscissa * ordinate +} + +case class Points(first: Point, second: Point) + +trait PointMixin { + @JsonProperty("x") @Min(0) @Max(100) def abscissa: Int + @JsonProperty("y") @Min(0) @Max(100) def ordinate: Int + @JsonIgnore def area: Int +} + +@JsonIgnoreType +case class IgnoreMe(id: Long) + +case class ContainsAnIgnoreTypeNoDefault(ignored: IgnoreMe, name: String, description: String) +case class ContainsAnIgnoreTypeWithDefault( + ignored: IgnoreMe = IgnoreMe(42L), + name: String, + description: String) + +object Views { + class Public + class Internal extends Public +} + +case class Item( + @JsonView(Array(classOf[Views.Public])) id: Long = 1L, + @JsonView(Array(classOf[Views.Public])) name: String = "", + @JsonView(Array(classOf[Views.Internal])) owner: String = "") + +case class ItemSomeViews( + @JsonView(Array(classOf[Views.Public])) id: Long = 1L, + @JsonView(Array(classOf[Views.Public])) name: String = "", + owner: String = "") + +case class ItemNoDefaultForView(@JsonView(Array(classOf[Views.Public])) name: String) + +@JsonNaming(classOf[PropertyNamingStrategy.KebabCaseStrategy]) +case class CaseClassWithKebabCase(pleaseUseKebabCase: Boolean) + +@JsonNaming(classOf[PropertyNamingStrategy.KebabCaseStrategy]) +trait KebabCaseMixin + +case class CaseClassShouldUseKebabCaseFromMixin(willThisGetTheRightCasing: Boolean) + +@JsonNaming +case class UseDefaultNamingStrategy(thisFieldShouldUseDefaultPropertyNamingStrategy: Boolean) diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/serde/DurationStringDeserializerTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/DurationStringDeserializerTest.scala new file mode 100644 index 0000000000..2b36c40ade --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/DurationStringDeserializerTest.scala @@ -0,0 +1,52 @@ +package com.twitter.util.jackson.serde + +import com.twitter.conversions.DurationOps._ +import com.twitter.util.Duration +import com.twitter.util.jackson.ScalaObjectMapper +import com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +case class CaseClassWithDuration(d: Duration) + +@RunWith(classOf[JUnitRunner]) +class DurationStringDeserializerTest extends AnyFunSuite with Matchers { + + private[this] val mapper = ScalaObjectMapper() + + test("deserializes values") { + Seq( + " 1.second" -> 1.second, + "+1.second" -> 1.second, + "-1.second" -> -1.second, + "1.SECOND" -> 1.second, + "1.day - 1.second" -> (1.day - 1.second), + "1.day" -> 1.day, + "1.microsecond" -> 1.microsecond, + "1.millisecond" -> 1.millisecond, + "1.second" -> 1.second, + "1.second+1.minute + 1.day" -> (1.second + 1.minute + 1.day), + "1.second+1.second" -> 2.seconds, + "2.hours" -> 2.hours, + "3.days" -> 3.days, + "321.nanoseconds" -> 321.nanoseconds, + "65.minutes" -> 65.minutes, + "876.milliseconds" -> 876.milliseconds, + "98.seconds" -> 98.seconds, + "Duration.Bottom" -> Duration.Bottom, + "Duration.Top" -> Duration.Top, + "Duration.Undefined" -> Duration.Undefined, + "duration.TOP" -> Duration.Top + ) foreach { + case (s, d) => + mapper.parse[CaseClassWithDuration](s"""{"d":"$s"}""") should equal( + CaseClassWithDuration(d)) + } + + intercept[CaseClassMappingException] { + mapper.parse[CaseClassWithDuration]("{}") + } + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/serde/LocalDateTimeSerdeTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/LocalDateTimeSerdeTest.scala new file mode 100644 index 0000000000..fea885cbdc --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/LocalDateTimeSerdeTest.scala @@ -0,0 +1,118 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.databind.{SerializationFeature, ObjectMapper => JacksonObjectMapper} +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.scala.{ + DefaultScalaModule, + ScalaObjectMapper => JacksonScalaObjectMapper +} +import com.twitter.util.jackson.ScalaObjectMapper +import java.time.format.DateTimeFormatter +import java.time.{LocalDateTime, ZoneId} +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +case class MyClass(dateTime: LocalDateTime) + +private object LocalDateTimeSerdeTest { + val UTCZoneId: ZoneId = ZoneId.of("UTC") + val ISO8601DateTimeFormatter: DateTimeFormatter = + DateTimeFormatter.ISO_OFFSET_DATE_TIME // "2014-05-30T03:57:59.302Z" + + // 1st one for the LocalDate, the 2nd one for LocalDateTime + // @JsonFormat(pattern = "yyyy-MM-dd") + // @JsonFormat(pattern = "yyyy-MM-dd HH:mm") +} + +@RunWith(classOf[JUnitRunner]) +class LocalDateTimeSerdeTest extends AnyFunSuite with Matchers { + import LocalDateTimeSerdeTest._ + + private final val NowUtc: LocalDateTime = LocalDateTime.now(UTCZoneId) + private[this] val myClass1 = MyClass(dateTime = NowUtc) + + // Most {@code java.time} types are serialized as numbers (integers or decimals as appropriate) if the + // {@link com.fasterxml.jackson.databind.SerializationFeature#WRITE_DATES_AS_TIMESTAMPS} feature is enabled, and otherwise are serialized in + // standard ISO-8601 string representation. ISO-8601 specifies formats + // for representing offset dates and times, zoned dates and times, local dates and times, periods, durations, zones, and more. All + // {@code java.time} types have built-in translation to and from ISO-8601 formats. + + // Granularity of timestamps is controlled through the companion features + // {@link com.fasterxml.jackson.databind.SerializationFeature#WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS} and + // {@link com.fasterxml.jackson.databind.DeserializationFeature#READ_DATE_TIMESTAMPS_AS_NANOSECONDS}. For serialization, timestamps are + // written as fractional numbers (decimals), where the number is seconds and the decimal is fractional seconds, if + // {@code WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS} is enabled (it is by default), with resolution as fine as nanoseconds depending on the + // underlying JDK implementation. If {@code WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS} is disabled, timestamps are written as a whole number of + // milliseconds. At deserialization time, decimal numbers are always read as fractional second timestamps with up-to-nanosecond resolution, + // since the meaning of the decimal is unambiguous. The more ambiguous integer types are read as fractional seconds without a decimal point + // if {@code READ_DATE_TIMESTAMPS_AS_NANOSECONDS} is enabled (it is by default), and otherwise they are read as milliseconds. + + // {@link LocalDate}, {@link LocalTime}, {@link LocalDateTime}, and {@link OffsetTime}, which cannot portably be converted to timestamps + // and are instead represented as arrays when WRITE_DATES_AS_TIMESTAMPS is enabled. + // https://github.com/FasterXML/jackson-datatype-jsr310/blob/master/src/main/java/com/fasterxml/jackson/datatype/jsr310/JavaTimeModule.java + test("serialize as array") { + val mapper = new JacksonObjectMapper() + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true) + mapper.registerModule(new JavaTimeModule) + val serialized = mapper.writeValueAsString(NowUtc) // [2021,5,10,0,0,0,0] + // remove beginning [ and ending ] from the string + val serializedParts = serialized.substring(1, serialized.length - 1).split(",") + serializedParts.size should equal(7) + serializedParts(0) should equal(NowUtc.getYear.toString) + serializedParts(1) should equal(NowUtc.getMonthValue.toString) + serializedParts(2) should equal(NowUtc.getDayOfMonth.toString) + serializedParts(3) should equal(NowUtc.getHour.toString) + serializedParts(4) should equal(NowUtc.getMinute.toString) + serializedParts(5) should equal(NowUtc.getSecond.toString) + serializedParts(6) should equal(NowUtc.getNano.toString) + } + + test("serialize as text") { + val mapper = new JacksonObjectMapper() + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) + mapper.registerModule(new JavaTimeModule) + val actual = mapper.writeValueAsString(NowUtc) + val expected = quote(NowUtc.toString) + // occasionally Jackson does not include the last 0 when writing the nano seconds + expected.startsWith(actual) should be(true) + } + + test("deserialize from text") { + val mapper = new JacksonObjectMapper with JacksonScalaObjectMapper + mapper.registerModule(new JavaTimeModule) + mapper.registerModule(DefaultSerdeModule) + mapper.readValue[LocalDateTime](quote(NowUtc.toString)) should equal(NowUtc) + } + + test("deserialize from text with ScalaObjectMapper with SerdeModule") { + val mapper = ScalaObjectMapper() + mapper.parse[LocalDateTime](quote(NowUtc.toString)) should equal(NowUtc) + } + + test("roundtrip text") { + val mapper = new JacksonObjectMapper with JacksonScalaObjectMapper + mapper.registerModule(DefaultScalaModule) + mapper.registerModule(new JavaTimeModule) + val json = mapper.writeValueAsString(myClass1) + val deserializedC1 = mapper.readValue[MyClass](json) + deserializedC1 should equal(myClass1) + } + + test("roundtrip text with ScalaObjectMapper with SerdeModule") { + val mapper = ScalaObjectMapper() + val json = mapper.writeValueAsString(myClass1) + val deserializedC1 = mapper.parse[MyClass](json) + deserializedC1 should equal(myClass1) + } + + test("roundtrip text with ScalaObjectMapper") { + val mapper = ScalaObjectMapper() + val json = mapper.writeValueAsString(myClass1) + val deserializedC1 = mapper.parse[MyClass](json) + deserializedC1 should equal(myClass1) + } + + private[this] def quote(str: String): String = s"""\"$str\"""" +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/serde/TimeStringDeserializerTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/TimeStringDeserializerTest.scala new file mode 100644 index 0000000000..bfd04f9329 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/TimeStringDeserializerTest.scala @@ -0,0 +1,113 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.annotation.JsonFormat +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.twitter.util.jackson.ScalaObjectMapper +import com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException +import com.twitter.util.{Time, TimeFormat} +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +private object TimeStringDeserializerTest { + case class WithoutJsonFormat(time: Time) + case class WithJsonFormat(@JsonFormat(pattern = "yyyy-MM-dd") time: Time) + case class WithJsonFormatAndTimezone( + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "America/Los_Angeles") time: Time) + case class WithEpochJsonFormat( + @JsonFormat(shape = JsonFormat.Shape.NUMBER, pattern = "s") time: Time) +} + +@RunWith(classOf[JUnitRunner]) +class TimeStringDeserializerTest extends AnyFunSuite with BeforeAndAfterAll with Matchers { + import TimeStringDeserializerTest._ + + private[this] final val Input1 = + """ + |{ + | "time": "2019-06-17T15:45:00.000+0000" + |} + """.stripMargin + + private[this] final val Input2 = + """ + |{ + | "time": "2019-06-17" + |} + """.stripMargin + + private[this] final val Input3 = + """ + |{ + | "time": "2019-06-17 16:30:00" + |} + """.stripMargin + + private[this] final val EpochInput = + """ + |{ + | "time": 1560786300 + |} + """.stripMargin + + private[this] val jacksonObjectMapper = new ObjectMapper() + private[this] val objectMapper = ScalaObjectMapper() + + override def beforeAll(): Unit = { + val modules = Seq(DefaultScalaModule, DefaultSerdeModule) + modules.foreach(jacksonObjectMapper.registerModule) + } + + test("should deserialize date without JsonFormat") { + val expected = 1560786300000L + val jacksonActual: WithoutJsonFormat = + jacksonObjectMapper.readValue(Input1, classOf[WithoutJsonFormat]) + jacksonActual.time.inMillis shouldEqual expected + + val frameworkActual = objectMapper.parse[WithoutJsonFormat](Input1) + frameworkActual.time.inMillis shouldEqual expected + } + + test("should deserialize date with JsonFormat") { + val expected: Time = new TimeFormat("yyyy-MM-dd").parse("2019-06-17") + val jacksonActual: WithJsonFormat = + jacksonObjectMapper.readValue(Input2, classOf[WithJsonFormat]) + jacksonActual.time shouldEqual expected + + val frameworkActual = objectMapper.parse[WithJsonFormat](Input2) + frameworkActual.time shouldEqual expected + } + + test("should deserialize date with JsonFormat and timezone") { + val expected = 1560814200000L + val jacksonActual: WithJsonFormatAndTimezone = + jacksonObjectMapper.readValue(Input3, classOf[WithJsonFormatAndTimezone]) + jacksonActual.time.inMillis shouldEqual expected + + val frameworkActual = objectMapper.parse[WithJsonFormatAndTimezone](Input3) + frameworkActual.time.inMillis shouldEqual expected + } + + test("should return a MappingException for empty value") { + val jacksonActual: WithoutJsonFormat = + jacksonObjectMapper.readValue("{}", classOf[WithoutJsonFormat]) + jacksonActual should equal(WithoutJsonFormat(null)) // underlying mapper allows value to be null + + intercept[CaseClassMappingException] { + objectMapper.parse[WithoutJsonFormat]("{}") + } + } + + test("should support JsonFormat with epoch timestamp") { + val expected = 1560786300000L + val jacksonActual: WithEpochJsonFormat = + jacksonObjectMapper.readValue(EpochInput, classOf[WithEpochJsonFormat]) + jacksonActual.time.inMillis shouldEqual expected + + val frameworkActual = objectMapper.parse[WithEpochJsonFormat](EpochInput) + frameworkActual.time.inMillis shouldEqual expected + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/serde/TimeStringSerializerTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/TimeStringSerializerTest.scala new file mode 100644 index 0000000000..2ae594cd14 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/TimeStringSerializerTest.scala @@ -0,0 +1,139 @@ +package com.twitter.util.jackson.serde + +import com.fasterxml.jackson.annotation.JsonFormat +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.twitter.util.jackson.ScalaObjectMapper +import com.twitter.util.{Time, TimeFormat} +import java.util.TimeZone +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +private object TimeStringSerializerTest { + case class WithoutJsonFormat(time: Time) + case class WithJsonFormat(@JsonFormat(pattern = "yyyy-MM-dd") time: Time) + // NOTE: JsonFormat.Shape is not respected, times are always written as Strings + case class WithInvalidJsonFormat( + @JsonFormat(shape = JsonFormat.Shape.NUMBER, pattern = "yyyy-MM-dd") time: Time) + case class WithJsonFormatAndTimezone( + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "America/Los_Angeles") time: Time) + // Note for serialization the pattern 's' represents the "second in minute": see: https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html + // s Second in minute Number 55 + case class WithEpochJsonFormat( + @JsonFormat(shape = JsonFormat.Shape.NUMBER, pattern = "s") time: Time) + // S Millisecond Number 978 + case class WithJsonFormatNumberShape( + @JsonFormat(shape = JsonFormat.Shape.NUMBER_INT, pattern = "S") time: Time) + case class WithJsonFormatNaturalShape( + @JsonFormat(shape = JsonFormat.Shape.NATURAL, pattern = "S") time: Time) + case class WithJsonFormatNoShape( + @JsonFormat(shape = JsonFormat.Shape.ANY, pattern = "S") time: Time) + case class JsonFormatNumberShape( + @JsonFormat(shape = JsonFormat.Shape.NUMBER, pattern = "s.SS") time: Time) + case class JsonFormatNumberShape2( + @JsonFormat(shape = JsonFormat.Shape.NUMBER, pattern = "yyyy.MM.dd") time: Time) + case class JsonFormatNumberShape3( + @JsonFormat(shape = JsonFormat.Shape.NUMBER, pattern = "dd") time: Time) + case class JsonFormatNumberShape4( + @JsonFormat(shape = JsonFormat.Shape.NUMBER, pattern = "EEE") time: Time) + case class JsonFormatNumberShape5( + @JsonFormat(shape = JsonFormat.Shape.NUMBER, pattern = "yyyy,MM,dd") time: Time) + + val DefaultTimeFormat: TimeFormat = new TimeFormat("s") + val DefaultTime: Time = DefaultTimeFormat.parse("1560786300") // "2019-06-17 15:45:00 +0000" +} + +@RunWith(classOf[JUnitRunner]) +class TimeStringSerializerTest extends AnyFunSuite with BeforeAndAfterAll with Matchers { + import TimeStringSerializerTest._ + + private[this] val jacksonObjectMapper = new ObjectMapper() + private[this] val objectMapper = ScalaObjectMapper() + + override def beforeAll(): Unit = { + val modules = Seq(DefaultScalaModule, DefaultSerdeModule) + modules.foreach(jacksonObjectMapper.registerModule) + } + + test("should serialize date without JsonFormat") { + val expected: String = + """{"time":"2019-06-17 15:45:00 +0000"}""" + + val value = WithoutJsonFormat(DefaultTime) + jacksonObjectMapper.writeValueAsString(value) shouldEqual expected + objectMapper.writeValueAsString(value) shouldEqual expected + } + + test("should serialize as String with invalid JsonFormat") { + val expected: String = + """{"time":"2019-06-17"}""" + + val value = WithInvalidJsonFormat(DefaultTime) + jacksonObjectMapper.writeValueAsString(value) shouldEqual expected + objectMapper.writeValueAsString(value) shouldEqual expected + } + + test("should serialize date with JsonFormat and timezone") { + val expected = """{"time":"2019-06-17 23:30:00"}""" + + val time = + new TimeFormat( + pattern = "s", + locale = None, + timezone = TimeZone.getTimeZone("America/Los_Angeles")).parse("1560814200") + val value = WithJsonFormatAndTimezone(time) + jacksonObjectMapper.writeValueAsString(value) shouldEqual expected + objectMapper.writeValueAsString(value) shouldEqual expected + } + + test("should serialize date with JsonFormat") { + val value0 = WithJsonFormat(DefaultTime) + jacksonObjectMapper.writeValueAsString(value0) shouldEqual """{"time":"2019-06-17"}""" + objectMapper.writeValueAsString(value0) shouldEqual """{"time":"2019-06-17"}""" + + // Note the @JsonFormat only ends up formatting the + // "second in minute", see: https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html + // which in this example is 0. + val value = WithEpochJsonFormat(DefaultTime) + jacksonObjectMapper.writeValueAsString(value) shouldEqual """{"time":"0"}""" + objectMapper.writeValueAsString(value) shouldEqual """{"time":"0"}""" + + // Note the @JsonFormat only ends up formatting the + // "millisecond", see: https://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html + // which in this example is 0. + val value1 = WithJsonFormatNumberShape(DefaultTime) + jacksonObjectMapper.writeValueAsString(value1) shouldEqual """{"time":"0"}""" + objectMapper.writeValueAsString(value1) shouldEqual """{"time":"0"}""" + + val value2 = WithJsonFormatNaturalShape(DefaultTime) + jacksonObjectMapper.writeValueAsString(value2) shouldEqual """{"time":"0"}""" + objectMapper.writeValueAsString(value2) shouldEqual """{"time":"0"}""" + + val value3 = WithJsonFormatNoShape(DefaultTime) + jacksonObjectMapper.writeValueAsString(value3) shouldEqual """{"time":"0"}""" + objectMapper.writeValueAsString(value3) shouldEqual """{"time":"0"}""" + + val value4 = JsonFormatNumberShape(DefaultTime) + jacksonObjectMapper.writeValueAsString(value4) shouldEqual """{"time":"0.00"}""" + objectMapper.writeValueAsString(value4) shouldEqual """{"time":"0.00"}""" + + val value5 = JsonFormatNumberShape2(DefaultTime) + jacksonObjectMapper.writeValueAsString(value5) shouldEqual """{"time":"2019.06.17"}""" + objectMapper.writeValueAsString(value5) shouldEqual """{"time":"2019.06.17"}""" + + val value6 = JsonFormatNumberShape3(DefaultTime) + jacksonObjectMapper.writeValueAsString(value6) shouldEqual """{"time":"17"}""" + objectMapper.writeValueAsString(value6) shouldEqual """{"time":"17"}""" + + val value7 = JsonFormatNumberShape4(DefaultTime) + jacksonObjectMapper.writeValueAsString(value7) shouldEqual """{"time":"Mon"}""" + objectMapper.writeValueAsString(value7) shouldEqual """{"time":"Mon"}""" + + val value8 = JsonFormatNumberShape5(DefaultTime) + jacksonObjectMapper.writeValueAsString(value8) shouldEqual """{"time":"2019,06,17"}""" + objectMapper.writeValueAsString(value8) shouldEqual """{"time":"2019,06,17"}""" + } +} From 7be3a9748a288e9fe3ec2f16d6716c1bf07975b4 Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Tue, 8 Jun 2021 18:48:12 +0000 Subject: [PATCH 494/796] *: Update Scala 2.13.1 to 2.13.6 Problem The latest Scala 2.13.x version is compatible with Scala 3.0.0 and includes many useful compiler-tools (e.g. warnings) to make the transition - and development in general - easier. Even if we don't have a real Scala 3 cross-build, 2.13.6 can be consumed by Scala 3 projects. Solution Updating Scala 2.13.1 to 2.13.6 Result The build uses Scala 2.13.6. I also used the 2.13.x version as default as it has many extremely useful warnings that 2.12.x doesn't have. The only part that needed fixing is some reflection code as 2.13.5 changed encoding of method-parameter names. Signed-off-by: Moses Nakamura Differential Revision: https://phabricator.twitter.biz/D680820 --- build.sbt | 4 ++-- .../scala/com/twitter/util/validation/ScalaValidator.scala | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index e0ae40bba0..012efd41cd 100644 --- a/build.sbt +++ b/build.sbt @@ -81,8 +81,8 @@ def jdk11GcJavaOptions: Seq[String] = { } val defaultProjectSettings = Seq( - scalaVersion := "2.12.12", - crossScalaVersions := Seq("2.12.12", "2.13.1") + scalaVersion := "2.13.6", + crossScalaVersions := Seq("2.12.12", "2.13.6") ) val baseSettings = Seq( diff --git a/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala b/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala index bca0626154..30e35e4aae 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala +++ b/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala @@ -864,7 +864,11 @@ class ScalaValidator private[validation] ( case Some(names) => names case _ => - getExecutableParameterNames(executableDescriptor.executable) + getExecutableParameterNames(executableDescriptor.executable).map { maybeMangled => + DescriptorFactory.unmangleName( + maybeMangled + ) // parameter names are encoded since Scala 2.13.5 + } } // executable parameter constraints From a2e2ee316a24523f81c57d84eb9235eb68828b76 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Wed, 9 Jun 2021 00:27:44 +0000 Subject: [PATCH 495/796] finagle|finatra|scrooge|ts|util: Use sbt version in GH actions cache key Problem The builds were failing because pre-sbt upgrade, the existing sbt launch jar was cached, which caused a conflict when trying to use the upgraded sbt launch jar (which had the same name as the old one). Solution Add the sbt version to the cache key so that when the sbt version is upgraded, the cache will be invalidated and everything will run smoothly. JIRA Issues: CSL-11029 Differential Revision: https://phabricator.twitter.biz/D683985 --- .github/workflows/continuous-integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 7a658004aa..db8a534c7d 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -3,6 +3,7 @@ name: continuous integration env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI + SBT_VERSION: 1.5.3 defaults: run: @@ -40,7 +41,7 @@ jobs: ~/.ivy2/cache ~/.ivy2/local/com.twitter ~/.sbt - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.scala }}-${{ matrix.java }} + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.SBT_VERSION }}-${{ matrix.scala }}-${{ matrix.java }} - name: update cache run: | if [ -f ~/.ivy2/cache ]; then From 6f9e1120cbc29936992bf061e2d69d11713edd9b Mon Sep 17 00:00:00 2001 From: Automated go/ktf Flagger Date: Wed, 9 Jun 2021 03:18:58 +0000 Subject: [PATCH 496/796] The test target "util/util-jackson/src/test/scala/com/twitter/util/jackson:jackson" is failing too often in master in CI. Summary: Problem: The test target "util/util-jackson/src/test/scala/com/twitter/util/jackson:jackson" is failing too often in master in CI. Logs available here: Known to fail ticket: [CSL-11031](https://jira.twitter.biz/browse/CSL-11031) Solution: It has been marked "known-to-fail" in the BUILD file Result: This will cause it to be skipped in future builds, and its failures ignored in submit queue. This implies that code changes that further break this test will not be prevented from being submitted to the master branch, until the problem is fixed and the test is no longer marked known-to-fail. TBR=true Reviewers: O52295 source:/util/util-jackson/! Differential Revision: https://phabricator.twitter.biz/D684676 --- util-jackson/src/test/scala/com/twitter/util/jackson/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD index fdeba22589..0f84ad0d71 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD @@ -5,6 +5,7 @@ junit_tests( "caseclass/*.scala", "serde/*.scala", ], + tags = ["known-to-fail-jira:CSL-11031"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", From e04b0a0ab901c28a6ef29cc67823e2a120a6ff7c Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 9 Jun 2021 16:51:28 +0000 Subject: [PATCH 497/796] util-jackson: Don't include fractions of a second in test Problem The LocalDateTimeSerdeTest#serialize as text test is highly sensitive as it tries to match serialized/deserialized LocalDateTime instances including fractions of a second. Solution Remove the fractions of a second from the comparison when testing to reduce the chance for non-determinism. JIRA Issues: CSL-11031 Differential Revision: https://phabricator.twitter.biz/D684685 --- .../src/test/scala/com/twitter/util/jackson/BUILD | 1 - .../jackson/serde/LocalDateTimeSerdeTest.scala | 15 +++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD index 0f84ad0d71..fdeba22589 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD @@ -5,7 +5,6 @@ junit_tests( "caseclass/*.scala", "serde/*.scala", ], - tags = ["known-to-fail-jira:CSL-11031"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/serde/LocalDateTimeSerdeTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/LocalDateTimeSerdeTest.scala index fea885cbdc..c972f504db 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/serde/LocalDateTimeSerdeTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/serde/LocalDateTimeSerdeTest.scala @@ -73,10 +73,17 @@ class LocalDateTimeSerdeTest extends AnyFunSuite with Matchers { val mapper = new JacksonObjectMapper() mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) mapper.registerModule(new JavaTimeModule) - val actual = mapper.writeValueAsString(NowUtc) - val expected = quote(NowUtc.toString) - // occasionally Jackson does not include the last 0 when writing the nano seconds - expected.startsWith(actual) should be(true) + val actualStr = mapper.writeValueAsString(NowUtc) + val actualStrUnquoted = + actualStr.substring(1, actualStr.length - 1) // drop beginning and end quotes + val actual = + actualStrUnquoted.substring( + 0, + actualStrUnquoted.lastIndexOf('.') + ) // // drop fractions of second + val expectStr = NowUtc.toString + val expected = expectStr.substring(0, expectStr.lastIndexOf('.')) // drop fractions of second + expected should be(actual) } test("deserialize from text") { From d22f657cad2db0b663f3155496bb3ce671256fdd Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 9 Jun 2021 19:24:36 +0000 Subject: [PATCH 498/796] util-validator: Unmangling more names in ScalaValidator Problem When merging in Felix's upgrade patch, I broke a unit test by trying to unmangle in a too-narrow way, and missed it through negligence. Solution Unmangle appropriately, the way the original commit worked! JIRA Issues: CSL-11030 Differential Revision: https://phabricator.twitter.biz/D684906 --- .../com/twitter/util/validation/ScalaValidator.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala b/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala index 30e35e4aae..c204ac2fc9 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala +++ b/util-validator/src/main/scala/com/twitter/util/validation/ScalaValidator.scala @@ -860,15 +860,15 @@ class ScalaValidator private[validation] ( val results = new mutable.ListBuffer[ConstraintViolation[T]]() val parameters: Array[Parameter] = executableDescriptor.executable.getParameters val parameterNamesList: Array[String] = - parameterNames match { + (parameterNames match { case Some(names) => names case _ => - getExecutableParameterNames(executableDescriptor.executable).map { maybeMangled => - DescriptorFactory.unmangleName( - maybeMangled - ) // parameter names are encoded since Scala 2.13.5 - } + getExecutableParameterNames(executableDescriptor.executable) + }).map { maybeMangled => + DescriptorFactory.unmangleName( + maybeMangled + ) // parameter names are encoded since Scala 2.13.5 } // executable parameter constraints From c9da7c43e902550ea23ec28b700d8cb57542a8ab Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 17 Jun 2021 03:55:13 +0000 Subject: [PATCH 499/796] util-core: Add c.t.io.ClasspathResource Problem We'd like a general and standard way to access classpath resources as an `InputStream`. Solution Add a simple utility to load resources from the classpath. Differential Revision: https://phabricator.twitter.biz/D687324 --- CHANGELOG.rst | 3 + .../com/twitter/io/ClasspathResource.scala | 46 +++++++ util-core/src/test/resources/BUILD | 3 + .../com/twitter/io/resource-test-file.txt | 23 ++++ util-core/src/test/resources/empty-file.txt | 0 .../src/test/resources/foo/bar/test-file.txt | 3 + util-core/src/test/resources/test.txt | 21 ++++ util-core/src/test/scala/com/twitter/io/BUILD | 1 + .../twitter/io/ClasspathResourceTest.scala | 116 ++++++++++++++++++ 9 files changed, 216 insertions(+) create mode 100644 util-core/src/main/scala/com/twitter/io/ClasspathResource.scala create mode 100644 util-core/src/test/resources/BUILD create mode 100644 util-core/src/test/resources/com/twitter/io/resource-test-file.txt create mode 100644 util-core/src/test/resources/empty-file.txt create mode 100644 util-core/src/test/resources/foo/bar/test-file.txt create mode 100644 util-core/src/test/resources/test.txt create mode 100644 util-core/src/test/scala/com/twitter/io/ClasspathResourceTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ec244827ec..ac6bea633f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,9 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-core: Add `ClasspathResource`, a utility for loading classpath resources as an + optional `InputStream`. ``PHAB_ID=D687324`` + * util-jackson: Introduce a new library for JSON serialization and deserialization based on the Jackson integration in `Finatra `__. diff --git a/util-core/src/main/scala/com/twitter/io/ClasspathResource.scala b/util-core/src/main/scala/com/twitter/io/ClasspathResource.scala new file mode 100644 index 0000000000..03f3d65f86 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/ClasspathResource.scala @@ -0,0 +1,46 @@ +package com.twitter.io + +import java.io.{BufferedInputStream, InputStream} + +object ClasspathResource { + + /** + * Loads the named resource from the classpath to return an optional [[InputStream]]. + * If the resolved resource exists and has a non-zero number of readable bytes an + * [[InputStream]] will be returned otherwise a None is returned. + * + * Resolution of the path to the named resource is ALWAYS assumed to be absolute, that is, + * if the given `name` does not begin with a `/` (`\u002f`), one will be prepended to the given name. + * E.g., `load("foo.txt")` will attempt to load the resource with the absolute name `/foo.txt`. + * + * ==Usage== + * {{{ + * load("foo.txt") // loads the classpath resource `/foo.txt` + * load("/foo.txt") // loads the classpath resource `/foo.txt` + * + * load("foo/bar/file.txt") // loads the classpath resource `/foo/bar/file.txt` + * load("/foo/bar/file.txt") // loads the classpath resource `/foo/bar/file.txt` + * }}} + * + * @note The caller is responsible for managing the closing of any returned [[InputStream]]. + * @see [[java.lang.Class#getResourceAsStream]] + */ + def load(name: String): Option[InputStream] = Option( + getClass.getResourceAsStream(absolutePath(name))) match { + case Some(inputStream) => + val bufferedInputStream: BufferedInputStream = new BufferedInputStream(inputStream) + if (bufferedInputStream.available() > 0) { + Some(bufferedInputStream) + } else { + // close the opened resource since we will not be returning the input stream + bufferedInputStream.close() + None + } + case _ => None + } + + // ensure the given name is prepended with a "/" to load as an absolute path. + private[this] def absolutePath(name: String): String = { + if (name.startsWith("/")) name else "/" + name + } +} diff --git a/util-core/src/test/resources/BUILD b/util-core/src/test/resources/BUILD new file mode 100644 index 0000000000..a001371a96 --- /dev/null +++ b/util-core/src/test/resources/BUILD @@ -0,0 +1,3 @@ +resources( + sources = ["**/*"], +) diff --git a/util-core/src/test/resources/com/twitter/io/resource-test-file.txt b/util-core/src/test/resources/com/twitter/io/resource-test-file.txt new file mode 100644 index 0000000000..6be28d302a --- /dev/null +++ b/util-core/src/test/resources/com/twitter/io/resource-test-file.txt @@ -0,0 +1,23 @@ +Nam bibendum cursus risus, at volutpat elit tincidunt eu. Aliquam id ligula in mi tempor ultricies. +Phasellus quis consequat metus, at congue ipsum. Nam eu interdum elit, scelerisque laoreet neque. +Proin efficitur nisl lectus, in condimentum sem fringilla eget. Morbi condimentum non ipsum in +auctor. Mauris at turpis faucibus, sodales massa quis, commodo nisl. + +Aliquam hendrerit, ante eget accumsan lobortis, nibh nunc varius velit, ac vulputate sem odio id +erat. Pellentesque a imperdiet ipsum. Sed diam augue, eleifend ut congue at, accumsan sed libero. +Nulla ipsum dui, rhoncus a purus sit amet, sagittis convallis sapien. Curabitur auctor volutpat +laoreet. Donec at elit est. Aliquam ultrices urna fringilla sagittis iaculis. Sed sit amet +accumsan eros. Nullam tristique nibh sed molestie varius. Vestibulum quis dictum orci, et +imperdiet nunc. Maecenas aliquet, justo ut finibus malesuada, erat nunc pellentesque lacus, sit +amet iaculis libero dolor in libero. Interdum et malesuada fames ac ante ipsum primis in faucibus. +Quisque metus libero, pulvinar in sagittis eget, commodo eu justo. Morbi auctor tincidunt nibh vel +dictum. + +Vivamus ut elementum lectus. In vehicula eget lectus eget ultrices. Ut vitae est mattis, imperdiet +nulla non, molestie mi. Duis a ultrices nisl, eu consectetur neque. Integer id quam id massa +mattis rhoncus vitae eget ligula. Praesent dapibus, elit sit amet sollicitudin gravida, velit nisl +pellentesque ante, id tempor ante ligula quis est. Donec quis diam nec magna porttitor rhoncus. +Pellentesque pellentesque ullamcorper felis, malesuada fringilla odio accumsan eget. Praesent +ultrices turpis in libero convallis, a porta lectus venenatis. Curabitur aliquet, lorem eu cursus +scelerisque, lorem mi luctus mauris, in rhoncus leo neque sed odio. Nunc efficitur posuere dolor, +in aliquet orci congue vel. diff --git a/util-core/src/test/resources/empty-file.txt b/util-core/src/test/resources/empty-file.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/util-core/src/test/resources/foo/bar/test-file.txt b/util-core/src/test/resources/foo/bar/test-file.txt new file mode 100644 index 0000000000..dddb718a87 --- /dev/null +++ b/util-core/src/test/resources/foo/bar/test-file.txt @@ -0,0 +1,3 @@ +Proin mattis nisi ac ipsum bibendum, eget lobortis lacus lobortis. Vivamus interdum ac tortor et accumsan. Aliquam scelerisque diam nibh, non efficitur dui eleifend luctus. Ut euismod risus a nunc tempus aliquet. Nulla facilisi. Vestibulum tempor justo id nulla rhoncus mattis. Praesent vitae congue diam. + +Sed euismod dictum fermentum. Nunc est dolor, auctor id vulputate id, commodo aliquam orci. Nam tincidunt vehicula sapien vitae dapibus. Proin nec metus magna. Suspendisse eget semper ex. Donec non mi lacus. Aliquam magna augue, eleifend quis scelerisque varius, congue et sapien. Praesent lacinia lorem quis urna efficitur, ut semper arcu blandit. diff --git a/util-core/src/test/resources/test.txt b/util-core/src/test/resources/test.txt new file mode 100644 index 0000000000..600d6e0166 --- /dev/null +++ b/util-core/src/test/resources/test.txt @@ -0,0 +1,21 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce et orci et felis aliquet luctus. In +a turpis tempus dolor condimentum pulvinar. Morbi pharetra suscipit velit. Phasellus et nisi +felis. Sed porta sapien vitae sem placerat, in vehicula massa cursus. Ut cursus ligula in justo +tincidunt pretium quis sit amet lorem. Phasellus feugiat, ante sed mattis sagittis, elit sapien +rhoncus quam, at varius enim eros faucibus metus. Suspendisse purus leo, convallis id luctus ut, +convallis nec neque. Suspendisse ut iaculis nisi. Vivamus cursus viverra faucibus. Vestibulum in +ligula at massa feugiat malesuada eget et urna. Sed nulla elit, rutrum et purus vel, aliquet +elementum risus. Suspendisse potenti. + +Vestibulum pellentesque, risus eget feugiat porttitor, mauris ligula bibendum risus, nec varius nisi +leo nec neque. Phasellus et sollicitudin elit, vitae ornare ante. Ut tempor risus rhoncus +tincidunt commodo. Phasellus efficitur nisl id lorem efficitur, vitae pharetra tortor fringilla. +Suspendisse tristique eros vel leo consectetur dictum. Praesent efficitur dolor ultricies, aliquam +odio et, consequat lorem. Proin dictum nunc sit amet nunc venenatis, eget venenatis nulla commodo. +Etiam ut mi sed sem interdum tincidunt. Phasellus magna tellus, suscipit sed commodo eu, +sollicitudin eget odio. Nulla massa elit, dignissim a quam ut, auctor dapibus augue. Etiam +interdum neque eu lectus ultricies, sed lacinia purus scelerisque. Proin nec consequat libero. Sed +eros nibh, cursus nec efficitur quis, pharetra ut nisl. Ut tincidunt, ex vel semper tempus, odio +leo pharetra eros, id interdum diam enim ut tortor. + +Sed ultricies luctus tempus. diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index f66980dee1..da3a621a43 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -14,5 +14,6 @@ junit_tests( "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/main/scala/com/twitter/io", + "util/util-core/src/test/resources", ], ) diff --git a/util-core/src/test/scala/com/twitter/io/ClasspathResourceTest.scala b/util-core/src/test/scala/com/twitter/io/ClasspathResourceTest.scala new file mode 100644 index 0000000000..4099eead82 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/ClasspathResourceTest.scala @@ -0,0 +1,116 @@ +package com.twitter.io + +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +class ClasspathResourceTest extends AnyFunSuite with Matchers { + + test("ClasspathResource#load absolute path") { + ClasspathResource.load("/test.txt") match { + case Some(inputStream) => + try { + inputStream.available() > 0 should be(true) + } finally { + inputStream.close() + } + case _ => fail() + } + } + + test("ClasspathResource#load path interpreted as absolute") { + ClasspathResource.load("test.txt") match { + case Some(inputStream) => + try { + inputStream.available() > 0 should be(true) + } finally { + inputStream.close() + } + case _ => fail() + } + } + + test("ClasspathResource#load absolute path multiple directories 1") { + // loads from /com/twitter/io/resource-test-file.txt + ClasspathResource.load("/com/twitter/io/resource-test-file.txt") match { + case Some(inputStream) => + try { + inputStream.available() > 0 should be(true) + } finally { + inputStream.close() + } + case _ => fail() + } + } + + test("ClasspathResource#load absolute path multiple directories 2") { + // loads from /foo/bar/test-file.txt + ClasspathResource.load("/foo/bar/test-file.txt") match { + case Some(inputStream) => + try { + inputStream.available() > 0 should be(true) + } finally { + inputStream.close() + } + case _ => fail() + } + } + + test("ClasspathResource#load path interpreted as absolute multiple directories 1") { + // loads from /com/twitter/io/resource-test-file.txt + ClasspathResource.load("com/twitter/io/resource-test-file.txt") match { + case Some(inputStream) => + try { + inputStream.available() > 0 should be(true) + } finally { + inputStream.close() + } + case _ => fail() + } + } + + test("ClasspathResource#load path interpreted as absolute multiple directories 2") { + // loads from /foo/bar/test-file.txt + ClasspathResource.load("foo/bar/test-file.txt") match { + case Some(inputStream) => + try { + inputStream.available() > 0 should be(true) + } finally { + inputStream.close() + } + case _ => fail() + } + } + + test("ClasspathResource#load does not exist 1") { + ClasspathResource.load("/does-not-exist.txt") match { + case Some(_) => fail() + case _ => + // should not return an inputstream + } + } + + test("ClasspathResource#load does not exist 2") { + ClasspathResource.load("does-not-exist.txt") match { + case Some(_) => fail() + case _ => + // should not return an inputstream + } + } + + test("ClasspathResource#load empty file 1") { + ClasspathResource.load("/empty-file.txt") match { + case Some(_) => fail() + case _ => + // should not return an inputstream + } + } + + test("ClasspathResource#load empty file 2") { + // loads from /empty-file.txt + ClasspathResource.load("empty-file.txt") match { + case Some(_) => fail() + case _ => + // should not return an inputstream + } + } +} From 8e964d9e14068d26de738220cbc4254be38f461e Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 17 Jun 2021 20:02:20 +0000 Subject: [PATCH 500/796] util-jackson: Add c.t.util.jackson.YAML Problem We'd like a YAML corollary to `c.t.util.jackson.JSON`. Solution Introduce `c.t.util.jackson.YAML` and add methods for parity to `c.t.util.jackson.JSON`. Depends on D687324 Differential Revision: https://phabricator.twitter.biz/D687327 --- CHANGELOG.rst | 4 + doc/src/sphinx/util-jackson/index.rst | 71 +++++++- .../main/scala/com/twitter/util/jackson/BUILD | 1 + .../scala/com/twitter/util/jackson/JSON.scala | 41 +++++ .../scala/com/twitter/util/jackson/YAML.scala | 65 +++++++ util-jackson/src/test/resources/BUILD | 3 + util-jackson/src/test/resources/test.json | 3 + util-jackson/src/test/resources/test.yml | 2 + .../test/scala/com/twitter/util/jackson/BUILD | 1 + .../twitter/util/jackson/FileResources.scala | 29 ++++ .../com/twitter/util/jackson/JSONTest.scala | 55 +++++- .../com/twitter/util/jackson/YAMLTest.scala | 159 ++++++++++++++++++ 12 files changed, 420 insertions(+), 14 deletions(-) create mode 100644 util-jackson/src/main/scala/com/twitter/util/jackson/YAML.scala create mode 100644 util-jackson/src/test/resources/BUILD create mode 100644 util-jackson/src/test/resources/test.json create mode 100644 util-jackson/src/test/resources/test.yml create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/FileResources.scala create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/YAMLTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ac6bea633f..0e5794ae93 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,10 @@ New Features * util-core: Add `ClasspathResource`, a utility for loading classpath resources as an optional `InputStream`. ``PHAB_ID=D687324`` +* util-jackson: Add `com.twitter.util.jackson.YAML` for YAML serde operations with a + default configured `ScalaObjectMapper.` Add more methods to `com.twitter.util.jackson.JSON` + ``PHAB_ID=D687327`` + * util-jackson: Introduce a new library for JSON serialization and deserialization based on the Jackson integration in `Finatra `__. diff --git a/doc/src/sphinx/util-jackson/index.rst b/doc/src/sphinx/util-jackson/index.rst index 448668c89b..8d70f01f83 100644 --- a/doc/src/sphinx/util-jackson/index.rst +++ b/doc/src/sphinx/util-jackson/index.rst @@ -798,15 +798,17 @@ than the `scala.util.parsing.json.JSON` utility. .. important:: - The `c.t.util.jackson.JSON` API does not return an exception when parsing but rather returns an - `Option[T]` result. When parsing is successful, this is a `Some(T)`, otherwise it is a `None`. But - note that the specifics of any failure are lost. + The `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__ API does not return an exception when + parsing but rather returns an `Option[T]` result. When parsing is successful, this is a `Some(T)`, + otherwise it is a `None`. But note that the specifics of any failure are lost. It is thus also important to note that for this reason that the `c.t.util.jackson.JSON` uses a - default configured `ScalaObjectMapper <#defaults>`__ **with validation specifically disabled**, + `default configured ScalaObjectMapper <#defaults>`__ **with validation specifically disabled**, such that no `Bean Validation 2.0 <../util-validator>`__ style validations are performed when - parsing with `c.t.util.jackson.JSON`. Users should prefer using a configured `ScalaObjectMapper` - to perform validations in order to be able to properly handle validation exceptions. + parsing with `c.t.util.jackson.JSON`. + + Users should prefer using a configured |ScalaObjectMapper|_ to perform validations in order to + be able to properly handle validation exceptions. .. code:: scala :emphasize-lines: 7, 13, 22, 25 @@ -841,6 +843,63 @@ than the `scala.util.parsing.json.JSON` utility. "id" : "99999999" } + scala> + + +`c.t.util.jackson.YAML` +----------------------- + +Similarly, there is also a utility for `YAML `__ serde operations with many of +the same methods as `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__ using a default |ScalaObjectMapper|_ +configured with a `YAMLFactory`. + +.. code:: scala + :emphasize-lines: 8, 9, 10, 11, 12, 18, 19, 20, 29 + + Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). + Type in expressions for evaluation. Or try :help. + + scala> import com.twitter.util.jackson.YAML + import com.twitter.util.jackson.YAML + + scala> val result = + | YAML.parse[Map[String, Int]](""" + | |--- + | |a: 1 + | |b: 2 + | |c: 3""".stripMargin) + result: Option[Map[String,Int]] = Some(Map(a -> 1, b -> 2, c -> 3)) + + scala> case class FooClass(id: String) + defined class FooClass + + scala> val result = YAML.parse[FooClass](""" + | |--- + | |id: abcde1234""".stripMargin) + result: Option[FooClass] = Some(FooClass(abcde1234)) + + scala> result.get + res0: FooClass = FooClass(abcde1234) + + scala> val f = FooClass("99999999") + f: FooClass = FooClass(99999999) + + scala> YAML.write(f) + res1: String = + "--- + id: "99999999" + " + + scala> + +.. important:: + + Like with `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__, the `c.t.util.jackson.YAML <#c-t-util-jackson-yaml>`__ + API does not return an exception when parsing but rather returns an `Option[T]` result. + + Users should prefer using a configured |ScalaObjectMapper|_ to perform validations in order to be + able to properly handle validation exceptions. + `c.t.util.jackson.JsonDiff` --------------------------- diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD index f352f749f0..61ab5add42 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD @@ -30,6 +30,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson-dataformat-yaml", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "3rdparty/jvm/jakarta/validation:jakarta.validation-api", + "util/util-core/src/main/scala/com/twitter/io", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], ) diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala index 8afa7f8036..2008cbf22b 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/JSON.scala @@ -1,6 +1,8 @@ package com.twitter.util.jackson +import com.twitter.io.{Buf, ClasspathResource} import com.twitter.util.Try +import java.io.{File, InputStream} /** * Uses an instance of a [[ScalaObjectMapper]] configured to not perform any type of validation. @@ -18,6 +20,25 @@ object JSON { def parse[T: Manifest](input: String): Option[T] = Try(Mapper.parse[T](input)).toOption + /** Simple utility to parse a JSON [[Buf]] into an Option[T] type. */ + def parse[T: Manifest](input: Buf): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** + * Simple utility to parse a JSON [[InputStream]] into an Option[T] type. + * @note the caller is responsible for managing the lifecycle of the given [[InputStream]]. + */ + def parse[T: Manifest](input: InputStream): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** + * Simple utility to load a JSON file and parse contents into an Option[T] type. + * + * @note the caller is responsible for managing the lifecycle of the given [[File]]. + */ + def parse[T: Manifest](f: File): Option[T] = + Try(Mapper.underlying.readValue[T](f)).toOption + /** Simple utility to write a value as a JSON encoded String. */ def write(any: Any): String = Mapper.writeValueAsString(any) @@ -25,4 +46,24 @@ object JSON { /** Simple utility to pretty print a JSON encoded String from the given instance. */ def prettyPrint(any: Any): String = Mapper.writePrettyString(any) + + object Resource { + + /** + * Simple utility to load a JSON resource from the classpath and parse contents into an Option[T] type. + * + * @note `name` resolution to locate the resource is governed by [[java.lang.Class#getResourceAsStream]] + */ + def parse[T: Manifest](name: String): Option[T] = { + ClasspathResource.load(name) match { + case Some(inputStream) => + try { + JSON.parse[T](inputStream) + } finally { + inputStream.close() + } + case _ => None + } + } + } } diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/YAML.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/YAML.scala new file mode 100644 index 0000000000..a15a4aae46 --- /dev/null +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/YAML.scala @@ -0,0 +1,65 @@ +package com.twitter.util.jackson + +import com.twitter.io.{Buf, ClasspathResource} +import com.twitter.util.Try +import java.io.{File, InputStream} + +/** + * Uses an instance of a [[ScalaObjectMapper]] configured to not perform any type of validation. + * Inspired by [[https://www.scala-lang.org/api/2.12.6/scala-parser-combinators/scala/util/parsing/json/JSON$.html]]. + * + * @note This is only intended for use from Scala (not Java). + * + * @see [[ScalaObjectMapper]] + */ +object YAML { + private final val Mapper: ScalaObjectMapper = + ScalaObjectMapper.builder.withNoValidation.yamlObjectMapper + + /** Simple utility to parse a YAML string into an Option[T] type. */ + def parse[T: Manifest](input: String): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** Simple utility to parse a YAML [[Buf]] into an Option[T] type. */ + def parse[T: Manifest](input: Buf): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** + * Simple utility to parse a YAML [[InputStream]] into an Option[T] type. + * @note the caller is responsible for managing the lifecycle of the given [[InputStream]]. + */ + def parse[T: Manifest](input: InputStream): Option[T] = + Try(Mapper.parse[T](input)).toOption + + /** + * Simple utility to load a YAML file and parse contents into an Option[T] type. + * + * @note the caller is responsible for managing the lifecycle of the given [[File]]. + */ + def parse[T: Manifest](f: File): Option[T] = + Try(Mapper.underlying.readValue[T](f)).toOption + + /** Simple utility to write a value as a YAML encoded String. */ + def write(any: Any): String = + Mapper.writeValueAsString(any) + + object Resource { + + /** + * Simple utility to load a YAML resource from the classpath and parse contents into an Option[T] type. + * + * @note `name` resolution to locate the resource is governed by [[java.lang.Class#getResourceAsStream]] + */ + def parse[T: Manifest](name: String): Option[T] = { + ClasspathResource.load(name) match { + case Some(inputStream) => + try { + YAML.parse[T](inputStream) + } finally { + inputStream.close() + } + case _ => None + } + } + } +} diff --git a/util-jackson/src/test/resources/BUILD b/util-jackson/src/test/resources/BUILD new file mode 100644 index 0000000000..a001371a96 --- /dev/null +++ b/util-jackson/src/test/resources/BUILD @@ -0,0 +1,3 @@ +resources( + sources = ["**/*"], +) diff --git a/util-jackson/src/test/resources/test.json b/util-jackson/src/test/resources/test.json new file mode 100644 index 0000000000..98e5dbd51e --- /dev/null +++ b/util-jackson/src/test/resources/test.json @@ -0,0 +1,3 @@ +{ + "id": "55555555" +} diff --git a/util-jackson/src/test/resources/test.yml b/util-jackson/src/test/resources/test.yml new file mode 100644 index 0000000000..ac1f85b4b0 --- /dev/null +++ b/util-jackson/src/test/resources/test.yml @@ -0,0 +1,2 @@ +--- +id: 55555555 diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD index fdeba22589..b630aac594 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD @@ -33,6 +33,7 @@ junit_tests( "util/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions", "util/util-jackson/src/main/scala/com/twitter/util/jackson/serde", "util/util-jackson/src/test/java/com/twitter/util/jackson", + "util/util-jackson/src/test/resources", "util/util-mock/src/main/scala/com/twitter/util/mock", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", "util/util-validator/src/main/java/com/twitter/util/validation", diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/FileResources.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/FileResources.scala new file mode 100644 index 0000000000..d68f8fc6b5 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/FileResources.scala @@ -0,0 +1,29 @@ +package com.twitter.util.jackson + +import com.twitter.io.TempFolder +import java.io.{FileOutputStream, OutputStream, File => JFile} +import java.nio.charset.StandardCharsets +import java.nio.file.{FileSystems, Files} + +trait FileResources extends TempFolder { + + protected def writeStringToFile( + directory: String, + name: String, + ext: String, + data: String + ): JFile = { + val file = Files.createTempFile(FileSystems.getDefault.getPath(directory), name, ext).toFile + try { + val out: OutputStream = new FileOutputStream(file, false) + out.write(data.getBytes(StandardCharsets.UTF_8)) + file + } finally { + file.deleteOnExit() + } + } + + protected def addSlash(directory: String): String = { + if (directory.endsWith("/")) directory else s"$directory/" + } +} diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala index 9c0e19953a..c7af26b123 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/JSONTest.scala @@ -1,29 +1,26 @@ package com.twitter.util.jackson +import com.twitter.io.Buf +import java.io.{ByteArrayInputStream, File => JFile} import org.junit.runner.RunWith import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatestplus.junit.JUnitRunner @RunWith(classOf[JUnitRunner]) -class JSONTest extends AnyFunSuite with Matchers { +class JSONTest extends AnyFunSuite with Matchers with FileResources { test("JSON#parse 1") { JSON.parse[Map[String, Int]]("""{"a": 1, "b": 2}""") match { case Some(map) => - map("a") should equal(1) - map("b") should equal(2) + map should be(Map("a" -> 1, "b" -> 2)) case _ => fail() } } test("JSON#parse 2") { JSON.parse[Seq[String]]("""["a", "b", "c"]""") match { - case Some(seq) => - seq.size should equal(3) - seq.head should be("a") - seq(1) should be("b") - seq.last should be("c") + case Some(Seq("a", "b", "c")) => // pass case _ => fail() } } @@ -46,6 +43,40 @@ class JSONTest extends AnyFunSuite with Matchers { } } + test("JSON#parse 5") { + val buf = Buf.Utf8("""{"id": "abcd1234"}""") + JSON.parse[FooClass](buf) match { + case Some(foo) => + foo.id should equal("abcd1234") + case _ => fail() + } + } + + test("JSON#parse 6") { + val inputStream = new ByteArrayInputStream("""{"id": "abcd1234"}""".getBytes("UTF-8")) + try { + JSON.parse[FooClass](inputStream) match { + case Some(foo) => + foo.id should equal("abcd1234") + case _ => fail() + } + } finally { + inputStream.close() + } + } + + test("JSON#parse 7") { + withTempFolder { + val file: JFile = + writeStringToFile(folderName, "test-file", ".json", """{"id": "999999999"}""") + JSON.parse[FooClass](file) match { + case Some(foo) => + foo.id should equal("999999999") + case _ => fail() + } + } + } + test("JSON#write 1") { JSON.write(Map("a" -> 1, "b" -> 2)) should equal("""{"a":1,"b":2}""") } @@ -78,4 +109,12 @@ class JSONTest extends AnyFunSuite with Matchers { | "id" : "abcd1234" |}""".stripMargin) } + + test("JSON.Resource#parse resource") { + JSON.Resource.parse[FooClass]("/test.json") match { + case Some(foo) => + foo.id should equal("55555555") + case _ => fail() + } + } } diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/YAMLTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/YAMLTest.scala new file mode 100644 index 0000000000..79d9b16f25 --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/YAMLTest.scala @@ -0,0 +1,159 @@ +package com.twitter.util.jackson + +import com.twitter.io.Buf +import java.io.{ByteArrayInputStream, File => JFile} +import org.junit.runner.RunWith +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class YAMLTest extends AnyFunSuite with Matchers with FileResources { + + test("YAML#parse 1") { + YAML.parse[Map[String, Int]](""" + |--- + |a: 1 + |b: 2 + |c: 3""".stripMargin) match { + case Some(map) => + map should be(Map("a" -> 1, "b" -> 2, "c" -> 3)) + case _ => fail() + } + } + + test("YAML#parse 2") { + // without quotes + YAML.parse[Seq[String]](""" + |--- + |- a + |- b + |- c""".stripMargin) match { + case Some(Seq("a", "b", "c")) => // pass + case _ => fail() + } + // with quotes + YAML.parse[Seq[String]](""" + |--- + |- "a" + |- "b" + |- "c"""".stripMargin) match { + case Some(Seq("a", "b", "c")) => // pass + case _ => fail() + } + } + + test("YAML#parse 3") { + // without quotes + YAML.parse[FooClass](""" + |--- + |id: abcde1234""".stripMargin) match { + case Some(foo) => + foo.id should equal("abcde1234") + case _ => fail() + } + // with quotes + YAML.parse[FooClass](""" + |--- + |id: "abcde1234"""".stripMargin) match { + case Some(foo) => + foo.id should equal("abcde1234") + case _ => fail() + } + } + + test("YAML#parse 4") { + YAML.parse[CaseClassWithNotEmptyValidation](""" + |--- + |name: '' + |make: vw""".stripMargin) match { + case Some(clazz) => + // performs no validations + clazz.name.isEmpty should be(true) + clazz.make should equal(CarMakeEnum.vw) + case _ => fail() + } + } + + test("YAML#parse 5") { + val buf = Buf.Utf8("""{"id": "abcd1234"}""") + YAML.parse[FooClass](buf) match { + case Some(foo) => + foo.id should equal("abcd1234") + case _ => fail() + } + } + + test("YAML#parse 6") { + val inputStream = new ByteArrayInputStream("""{"id": "abcd1234"}""".getBytes("UTF-8")) + try { + YAML.parse[FooClass](inputStream) match { + case Some(foo) => + foo.id should equal("abcd1234") + case _ => fail() + } + } finally { + inputStream.close() + } + } + + test("YAML#parse 7") { + withTempFolder { + // without quotes + val file1: JFile = + writeStringToFile( + folderName, + "test-file-quotes", + ".yaml", + """|--- + |id: 999999999""".stripMargin) + YAML.parse[FooClass](file1) match { + case Some(foo) => + foo.id should equal("999999999") + case _ => fail() + } + // with quotes + val file2: JFile = + writeStringToFile( + folderName, + "test-file-noquotes", + ".yaml", + """|--- + |id: "999999999"""".stripMargin) + YAML.parse[FooClass](file2) match { + case Some(foo) => + foo.id should equal("999999999") + case _ => fail() + } + } + } + + test("YAML#write 1") { + YAML.write(Map("a" -> 1, "b" -> 2)) should equal("""|--- + |a: 1 + |b: 2 + |""".stripMargin) + } + + test("YAML#write 2") { + YAML.write(Seq("a", "b", "c")) should equal("""|--- + |- "a" + |- "b" + |- "c" + |""".stripMargin) + } + + test("YAML#write 3") { + YAML.write(FooClass("abcd1234")) should equal("""|--- + |id: "abcd1234" + |""".stripMargin) + } + + test("YAML.Resource#parse resource") { + YAML.Resource.parse[FooClass]("/test.yml") match { + case Some(foo) => + foo.id should equal("55555555") + case _ => fail() + } + } +} From 465601a701d3f604f73e3efcb194bc448efcf492 Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Mon, 21 Jun 2021 17:56:41 +0000 Subject: [PATCH 501/796] Match Scala version for 2.13.x in CI Problem / Solution We recently upgraded scala to 2.13.6 but neglected to update the version for Github Actions. This PR fixes that. Signed-off-by: Ruben Oanta Differential Revision: https://phabricator.twitter.biz/D691500 --- .github/workflows/continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index db8a534c7d..cff19eeb5c 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - scala: [2.12.12, 2.13.1] + scala: [2.12.12, 2.13.6] java: ['1.8', '1.11'] runs-on: ubuntu-latest steps: From 4b8e79fda435f1ebe4e20d0ac08456a478da9a21 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 22 Jun 2021 15:25:13 +0000 Subject: [PATCH 502/796] Twitter-oss: Prepare OSS libraries for release 21.6.0 Problem We want to release the next versions of our Twitter OSS libraries 21.6.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D692183 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0e5794ae93..4bede7a3e5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.6.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index ed4aeede24..ecc20578fc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.5.0" +val utilCore = "com.twitter" %% "util-core" % "21.6.0" ``` # Units diff --git a/build.sbt b/build.sbt index 012efd41cd..5660aec399 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.6.0-SNAPSHOT" +val releaseVersion = "21.6.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From cd7ff106d537fa6deb393895b202b22e02a45687 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 22 Jun 2021 15:51:56 +0000 Subject: [PATCH 503/796] Twitter-oss: update OSS libraries post-release to 21.7.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.7.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D692206 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ecc20578fc..8f58fb8aef 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 5660aec399..747a09f713 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.6.0" +val releaseVersion = "21.7.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 8d133d4e3002989e1d0120bdfd325172000da0ce Mon Sep 17 00:00:00 2001 From: Lily He Date: Tue, 22 Jun 2021 21:58:20 +0000 Subject: [PATCH 504/796] twitter-oss: Move templates to PULL_REQUEST_TEMPLATE folder Problem: GitHub needs the special keyword "closes" to automatically close PRs when a commit is merged. Solution: Add "closes twitter/$REPOSITORY#PR_NUMBER" to the pull request template so PRs from open source automatically close when the commit is merged in. This avoids manually closing the PR when possible. Also, move all the pull request templates to a PULL_REQUEST_TEMPLATE folder because this is the recommended standard from GitHub Differential Revision: https://phabricator.twitter.biz/D692645 --- .../pull_request_template.md} | 2 ++ 1 file changed, 2 insertions(+) rename .github/{PULL_REQUEST_TEMPLATE.md => PULL_REQUEST_TEMPLATE/pull_request_template.md} (93%) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md similarity index 93% rename from .github/PULL_REQUEST_TEMPLATE.md rename to .github/PULL_REQUEST_TEMPLATE/pull_request_template.md index bfb079c94c..6454011e26 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -13,3 +13,5 @@ Result What will change as a result of your pull request? Note that sometimes this section is unnecessary because it is self-explanatory based on the solution. + +Closes twitter/util#PR_NUMBER From efaab449e020567e70fce8c8e0807cf1da6c1169 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Wed, 23 Jun 2021 02:53:22 +0000 Subject: [PATCH 505/796] util-cache|finagle: upgrade caffeine cache to version 2.9.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem/Solution We want to keep our dependencies up to date for the ben-manes Caffeine library. Let’s update to version 2.9.1. JIRA Issues: CSL-10751 Differential Revision: https://phabricator.twitter.biz/D660908 --- CHANGELOG.rst | 7 ++++++- build.sbt | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4bede7a3e5..b2f28e3f52 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -34,6 +34,11 @@ Breaking API Changes StatReceiver derived classes use MetricBuilder directly to create counters, gauges and stats. ``PHAB_ID=D668739`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-cache: Update Caffeine cache library to version 2.9.1 ``PHAB_ID=D660908`` + 21.5.0 ------ @@ -45,7 +50,7 @@ New Features the Hibernate Validator library and thus supports `jakarta.validation.Constraint` annotations and validators for annotating and validating fields of Scala case classes. ``PHAB_ID=D638603`` -* util-app: Introduce a Java-friendly API `c.t.app.App#runOnExit(Runnable)` and +* util-app: Introduce a Java-friendly API `c.t.app.App#runOnExit(Runnable)` and `c.t.app.App#runOnExitLast(Runnable)` to allow Java 8 users to call `c.t.app.App#runOnExit` and `c.t.app.App#runOnExitLast` with lambda expressions. ``PHAB_ID=D511536`` diff --git a/build.sbt b/build.sbt index 747a09f713..5d4485eb2d 100644 --- a/build.sbt +++ b/build.sbt @@ -21,7 +21,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll ( ) val guavaLib = "com.google.guava" % "guava" % "25.1-jre" -val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.8.5" +val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.9.1" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.3" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion From 8b88bcc60c20315ce3a7c3a6a342aba76e095de7 Mon Sep 17 00:00:00 2001 From: Sameer Brenn Date: Wed, 23 Jun 2021 04:52:09 +0000 Subject: [PATCH 506/796] util-app: Add a Command class which provides a Reader interface to shell commands Problem In Scala & Java we can shell out to a command and there are ways to get the stream of stderr/stdout in real time rather than waiting for the command to complete, but it's hard to get a streaming Reader from it, while also being notified when the command exits with a failure (non-zero exit code), Solution Add a new class which uses Java's `Process` API to shell out a command, and convert the `InputStream` from the command's stdout and stderr into two `Reader`s, where the stdout `Reader` will return a failed `read()` if the process exited non-zero. JIRA Issues: DEPLOY-1186, CSL-11042 Differential Revision: https://phabricator.twitter.biz/D686134 --- CHANGELOG.rst | 6 + util-app/src/main/scala/BUILD | 2 + .../com/twitter/app/command/Command.scala | 46 ++++ .../twitter/app/command/CommandExecutor.scala | 26 ++ .../twitter/app/command/CommandOutput.scala | 95 +++++++ .../app/command/NonZeroReturnCode.scala | 14 + util-app/src/test/resources/command-test.sh | 15 ++ .../com/twitter/app/command/CommandTest.scala | 250 ++++++++++++++++++ .../twitter/app/command/RealCommandTest.scala | 55 ++++ 9 files changed, 509 insertions(+) create mode 100644 util-app/src/main/scala/com/twitter/app/command/Command.scala create mode 100644 util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala create mode 100644 util-app/src/main/scala/com/twitter/app/command/CommandOutput.scala create mode 100644 util-app/src/main/scala/com/twitter/app/command/NonZeroReturnCode.scala create mode 100755 util-app/src/test/resources/command-test.sh create mode 100644 util-app/src/test/scala/com/twitter/app/command/CommandTest.scala create mode 100644 util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b2f28e3f52..3bcd91ab41 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-app: Introduce a new `Command` class which provides a `Reader` interface to the output + of a shell command. ``PHAB_ID=D686134`` + 21.6.0 ------ diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index dec16f5047..811984fd3b 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -11,6 +11,7 @@ scala_library( "util/util-app/src/main/java/com/twitter/app", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io", "util/util-registry/src/main/scala/com/twitter/util/registry", ], exports = [ @@ -18,5 +19,6 @@ scala_library( "util/util-app/src/main/java/com/twitter/app", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", + "util/util-core/src/main/scala/com/twitter/io", ], ) diff --git a/util-app/src/main/scala/com/twitter/app/command/Command.scala b/util-app/src/main/scala/com/twitter/app/command/Command.scala new file mode 100644 index 0000000000..aa52a25380 --- /dev/null +++ b/util-app/src/main/scala/com/twitter/app/command/Command.scala @@ -0,0 +1,46 @@ +package com.twitter.app.command + +import java.io.File + +/** + * A utility to allow for processing the outout from an external process in a streaming manner. + * When [[Command.run]] is called, a [[CommandOutput]] is returned, which provides a + * [[com.twitter.io.Reader]] interface to both the stdout and stderr of the external process. + * + * Example usage: + * {{{ + * // Run a shell command and log the output as it shows up, and then discards the output + * + * val commandOutput: CommandOutput = Command.run(Seq("./slow-shell-command.sh") + * val logged: Reader[Unit] = commandOutput.stdout.map { + * case Buf.Utf8(str) => info(s"stdout: $str") + * } + * Await.result(Reader.readAllLines(logged).onFailure { + * case NonZeroReturnCode(_, code, stderr) => + * val errors = stderr.map { + * case Buf.Utf8(str) => str + * } + * error(s"failure: $code") + * errors.foreach(errLine => error(s"stderr: $errLine") + * }) + * }}} + */ +object Command extends Command(new CommandExecutorImpl) + +class Command private[command] ( + commandExecutor: CommandExecutor) { + + /** + * Runs a shell command and returns a [[CommandOutput]]. See the CommandOutput scaladoc + * for more details on the behavior of fields. + * + * @param cmd The shell command to run + * @param workingDirectory An optional directory from which the command should be run + * @return A [[CommandOutput]] with stdout & stderr from the command + */ + def run(cmd: Seq[String], workingDirectory: Option[File] = None): CommandOutput = { + val process = commandExecutor(cmd, workingDirectory) + CommandOutput.fromProcess(cmd, process) + } + +} diff --git a/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala b/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala new file mode 100644 index 0000000000..c111f3543b --- /dev/null +++ b/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala @@ -0,0 +1,26 @@ +package com.twitter.app.command + +import java.io.File + +/** + * CommandExecutor is a private trait used for testing so that the actual forking of the + * command can be mocked out. + */ +private[command] trait CommandExecutor { + def apply(cmd: Seq[String], workingDirectory: Option[File]): Process +} + +private[command] class CommandExecutorImpl extends CommandExecutor { + + /** + * + * @param cmd A [[Seq]] of [[String]] for the actual command + * @param workingDirectory The directory in which the command will be run + * @return a [[java.lang.Process]] representing the subprocess kicked off + */ + def apply(cmd: Seq[String], workingDirectory: Option[File]): Process = { + val processBuilder = new java.lang.ProcessBuilder(cmd: _*) + workingDirectory.foreach(dir => processBuilder.directory(dir)) + processBuilder.start() + } +} diff --git a/util-app/src/main/scala/com/twitter/app/command/CommandOutput.scala b/util-app/src/main/scala/com/twitter/app/command/CommandOutput.scala new file mode 100644 index 0000000000..f9a3a74635 --- /dev/null +++ b/util-app/src/main/scala/com/twitter/app/command/CommandOutput.scala @@ -0,0 +1,95 @@ +package com.twitter.app.command + +import com.twitter.io.{Buf, Pipe, Reader} +import com.twitter.util.{Future, FuturePool} +import scala.util.control.NonFatal + +object CommandOutput { + private val Delim = '\n'.toByte + private val DelimLength = 1 + private val futurePool = FuturePool.unboundedPool + + /** + * Turns a [[java.lang.Process]] into a [[CommandOutput]]. The stdout and stderr inputstreams are + * converted into a newline-delimited [[Reader]]. If the [[Process]] exits with a non-zero + * return code, the final [[Reader#read]] on stdout will return an exceptional [[Future]]. + * + * @param cmd The command used to create the process (used for the exception message) + * @param process The [[java.lang.Process]] that we are turning into a CommandOutput + * @return + */ + def fromProcess(cmd: Seq[String], process: Process): CommandOutput = { + val outputStream = splitOnNewLines(Reader.fromStream(process.getInputStream)) + val errorStream = splitOnNewLines(Reader.fromStream(process.getErrorStream)) + + val returnCode = futurePool { + process.waitFor() + } + + val endReader = Reader + .fromFuture(returnCode.map { + case 0 => + Reader.empty + case failure => + Reader.exception(NonZeroReturnCode(cmd, failure, errorStream)) + }).flatten + + CommandOutput(Reader.concat(Seq(outputStream, endReader)), errorStream) + } + + /** + * Takes a [[com.twitter.io.Reader]] of [[com.twitter.io.Buf]] and transforms to a [[Reader]] of + * [[Buf]] where each element in the resulting reader is the original data delimited by + * a newline. Delimiters are not included in the output. + * + * @param in The incoming [[Reader]] + * @return A [[Reader]] of the delimited records + * + */ + private[command] def splitOnNewLines( + in: Reader[Buf] + ): Reader[Buf] = { + val pipe = new Pipe[Buf]() + + def copyLoop(accum: Buf): Future[Unit] = { + val delimIndex: Int = accum.process((byte: Byte) => byte != Delim) + if (delimIndex != -1) { + pipe.write(accum.slice(0, delimIndex)).before { + copyLoop(accum.slice(delimIndex + DelimLength, Int.MaxValue)) + } + } else { + in.read().flatMap { + case Some(buf) => copyLoop(accum.concat(buf)) + case None => + (if (!accum.isEmpty) pipe.write(accum) else Future.Unit).before { + pipe.close() + } + }.rescue { + case NonFatal(e) => + pipe.fail(e) + Future.exception(e) + } + } + } + copyLoop(Buf.Empty).onFailure { + case NonFatal(_) => + in.discard() + } + pipe + } +} + +/** + * The output of a shell command. + * + * @param stdout A [[com.twitter.io.Reader]] representing what is written to stdout + * by the shell command. Each [[Buf]] is a line of output from + * the command. If the command exits with a non zero return code, + * the final [[Reader#read()]] will return a failed [[com.twitter.util.Future]] + * with a [[NonZeroReturnCode]] exception. If the shell command + * completes successfully the final read will return a [[None]] + * @param stderr A [[Reader]] with the stderr from the shell command. The final + * read on stderr will always return a [[None]], even if the command exited with a + * nonzero return code. + */ +case class CommandOutput(stdout: Reader[Buf], stderr: Reader[Buf]) diff --git a/util-app/src/main/scala/com/twitter/app/command/NonZeroReturnCode.scala b/util-app/src/main/scala/com/twitter/app/command/NonZeroReturnCode.scala new file mode 100644 index 0000000000..4293b17d4a --- /dev/null +++ b/util-app/src/main/scala/com/twitter/app/command/NonZeroReturnCode.scala @@ -0,0 +1,14 @@ +package com.twitter.app.command + +import com.twitter.io.{Buf, Reader} + +/** + * The exception thrown when the command returns with a nonzero return code. + * + * @param code The actual return code from the command + * @param stdErr A [[Reader]] with the stderr from the command. Note that if the stderr has already + * been consumed, then this [[Reader]] will only contain the portion of the stderr, + * that has not yet been consumed. + */ +case class NonZeroReturnCode(command: Seq[String], code: Int, stdErr: Reader[Buf]) + extends Exception(s"Command $command exited with return code: $code") diff --git a/util-app/src/test/resources/command-test.sh b/util-app/src/test/resources/command-test.sh new file mode 100755 index 0000000000..e119e0a42a --- /dev/null +++ b/util-app/src/test/resources/command-test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# Simple script used for testing Command +# +REPS=$1 +TIME=$2 +EXITCODE=$3 + +for rep in $(seq 1 "$REPS"); do + echo "Stdout # $rep" + >&2 echo "Stderr # $rep" + sleep "$TIME" +done + +exit "$EXITCODE" diff --git a/util-app/src/test/scala/com/twitter/app/command/CommandTest.scala b/util-app/src/test/scala/com/twitter/app/command/CommandTest.scala new file mode 100644 index 0000000000..52c49f95e9 --- /dev/null +++ b/util-app/src/test/scala/com/twitter/app/command/CommandTest.scala @@ -0,0 +1,250 @@ +package com.twitter.app.command + +import com.twitter.conversions.DurationOps._ +import com.twitter.io.{Buf, Reader} +import com.twitter.util.{Await, Awaitable, Promise} +import java.io.{File, InputStream, OutputStream, PipedInputStream, PipedOutputStream} +import org.scalatest.BeforeAndAfterEach +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +class CommandTest extends AnyFunSuite with BeforeAndAfterEach with Matchers { + private def await[A](awaitable: Awaitable[A]): A = Await.result(awaitable, 5.seconds) + + private class FakeCommandExecutorImpl extends CommandExecutor { + private var retValue: Promise[Int] = _ + private var command: Seq[String] = _ + private var workingDirectory: Option[File] = _ + private var errorOutputStream: PipedOutputStream = _ + private var errorInputStream: PipedInputStream = _ + private var stdOutputStream: PipedOutputStream = _ + private var stdInputStream: PipedInputStream = _ + + override def apply( + cmd: Seq[String], + directory: Option[File] + ): Process = { + retValue = new Promise[Int]() + command = cmd + workingDirectory = directory + errorOutputStream = new PipedOutputStream() + errorInputStream = new PipedInputStream(errorOutputStream) + stdOutputStream = new PipedOutputStream() + stdInputStream = new PipedInputStream(stdOutputStream) + + new Process { + override def getOutputStream: OutputStream = ??? + + override def getInputStream: InputStream = stdInputStream + + override def getErrorStream: InputStream = errorInputStream + + override def waitFor(): Int = { + Await.result(retValue, 1.second) + } + + override def exitValue(): Int = ??? + + override def destroy(): Unit = ??? + } + } + + private def checkInitialized(): Unit = { + if (retValue == null || errorInputStream == null || stdInputStream == null || command == null) { + throw new IllegalStateException( + "Can't use fake command executor until the command has been kicked off with apply()") + } + } + + private def writeToStream(buf: Buf, stream: OutputStream): Unit = { + stream.write(Buf.ByteArray.Shared.extract(buf)) + } + + def writeToStdOut(line: Buf): Unit = { + checkInitialized() + writeToStream(line, stdOutputStream) + } + + def writeToStdErr(line: Buf): Unit = { + checkInitialized() + writeToStream(line, errorOutputStream) + } + + def setReturn(value: Int): Unit = { + checkInitialized() + retValue.setValue(value) + errorOutputStream.flush() + stdOutputStream.flush() + errorOutputStream.close() + stdOutputStream.close() + } + + def reset(): Unit = { + command = null + retValue = null + errorOutputStream = null + errorInputStream = null + stdOutputStream = null + stdInputStream = null + } + } + + private val fakeCommandExecutor = new FakeCommandExecutorImpl + private val exposedCommandExecutor: CommandExecutor = fakeCommandExecutor + private val command = new Command(exposedCommandExecutor) + + private def fakeCommandReturnsLines(lines: Buf*): Unit = { + lines.foreach(fakeCommandExecutor.writeToStdOut) + } + + private def fakeCommandReturnsValue(retVal: Int): Unit = { + fakeCommandExecutor.setReturn(retVal) + } + + private def fakeCommandReturnsStdErr(stdErrLines: Buf*): Unit = { + stdErrLines.foreach(fakeCommandExecutor.writeToStdErr) + } + + private def parseUtf8Buf(buf: Buf): String = buf match { + case Buf.Utf8(str) => str + } + + override protected def beforeEach(): Unit = { + fakeCommandExecutor.reset() + } + + test("When command succeeds return reader of lines with stdout and stderr") { + val result = command.run(Seq("command")) + val expectedStdOut = Seq("line 1\n", "line 2\n").map(Buf.Utf8(_)) + val expectedStdErr = Seq("error 1\n", "error 2\n").map(Buf.Utf8(_)) + fakeCommandReturnsLines(expectedStdOut: _*) + fakeCommandReturnsStdErr(expectedStdErr: _*) + fakeCommandReturnsValue(0) + val actualStdOut = await(Reader.readAllItems(result.stdout)) + actualStdOut.map(parseUtf8Buf) shouldBe Seq("line 1", "line 2") + + val actualStdErr = await(Reader.readAllItems(result.stderr)) + actualStdErr.map(parseUtf8Buf) shouldBe Seq("error 1", "error 2") + } + + test("When command fails at end return reader of lines") { + val result = command.run(Seq("command")) + val expectedStdOut = Seq("line 1\n", "line 2\n").map(Buf.Utf8(_)) + val expectedStdErr = Seq("error 1\n", "error 2\n").map(Buf.Utf8(_)) + fakeCommandReturnsLines(expectedStdOut: _*) + fakeCommandReturnsStdErr(expectedStdErr: _*) + fakeCommandReturnsValue(-1) + val stdOutLine1 = await(result.stdout.read()) + val stdOutLine2 = await(result.stdout.read()) + stdOutLine1.map(parseUtf8Buf) shouldBe Some("line 1") + stdOutLine2.map(parseUtf8Buf) shouldBe Some("line 2") + + val stdErrLine1 = await(result.stderr.read()) + stdErrLine1.map(parseUtf8Buf) shouldBe Some("error 1") + + val ex = the[NonZeroReturnCode] thrownBy { + await(result.stdout.read()) + } + + // We don't get the first line because it was already read + await(Reader.readAllItems(ex.stdErr)).map(parseUtf8Buf) shouldBe Seq("error 2") + ex.code shouldBe -1 + } + + test("splitOnNewlinees prpoerly a simple single line with no newline") { + assertSplitOnNewlines(Seq("line 1"), Seq.empty, Some("line 1")) + } + + test("splitOnNewlinees prpoerly splits up a simple single line with a newline") { + assertSplitOnNewlines(Seq("line 1\n"), Seq("line 1"), None) + } + + test("splitOnNewlinees prpoerly splits up a single line split into 3 without a newline") { + assertSplitOnNewlines(Seq("li", "ne", " 1"), Seq.empty, Some("line 1")) + } + + test("splitOnNewlinees prpoerly splits up a single line split into 3 with a newline") { + assertSplitOnNewlines(Seq("li", "ne", " 1\n"), Seq("line 1"), None) + } + + test("splitOnNewlinees splits up a simple single line with a newline in a different segment") { + assertSplitOnNewlines(Seq("line 1", "\n"), Seq("line 1"), None) + } + + test("splitOnNewlinees splits up an empty input") { + assertSplitOnNewlines(Seq.empty, Seq.empty, None) + } + + test("splitOnNewlinees splits up an empty string input") { + assertSplitOnNewlines(Seq(""), Seq.empty, None) + } + + test("splitOnNewlinees splits up just a newline") { + assertSplitOnNewlines(Seq("\n"), Seq(""), None) + } + + test("splitOnNewlinees splits up a multiple lines in one segment with no trailing newline") { + assertSplitOnNewlines(Seq("line 1\nline 2"), Seq("line 1"), Some("line 2")) + } + + test("splitOnNewlinees splits up a multiple lines in one segment with a trailing newline") { + assertSplitOnNewlines(Seq("line 1\nline 2\n"), Seq("line 1", "line 2"), None) + } + + test( + "splitOnNewlinees splits up a multiple lines in multiple segments with no trailing newline") { + assertSplitOnNewlines( + Seq("li", "ne 1\nli", "ne 2\nfinal text"), + Seq("line 1", "line 2"), + Some("final text")) + } + + test("splitOnNewlinees splits up a multiple lines in multiple segments with a trailing newline") { + assertSplitOnNewlines( + Seq("li", "ne 1\nli", "ne 2\nfinal text\n"), + Seq("line 1", "line 2", "final text"), + None) + } + + /** + * A helper function to test many cases around splitOnNewlines. + * + * @param input The strings used to generate the input [[Reader]] + * @param expectedMainOutput What we expect to read from the [[Reader]] before the input [[Reader]] is + * finished + * @param expectedLastLine What we expect to read from the [[Reader]] once the input [[Reader]] is + * finished + */ + private def assertSplitOnNewlines( + input: Seq[String], + expectedMainOutput: Seq[String], + expectedLastLine: Option[String] + ): Unit = { + val finishingPromise = new Promise[Buf]() + val inputReader = Reader.concat( + Seq(Reader.fromSeq(input.map(Buf.Utf8(_))), Reader.fromFuture(finishingPromise))) + + val splitReader = CommandOutput.splitOnNewLines(inputReader) + + // Check main output + expectedMainOutput.foreach { expectedLine => + await(splitReader.read()).map(parseUtf8Buf) shouldBe Some(expectedLine) + } + + // The last line isn't available yet because the reader is not yet done + val finalTextFut = splitReader.read() + assert(!finalTextFut.isDefined) + + finishingPromise.setValue(Buf.Empty) + + // Now we should have a line read, if it's available + await(finalTextFut).map(parseUtf8Buf) shouldBe expectedLastLine + + // If there was content then read again to ensure we are done + if (expectedLastLine.isDefined) { + await(splitReader.read()) shouldBe None + } + + } + +} diff --git a/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala new file mode 100644 index 0000000000..2be08ed5ab --- /dev/null +++ b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala @@ -0,0 +1,55 @@ +package com.twitter.app.command + +import com.twitter.conversions.DurationOps._ +import com.twitter.io.{Buf, Reader} +import com.twitter.util.{Await, Awaitable, Future} +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers + +class RealCommandTest extends AnyFunSuite with Matchers { + private def await[A](awaitable: Awaitable[A]): A = Await.result(awaitable, 5.seconds) + + private def parseUtf8Buf(buf: Buf): String = { + val Buf.Utf8(str) = buf + str + } + + test("Executes a script and gets output") { + val output = + Command.run(Seq("util/util-app/src/test/resources/command-test.sh", "10", "0.1", "0")) + val firstLine = await(output.stdout.read()) + val firstError = await(output.stderr.read()) + firstLine.map(parseUtf8Buf) shouldBe Some("Stdout # 1") + firstError.map(parseUtf8Buf) shouldBe Some("Stderr # 1") + + // rest of lines/errors + val restOut = await(Reader.readAllItems(output.stdout)) + val restErr = await(Reader.readAllItems(output.stderr)) + restOut.map(parseUtf8Buf) shouldBe (2 to 10).map(rep => s"Stdout # $rep") + restErr.map(parseUtf8Buf) shouldBe (2 to 10).map(rep => s"Stderr # $rep") + } + + test("Executes a script and gets failure") { + val output = + Command.run(Seq("util/util-app/src/test/resources/command-test.sh", "10", "0.1", "1")) + + // read 10 lines + val tenLines = await(Future.traverseSequentially(1 to 10) { _ => + output.stdout.read() + }) + + // Read a line from stderr + val stdErrLine = await(output.stderr.read()) + stdErrLine.map(parseUtf8Buf) shouldBe Some("Stderr # 1") + + tenLines.flatten.map(parseUtf8Buf) shouldBe (1 to 10).map(rep => s"Stdout # $rep") + // read last exception + val ex = the[NonZeroReturnCode] thrownBy { await(output.stdout.read()) } + ex.code shouldBe 1 + + // Note that stdErr doesn't have the first line, because it was already read + await(Reader.readAllItems(ex.stdErr)).map(parseUtf8Buf) shouldBe (2 to 10).map { rep => + s"Stderr # $rep" + } + } +} From 643503b21a99c6e937554292fb295a90f0d82840 Mon Sep 17 00:00:00 2001 From: Sameer Brenn Date: Wed, 23 Jun 2021 22:57:16 +0000 Subject: [PATCH 507/796] util-app: Add support for extra environment variables to Command Problem: The `com.twitter.app.command.Command` interface doesn't permit passing additional environment variables to the shell running the command. Solution: Add an `extraEnvs` parameter that will add the keys & values passed in to the environment of the shell command. JIRA Issues: DEPLOY-1186, CSL-11042 Differential Revision: https://phabricator.twitter.biz/D693254 --- .../com/twitter/app/command/Command.scala | 9 ++++-- .../twitter/app/command/CommandExecutor.scala | 15 ++++++++-- util-app/src/test/resources/command-test.sh | 4 +-- .../com/twitter/app/command/CommandTest.scala | 3 +- .../twitter/app/command/RealCommandTest.scala | 30 ++++++++++++------- 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/command/Command.scala b/util-app/src/main/scala/com/twitter/app/command/Command.scala index aa52a25380..1ed0eae60a 100644 --- a/util-app/src/main/scala/com/twitter/app/command/Command.scala +++ b/util-app/src/main/scala/com/twitter/app/command/Command.scala @@ -36,10 +36,15 @@ class Command private[command] ( * * @param cmd The shell command to run * @param workingDirectory An optional directory from which the command should be run + * @param extraEnv A map of extra environment variables to set for the command to be run * @return A [[CommandOutput]] with stdout & stderr from the command */ - def run(cmd: Seq[String], workingDirectory: Option[File] = None): CommandOutput = { - val process = commandExecutor(cmd, workingDirectory) + def run( + cmd: Seq[String], + workingDirectory: Option[File] = None, + extraEnv: Map[String, String] = Map.empty + ): CommandOutput = { + val process = commandExecutor(cmd, workingDirectory, extraEnv) CommandOutput.fromProcess(cmd, process) } diff --git a/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala b/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala index c111f3543b..a33452f46d 100644 --- a/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala +++ b/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala @@ -1,13 +1,18 @@ package com.twitter.app.command import java.io.File +import scala.collection.JavaConverters._ /** * CommandExecutor is a private trait used for testing so that the actual forking of the * command can be mocked out. */ private[command] trait CommandExecutor { - def apply(cmd: Seq[String], workingDirectory: Option[File]): Process + def apply( + cmd: Seq[String], + workingDirectory: Option[File], + extraEnv: Map[String, String] + ): Process } private[command] class CommandExecutorImpl extends CommandExecutor { @@ -16,11 +21,17 @@ private[command] class CommandExecutorImpl extends CommandExecutor { * * @param cmd A [[Seq]] of [[String]] for the actual command * @param workingDirectory The directory in which the command will be run + * @param extraEnv A map of extra environment variables to set for the command to be run * @return a [[java.lang.Process]] representing the subprocess kicked off */ - def apply(cmd: Seq[String], workingDirectory: Option[File]): Process = { + def apply( + cmd: Seq[String], + workingDirectory: Option[File], + extraEnv: Map[String, String] + ): Process = { val processBuilder = new java.lang.ProcessBuilder(cmd: _*) workingDirectory.foreach(dir => processBuilder.directory(dir)) + processBuilder.environment().putAll(extraEnv.asJava) processBuilder.start() } } diff --git a/util-app/src/test/resources/command-test.sh b/util-app/src/test/resources/command-test.sh index e119e0a42a..525fddc53f 100755 --- a/util-app/src/test/resources/command-test.sh +++ b/util-app/src/test/resources/command-test.sh @@ -7,8 +7,8 @@ TIME=$2 EXITCODE=$3 for rep in $(seq 1 "$REPS"); do - echo "Stdout # $rep" - >&2 echo "Stderr # $rep" + echo "Stdout # $rep $EXTRA_ENV" + >&2 echo "Stderr # $rep $EXTRA_ENV" sleep "$TIME" done diff --git a/util-app/src/test/scala/com/twitter/app/command/CommandTest.scala b/util-app/src/test/scala/com/twitter/app/command/CommandTest.scala index 52c49f95e9..b07851a15e 100644 --- a/util-app/src/test/scala/com/twitter/app/command/CommandTest.scala +++ b/util-app/src/test/scala/com/twitter/app/command/CommandTest.scala @@ -22,7 +22,8 @@ class CommandTest extends AnyFunSuite with BeforeAndAfterEach with Matchers { override def apply( cmd: Seq[String], - directory: Option[File] + directory: Option[File], + extraEnv: Map[String, String] ): Process = { retValue = new Promise[Int]() command = cmd diff --git a/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala index 2be08ed5ab..45e140ce91 100644 --- a/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala +++ b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala @@ -3,6 +3,7 @@ package com.twitter.app.command import com.twitter.conversions.DurationOps._ import com.twitter.io.{Buf, Reader} import com.twitter.util.{Await, Awaitable, Future} +import java.io.File import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -16,23 +17,32 @@ class RealCommandTest extends AnyFunSuite with Matchers { test("Executes a script and gets output") { val output = - Command.run(Seq("util/util-app/src/test/resources/command-test.sh", "10", "0.1", "0")) + Command.run( + Seq("./command-test.sh", "10", "0.1", "0"), + Some(new File("util/util-app/src/test/resources")), + Map( + "EXTRA_ENV" -> "test value" + )) val firstLine = await(output.stdout.read()) val firstError = await(output.stderr.read()) - firstLine.map(parseUtf8Buf) shouldBe Some("Stdout # 1") - firstError.map(parseUtf8Buf) shouldBe Some("Stderr # 1") + firstLine.map(parseUtf8Buf) shouldBe Some("Stdout # 1 test value") + firstError.map(parseUtf8Buf) shouldBe Some("Stderr # 1 test value") // rest of lines/errors val restOut = await(Reader.readAllItems(output.stdout)) val restErr = await(Reader.readAllItems(output.stderr)) - restOut.map(parseUtf8Buf) shouldBe (2 to 10).map(rep => s"Stdout # $rep") - restErr.map(parseUtf8Buf) shouldBe (2 to 10).map(rep => s"Stderr # $rep") + restOut.map(parseUtf8Buf) shouldBe (2 to 10).map(rep => s"Stdout # $rep test value") + restErr.map(parseUtf8Buf) shouldBe (2 to 10).map(rep => s"Stderr # $rep test value") } test("Executes a script and gets failure") { val output = - Command.run(Seq("util/util-app/src/test/resources/command-test.sh", "10", "0.1", "1")) - + Command.run( + Seq("./command-test.sh", "10", "0.1", "1"), + Some(new File("util/util-app/src/test/resources")), + Map( + "EXTRA_ENV" -> "test value" + )) // read 10 lines val tenLines = await(Future.traverseSequentially(1 to 10) { _ => output.stdout.read() @@ -40,16 +50,16 @@ class RealCommandTest extends AnyFunSuite with Matchers { // Read a line from stderr val stdErrLine = await(output.stderr.read()) - stdErrLine.map(parseUtf8Buf) shouldBe Some("Stderr # 1") + stdErrLine.map(parseUtf8Buf) shouldBe Some("Stderr # 1 test value") - tenLines.flatten.map(parseUtf8Buf) shouldBe (1 to 10).map(rep => s"Stdout # $rep") + tenLines.flatten.map(parseUtf8Buf) shouldBe (1 to 10).map(rep => s"Stdout # $rep test value") // read last exception val ex = the[NonZeroReturnCode] thrownBy { await(output.stdout.read()) } ex.code shouldBe 1 // Note that stdErr doesn't have the first line, because it was already read await(Reader.readAllItems(ex.stdErr)).map(parseUtf8Buf) shouldBe (2 to 10).map { rep => - s"Stderr # $rep" + s"Stderr # $rep test value" } } } From aaa27deacd64354dbff6d71f091568ccff3c4cce Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 24 Jun 2021 14:34:34 +0000 Subject: [PATCH 508/796] twitter-oss: delete old issue templates and use Twitter org ones Problem: The old version of issue templates is a ISSUE_TEMPLATE.md file. GitHub recommends updating to the new issue template workflow, where templates are placed in a subdirectory called ISSUE_TEMPLATE. The Twitter organization on GitHub already provides templates for bug report, feature request, and custom issues in this new workflow. When someone creates a GitHub issue in Twitter repositories, these are the templates to choose from. Solution: Delete ISSUE_TEMPLATE.md from finagle, finatra, scrooge, twitter-server, util and dodo. This template is not used anyways because the Twitter organization templates are applied. JIRA Issues: CSL-8687 Differential Revision: https://phabricator.twitter.biz/D692804 --- .github/ISSUE_TEMPLATE.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index ef3b9bc736..0000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,13 +0,0 @@ -One line summary of the issue here. - -### Expected behavior - -As concisely as possible, describe the expected behavior. - -### Actual behavior - -As concisely as possible, describe the observed behavior. - -### Steps to reproduce the behavior - -Please list all relevant steps to reproduce the observed behavior. \ No newline at end of file From 981e622a55dd4365f78472ed6435699d47e90337 Mon Sep 17 00:00:00 2001 From: Sameer Brenn Date: Fri, 25 Jun 2021 03:15:01 +0000 Subject: [PATCH 509/796] util-app: Fix RealCommandTest when run via sbt Problem: RealCommandTest does not work when run via sbt because the current working directory is different than when the test is run via pants. Solution: Use the path to the resource in the classloader to find the test script rather than making an assumption about where it is. JIRA Issues: CSL-11042 Differential Revision: https://phabricator.twitter.biz/D694390 --- .../twitter/app/command/RealCommandTest.scala | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala index 45e140ce91..c32ab04aaa 100644 --- a/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala +++ b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala @@ -1,12 +1,16 @@ package com.twitter.app.command +import com.twitter.app.command.RealCommandTest.ScriptPath import com.twitter.conversions.DurationOps._ import com.twitter.io.{Buf, Reader} import com.twitter.util.{Await, Awaitable, Future} -import java.io.File import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +object RealCommandTest { + private val ScriptPath = getClass.getClassLoader.getResource("command-test.sh").getPath +} + class RealCommandTest extends AnyFunSuite with Matchers { private def await[A](awaitable: Awaitable[A]): A = Await.result(awaitable, 5.seconds) @@ -16,13 +20,13 @@ class RealCommandTest extends AnyFunSuite with Matchers { } test("Executes a script and gets output") { - val output = - Command.run( - Seq("./command-test.sh", "10", "0.1", "0"), - Some(new File("util/util-app/src/test/resources")), - Map( - "EXTRA_ENV" -> "test value" - )) + val output = Command.run( + Seq(ScriptPath, "10", "0.1", "0"), + None, + Map( + "EXTRA_ENV" -> "test value" + )) + val firstLine = await(output.stdout.read()) val firstError = await(output.stderr.read()) firstLine.map(parseUtf8Buf) shouldBe Some("Stdout # 1 test value") @@ -38,8 +42,8 @@ class RealCommandTest extends AnyFunSuite with Matchers { test("Executes a script and gets failure") { val output = Command.run( - Seq("./command-test.sh", "10", "0.1", "1"), - Some(new File("util/util-app/src/test/resources")), + Seq(ScriptPath, "10", "0.1", "1"), + None, Map( "EXTRA_ENV" -> "test value" )) From 440d0accf0137782ddb8100a3beba7a564341743 Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Tue, 29 Jun 2021 16:53:37 +0000 Subject: [PATCH 510/796] [util] Fix RealCommandTest on Bazel Problem ------- RealCommandTest currently fails on Bazel because command-test.sh is presented as an in-JAR resource. Solution -------- Copy it out to a temp file. Differential Revision: https://phabricator.twitter.biz/D695878 --- .../twitter/app/command/RealCommandTest.scala | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala index c32ab04aaa..ae03c0c4ab 100644 --- a/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala +++ b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala @@ -2,13 +2,33 @@ package com.twitter.app.command import com.twitter.app.command.RealCommandTest.ScriptPath import com.twitter.conversions.DurationOps._ -import com.twitter.io.{Buf, Reader} +import com.twitter.io.{Buf, Reader, TempFile} import com.twitter.util.{Await, Awaitable, Future} +import java.nio.file.attribute.PosixFilePermission +import java.nio.file.Files import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import scala.collection.JavaConverters._ object RealCommandTest { - private val ScriptPath = getClass.getClassLoader.getResource("command-test.sh").getPath + // Bazel provides the internal dependency as a JAR, and consequently, + // we need to copy the script into a temp directory to use it. + // http://go/bazel-compatibility/no-such-file + private val ScriptPath: String = { + val temp = TempFile.fromResourcePath(this.getClass, "/command-test.sh") + Files.setPosixFilePermissions( + temp.toPath, + Set( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_EXECUTE, + PosixFilePermission.GROUP_READ, + PosixFilePermission.GROUP_EXECUTE, + PosixFilePermission.OTHERS_READ, + PosixFilePermission.OTHERS_EXECUTE + ).asJava + ) + temp.toString + } } class RealCommandTest extends AnyFunSuite with Matchers { From 1efeb9d91e8f790c736a768582ce65834cb0263c Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Wed, 30 Jun 2021 21:22:09 +0000 Subject: [PATCH 511/796] util|finagle|finatra|twitter-server|scrooge: Bump Scalacheck to 1.15.4 in sbt Problem Scala 3 requires Scalacheck 1.15.x but the sbt builds currently uses 1.14.x Solution Upgrade Scalacheck in build.sbt to 1.15.4 to simplify Scala 3 cross-building. Result The tests use Scalacheck 1.15.4. We still use `scalatestplus scalacheck-1-14` because there is no `1-15` version available for Scalatest 3.1.x. Closes twitter/util#291 Signed-off-by: Lily He JIRA Issues: CSL-11064 Differential Revision: https://phabricator.twitter.biz/D691691 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3bcd91ab41..77c82f2a3b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,11 @@ New Features * util-app: Introduce a new `Command` class which provides a `Reader` interface to the output of a shell command. ``PHAB_ID=D686134`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Update Scalatest to version 1.15.4 ``PHAB_ID=D691691`` + 21.6.0 ------ diff --git a/build.sbt b/build.sbt index 5d4485eb2d..b35c9723ed 100644 --- a/build.sbt +++ b/build.sbt @@ -23,7 +23,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll ( val guavaLib = "com.google.guava" % "guava" % "25.1-jre" val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.9.1" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" -val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.14.3" % "test" +val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.15.4" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion def travisTestJavaOptions: Seq[String] = { From 1a7e3051d5b86fa5f4d35db83c2ed49962fec43e Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Fri, 2 Jul 2021 02:59:45 +0000 Subject: [PATCH 512/796] [util] Tag BUILD files as bazel compatible Problem We need internal tracking of `BUILD` files that are bazel-compatible. Solution Tag `BUILD` files with `bazel-compatible`. JIRA Issues: DPB-13999 Differential Revision: https://phabricator.twitter.biz/D691882 --- .../src/main/scala/com/twitter/app/lifecycle/BUILD | 1 + .../src/test/scala/com/twitter/app/lifecycle/BUILD | 5 ++++- util-app/BUILD | 2 ++ util-app/src/main/java/BUILD | 1 + util-app/src/main/java/com/twitter/app/BUILD | 1 + util-app/src/main/scala/BUILD | 1 + util-app/src/test/java/BUILD | 5 ++++- util-app/src/test/resources/BUILD | 1 + util-app/src/test/scala/BUILD | 5 ++++- util-benchmark/BUILD | 1 + util-benchmark/src/main/scala/BUILD | 2 ++ util-benchmark/src/test/scala/BUILD | 1 + util-cache-guava/src/main/scala/BUILD | 1 + .../src/main/scala/com/twitter/cache/guava/BUILD | 1 + util-cache-guava/src/test/java/BUILD | 1 + .../src/test/java/com/twitter/cache/guava/BUILD | 5 ++++- util-cache-guava/src/test/scala/BUILD | 1 + .../src/test/scala/com/twitter/cache/guava/BUILD | 5 ++++- util-cache/BUILD | 2 ++ util-cache/src/main/scala/BUILD | 1 + util-cache/src/main/scala/com/twitter/cache/BUILD | 1 + util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD | 1 + util-cache/src/test/java/BUILD | 5 ++++- util-cache/src/test/scala/BUILD | 1 + util-cache/src/test/scala/com/twitter/cache/BUILD | 6 +++++- util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD | 5 ++++- util-codec/BUILD | 2 ++ util-codec/src/main/scala/BUILD | 1 + util-codec/src/main/scala/com/twitter/util/BUILD | 1 + util-codec/src/test/scala/BUILD | 1 + util-codec/src/test/scala/com/twitter/util/BUILD | 5 ++++- util-core/BUILD | 4 ++++ util-core/src/main/java/BUILD | 1 + util-core/src/main/java/com/twitter/concurrent/BUILD | 1 + util-core/src/main/java/com/twitter/io/BUILD | 1 + util-core/src/main/java/com/twitter/service/BUILD | 1 + util-core/src/main/java/com/twitter/util/BUILD | 1 + util-core/src/main/java/com/twitter/util/javainterop/BUILD | 1 + util-core/src/main/scala/com/twitter/concurrent/BUILD | 1 + util-core/src/main/scala/com/twitter/conversions/BUILD | 1 + util-core/src/main/scala/com/twitter/io/BUILD | 3 ++- util-core/src/test/java/BUILD | 1 + util-core/src/test/java/com/twitter/concurrent/BUILD | 5 ++++- util-core/src/test/java/com/twitter/io/BUILD | 5 ++++- util-core/src/test/java/com/twitter/service/BUILD | 5 ++++- util-core/src/test/java/com/twitter/util/BUILD | 6 +++++- util-core/src/test/java/com/twitter/util/javainterop/BUILD | 5 ++++- util-core/src/test/resources/BUILD | 1 + util-core/src/test/scala/BUILD | 1 + util-core/src/test/scala/com/twitter/concurrent/BUILD | 5 ++++- util-core/src/test/scala/com/twitter/conversions/BUILD | 5 ++++- util-core/src/test/scala/com/twitter/io/BUILD | 5 ++++- util-core/src/test/scala/com/twitter/util/BUILD | 5 ++++- util-function/BUILD | 1 + util-function/src/main/java/BUILD | 1 + util-function/src/main/java/com/twitter/function/BUILD | 1 + util-hashing/BUILD | 2 ++ util-hashing/src/main/scala/BUILD | 1 + util-hashing/src/main/scala/com/twitter/hashing/BUILD | 1 + util-hashing/src/test/resources/BUILD | 1 + util-hashing/src/test/scala/BUILD | 1 + util-hashing/src/test/scala/com/twitter/hashing/BUILD | 5 ++++- .../src/main/java/com/twitter/util/inject/flag/BUILD | 1 + util-inject/src/main/scala/com/twitter/util/inject/BUILD | 1 + .../src/main/scala/com/twitter/util/inject/internal/BUILD | 1 + util-inject/src/test/java/com/twitter/util/inject/BUILD | 1 + util-inject/src/test/scala/com/twitter/util/inject/BUILD | 5 ++++- .../main/java/com/twitter/util/jackson/annotation/BUILD | 1 + .../src/main/scala/com/fasterxml/jackson/databind/BUILD | 1 + util-jackson/src/main/scala/com/twitter/util/jackson/BUILD | 1 + .../main/scala/com/twitter/util/jackson/caseclass/BUILD | 1 + .../com/twitter/util/jackson/caseclass/exceptions/BUILD | 1 + .../src/main/scala/com/twitter/util/jackson/serde/BUILD | 1 + util-jackson/src/test/java/com/twitter/util/jackson/BUILD | 1 + .../src/test/java/com/twitter/util/jackson/tests/BUILD | 1 + util-jackson/src/test/resources/BUILD | 1 + util-jackson/src/test/scala/com/twitter/util/jackson/BUILD | 1 + util-jvm/BUILD | 2 ++ util-jvm/src/main/scala/BUILD | 1 + util-jvm/src/main/scala/com/twitter/jvm/BUILD | 3 ++- util-jvm/src/test/java/BUILD | 5 ++++- util-jvm/src/test/scala/BUILD | 7 ++++++- util-lint/BUILD | 2 ++ util-lint/src/main/scala/BUILD | 1 + util-lint/src/main/scala/com/twitter/util/lint/BUILD | 1 + util-lint/src/test/scala/BUILD | 1 + util-lint/src/test/scala/com/twitter/util/lint/BUILD | 5 ++++- util-logging/BUILD | 2 ++ util-logging/src/main/scala/BUILD | 1 + util-logging/src/main/scala/com/twitter/logging/BUILD | 3 ++- util-logging/src/test/java/BUILD | 5 ++++- util-logging/src/test/scala/BUILD | 5 ++++- util-mock/src/main/scala/com/twitter/util/mock/BUILD | 1 + util-mock/src/test/java/com/twitter/util/mock/BUILD | 1 + util-mock/src/test/scala/com/twitter/util/mock/BUILD | 5 ++++- util-reflect/BUILD | 2 ++ util-reflect/src/main/scala/BUILD | 1 + util-reflect/src/main/scala/com/twitter/util/reflect/BUILD | 1 + util-reflect/src/test/java/com/twitter/util/reflect/BUILD | 1 + util-reflect/src/test/scala/com/twitter/util/reflect/BUILD | 1 + util-registry/BUILD | 2 ++ util-registry/src/main/scala/BUILD | 1 + .../src/main/scala/com/twitter/util/registry/BUILD | 1 + util-registry/src/test/java/BUILD | 5 ++++- util-registry/src/test/scala/BUILD | 5 ++++- util-routing/src/main/scala/com/twitter/util/routing/BUILD | 1 + util-routing/src/test/java/com/twitter/util/routing/BUILD | 5 ++++- util-routing/src/test/scala/com/twitter/util/routing/BUILD | 5 ++++- .../src/test/scala/com/twitter/util/routing/dynamic/BUILD | 1 + .../src/test/scala/com/twitter/util/routing/simple/BUILD | 1 + util-security-test-certs/src/main/resources/BUILD | 1 + util-security/BUILD | 1 + util-security/src/main/scala/BUILD | 1 + .../src/main/scala/com/twitter/util/security/BUILD | 1 + util-security/src/test/java/BUILD | 1 + .../src/test/java/com/twitter/util/security/BUILD | 5 ++++- util-security/src/test/scala/BUILD | 1 + .../src/test/scala/com/twitter/util/security/BUILD | 5 ++++- util-slf4j-api/BUILD | 2 ++ util-slf4j-api/src/main/scala/BUILD | 1 + .../src/main/scala/com/twitter/util/logging/BUILD | 1 + util-slf4j-api/src/test/java/BUILD | 1 + .../src/test/java/com/twitter/util/logging/BUILD | 5 ++++- util-slf4j-api/src/test/resources/BUILD | 1 + util-slf4j-api/src/test/scala/BUILD | 1 + .../src/test/scala/com/twitter/util/logging/BUILD | 6 +++++- util-slf4j-jul-bridge/BUILD | 1 + util-slf4j-jul-bridge/src/main/scala/BUILD | 1 + .../src/main/scala/com/twitter/util/logging/BUILD | 1 + util-stats/BUILD | 2 ++ util-stats/src/main/scala/BUILD | 1 + util-stats/src/main/scala/com/twitter/finagle/stats/BUILD | 1 + util-stats/src/test/java/BUILD | 5 ++++- util-stats/src/test/scala/BUILD | 5 ++++- util-test/BUILD | 2 ++ util-test/src/main/scala/BUILD | 1 + util-test/src/main/scala/com/twitter/logging/BUILD | 1 + util-test/src/main/scala/com/twitter/util/testing/BUILD | 1 + util-test/src/test/scala/BUILD | 1 + util-test/src/test/scala/com/twitter/util/testing/BUILD | 5 ++++- util-thrift/BUILD | 2 ++ util-thrift/src/main/scala/BUILD | 1 + util-thrift/src/main/scala/com/twitter/util/BUILD | 1 + util-thrift/src/test/java/com/twitter/util/BUILD | 1 + util-thrift/src/test/scala/BUILD | 1 + util-thrift/src/test/scala/com/twitter/util/BUILD | 5 ++++- util-tunable/BUILD | 1 + util-tunable/src/main/scala/BUILD | 1 + util-tunable/src/main/scala/com/twitter/util/tunable/BUILD | 3 ++- .../src/main/scala/com/twitter/util/tunable/linter/BUILD | 2 ++ util-tunable/src/test/java/com/twitter/util/tunable/BUILD | 5 ++++- util-tunable/src/test/resources/BUILD | 1 + util-tunable/src/test/scala/BUILD | 1 + util-tunable/src/test/scala/com/twitter/util/tunable/BUILD | 5 ++++- .../src/main/java/com/twitter/util/validation/BUILD | 1 + .../java/com/twitter/util/validation/constraints/BUILD | 1 + util-validator/src/main/resources/BUILD | 1 + .../src/main/scala/com/twitter/util/validation/BUILD | 1 + .../src/main/scala/com/twitter/util/validation/cfg/BUILD | 1 + .../com/twitter/util/validation/constraintvalidation/BUILD | 1 + .../scala/com/twitter/util/validation/conversions/BUILD | 1 + .../main/scala/com/twitter/util/validation/engine/BUILD | 1 + .../scala/com/twitter/util/validation/executable/BUILD | 1 + .../main/scala/com/twitter/util/validation/internal/BUILD | 1 + .../util/validation/internal/constraintvalidation/BUILD | 1 + .../com/twitter/util/validation/internal/engine/BUILD | 1 + .../util/validation/internal/metadata/descriptor/BUILD | 1 + .../com/twitter/util/validation/internal/validators/BUILD | 1 + .../util/validation/internal/validators/notempty/BUILD | 1 + .../twitter/util/validation/internal/validators/size/BUILD | 1 + .../main/scala/com/twitter/util/validation/metadata/BUILD | 1 + .../scala/org/hibernate/validator/internal/engine/BUILD | 1 + .../hibernate/validator/internal/properties/javabean/BUILD | 1 + .../src/test/java/com/twitter/util/validation/BUILD | 1 + .../src/test/java/com/twitter/util/validation/tests/BUILD | 1 + .../src/test/scala/com/twitter/util/validation/BUILD | 1 + util-zk-test/BUILD | 2 ++ util-zk-test/src/main/scala/BUILD | 1 + util-zk-test/src/main/scala/com/twitter/zk/BUILD | 1 + util-zk-test/src/test/java/BUILD | 5 ++++- util-zk-test/src/test/scala/BUILD | 5 ++++- util-zk/BUILD | 2 ++ util-zk/src/main/scala/BUILD | 1 + util-zk/src/main/scala/com/twitter/zk/BUILD | 5 +++-- util-zk/src/main/scala/com/twitter/zk/coordination/BUILD | 5 +++-- util-zk/src/test/scala/BUILD | 1 + util-zk/src/test/scala/com/twitter/zk/BUILD | 5 ++++- util-zk/src/test/scala/com/twitter/zk/coordination/BUILD | 5 ++++- 188 files changed, 353 insertions(+), 52 deletions(-) diff --git a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD index 1ec3f09863..2e1962913b 100644 --- a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD +++ b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-app-lifecycle", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", ], diff --git a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD index 32ececd3e5..2bcfb504e7 100644 --- a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD +++ b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-app/BUILD b/util-app/BUILD index 6fd9f53aa0..396a12cf8f 100644 --- a/util-app/BUILD +++ b/util-app/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-app/src/main/java", "util/util-app/src/main/scala", @@ -7,6 +8,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-app/src/test/java", "util/util-app/src/test/scala", diff --git a/util-app/src/main/java/BUILD b/util-app/src/main/java/BUILD index 135fed783f..dcbe740001 100644 --- a/util-app/src/main/java/BUILD +++ b/util-app/src/main/java/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-app/src/main/java/com/twitter/app", "util/util-app/src/main/scala", diff --git a/util-app/src/main/java/com/twitter/app/BUILD b/util-app/src/main/java/com/twitter/app/BUILD index 07b35e6f1a..47798edc03 100644 --- a/util-app/src/main/java/com/twitter/app/BUILD +++ b/util-app/src/main/java/com/twitter/app/BUILD @@ -7,4 +7,5 @@ java_library( ), # Not detected as required by zinc. scope = "forced", + tags = ["bazel-compatible"], ) diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index 811984fd3b..259e20a441 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-app", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle", "util/util-app/src/main/java/com/twitter/app", diff --git a/util-app/src/test/java/BUILD b/util-app/src/test/java/BUILD index c04d96c844..bc7b8f6f54 100644 --- a/util-app/src/test/java/BUILD +++ b/util-app/src/test/java/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-app/src/test/resources/BUILD b/util-app/src/test/resources/BUILD index 45a90181cc..07015f79b6 100644 --- a/util-app/src/test/resources/BUILD +++ b/util-app/src/test/resources/BUILD @@ -4,4 +4,5 @@ resources( "!BUILD*", "**/*", ], + tags = ["bazel-compatible"], ) diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index e0a7f5da4d..3fe4b661a6 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-benchmark/BUILD b/util-benchmark/BUILD index a3e0646ab6..bf846a7ed7 100644 --- a/util-benchmark/BUILD +++ b/util-benchmark/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-benchmark/src/main/scala", ], diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index e884e380b1..78af6dda35 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -4,6 +4,7 @@ scala_library( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/jakarta/validation:jakarta.validation-api", @@ -22,6 +23,7 @@ scala_library( jvm_binary( name = "jmh", main = "org.openjdk.jmh.Main", + tags = ["bazel-compatible"], dependencies = [ ":scala", ], diff --git a/util-benchmark/src/test/scala/BUILD b/util-benchmark/src/test/scala/BUILD index 0a92dcf982..9f3bc2c1cb 100644 --- a/util-benchmark/src/test/scala/BUILD +++ b/util-benchmark/src/test/scala/BUILD @@ -3,6 +3,7 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", diff --git a/util-cache-guava/src/main/scala/BUILD b/util-cache-guava/src/main/scala/BUILD index 9be4964b4a..07a15b9193 100644 --- a/util-cache-guava/src/main/scala/BUILD +++ b/util-cache-guava/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-cache-guava/src/main/scala/com/twitter/cache/guava", ], diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD index cbd02c37cc..e174575ae9 100644 --- a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-cache-guava", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/guava", "util/util-cache/src/main/scala/com/twitter/cache", diff --git a/util-cache-guava/src/test/java/BUILD b/util-cache-guava/src/test/java/BUILD index e4a98fbfe1..c922f72e7b 100644 --- a/util-cache-guava/src/test/java/BUILD +++ b/util-cache-guava/src/test/java/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-cache-guava/src/test/java/com/twitter/cache/guava", ], diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD index 198952405c..99fb6f4bcb 100644 --- a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/com/google/guava", "3rdparty/jvm/junit", diff --git a/util-cache-guava/src/test/scala/BUILD b/util-cache-guava/src/test/scala/BUILD index 247847df14..4970d3153b 100644 --- a/util-cache-guava/src/test/scala/BUILD +++ b/util-cache-guava/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-cache-guava/src/test/scala/com/twitter/cache/guava", ], diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD index 234b8cf7ab..0fa75f778e 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/com/google/code/findbugs:jsr305", "3rdparty/jvm/com/google/guava", diff --git a/util-cache/BUILD b/util-cache/BUILD index 924027a1a8..2efa01e029 100644 --- a/util-cache/BUILD +++ b/util-cache/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-cache/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-cache/src/test/java", "util/util-cache/src/test/scala", diff --git a/util-cache/src/main/scala/BUILD b/util-cache/src/main/scala/BUILD index 9fade1261a..e280d8024a 100644 --- a/util-cache/src/main/scala/BUILD +++ b/util-cache/src/main/scala/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-cache", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", diff --git a/util-cache/src/main/scala/com/twitter/cache/BUILD b/util-cache/src/main/scala/com/twitter/cache/BUILD index 3a3ff184c8..bf995eff07 100644 --- a/util-cache/src/main/scala/com/twitter/cache/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-cache-cache", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", ], diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD index 3e53ea41d0..e7cb5b2e03 100644 --- a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-cache-caffeine", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "util/util-cache/src/main/scala/com/twitter/cache", diff --git a/util-cache/src/test/java/BUILD b/util-cache/src/test/java/BUILD index 98b491de57..530e0dc9f9 100644 --- a/util-cache/src/test/java/BUILD +++ b/util-cache/src/test/java/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-cache/src/test/scala/BUILD b/util-cache/src/test/scala/BUILD index b368a969e4..99b826c8d9 100644 --- a/util-cache/src/test/scala/BUILD +++ b/util-cache/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-cache/src/test/scala/com/twitter/cache", "util/util-cache/src/test/scala/com/twitter/cache/caffeine", diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index ce5e374b2e..ff9989b804 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -5,7 +5,10 @@ junit_tests( "*.scala", ], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ ":abstract_tests", "3rdparty/jvm/com/github/ben-manes/caffeine", @@ -28,6 +31,7 @@ scala_library( "AbstractLoadingFutureCacheTest.scala", ], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index 928759b970..d134aa8d72 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/com/google/code/findbugs:jsr305", diff --git a/util-codec/BUILD b/util-codec/BUILD index fe7f171d97..4a62ee8b7e 100644 --- a/util-codec/BUILD +++ b/util-codec/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-codec/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-codec/src/test/scala", ], diff --git a/util-codec/src/main/scala/BUILD b/util-codec/src/main/scala/BUILD index 6d8306a0d3..a1251e2edf 100644 --- a/util-codec/src/main/scala/BUILD +++ b/util-codec/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-codec/src/main/scala/com/twitter/util", ], diff --git a/util-codec/src/main/scala/com/twitter/util/BUILD b/util-codec/src/main/scala/com/twitter/util/BUILD index e500b175e8..7121b15b7e 100644 --- a/util-codec/src/main/scala/com/twitter/util/BUILD +++ b/util-codec/src/main/scala/com/twitter/util/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-codec", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core/src/main/scala/com/twitter/io", ], diff --git a/util-codec/src/test/scala/BUILD b/util-codec/src/test/scala/BUILD index 9d2bcbe66e..a3e7a3addd 100644 --- a/util-codec/src/test/scala/BUILD +++ b/util-codec/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-codec/src/test/scala/com/twitter/util", ], diff --git a/util-codec/src/test/scala/com/twitter/util/BUILD b/util-codec/src/test/scala/com/twitter/util/BUILD index 153c15c7ef..413e57007b 100644 --- a/util-codec/src/test/scala/com/twitter/util/BUILD +++ b/util-codec/src/test/scala/com/twitter/util/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-core/BUILD b/util-core/BUILD index 4cbea79d7c..e4143e2ea9 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ ":scala", "util/util-core/src/main/java", @@ -7,6 +8,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-core/src/test/java", "util/util-core/src/test/scala", @@ -29,6 +31,7 @@ scala_library( name = "util-core-util", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-reflect", "3rdparty/jvm/org/scala-lang/modules:scala-collection-compat", @@ -51,6 +54,7 @@ scala_library( name = "util-core", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ ":util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", diff --git a/util-core/src/main/java/BUILD b/util-core/src/main/java/BUILD index 140c441eee..4c75a6e28e 100644 --- a/util-core/src/main/java/BUILD +++ b/util-core/src/main/java/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-core/src/main/java/com/twitter/concurrent", "util/util-core/src/main/java/com/twitter/io", diff --git a/util-core/src/main/java/com/twitter/concurrent/BUILD b/util-core/src/main/java/com/twitter/concurrent/BUILD index a89617e9a8..29917ba462 100644 --- a/util-core/src/main/java/com/twitter/concurrent/BUILD +++ b/util-core/src/main/java/com/twitter/concurrent/BUILD @@ -5,6 +5,7 @@ java_library( name = "util-core-java-concurrent", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", "util/util-core:scala", diff --git a/util-core/src/main/java/com/twitter/io/BUILD b/util-core/src/main/java/com/twitter/io/BUILD index 92ea031f28..83f7ca73fc 100644 --- a/util-core/src/main/java/com/twitter/io/BUILD +++ b/util-core/src/main/java/com/twitter/io/BUILD @@ -5,6 +5,7 @@ java_library( name = "util-core-java-io", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", "util/util-core:scala", diff --git a/util-core/src/main/java/com/twitter/service/BUILD b/util-core/src/main/java/com/twitter/service/BUILD index 927ce868bd..fd6484b981 100644 --- a/util-core/src/main/java/com/twitter/service/BUILD +++ b/util-core/src/main/java/com/twitter/service/BUILD @@ -6,4 +6,5 @@ java_library( name = "util-core-java-service", repo = artifactory, ), + tags = ["bazel-compatible"], ) diff --git a/util-core/src/main/java/com/twitter/util/BUILD b/util-core/src/main/java/com/twitter/util/BUILD index 2af6b2cb13..d0e3d94d13 100644 --- a/util-core/src/main/java/com/twitter/util/BUILD +++ b/util-core/src/main/java/com/twitter/util/BUILD @@ -5,6 +5,7 @@ java_library( name = "util-core-java", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", "util/util-core:scala", diff --git a/util-core/src/main/java/com/twitter/util/javainterop/BUILD b/util-core/src/main/java/com/twitter/util/javainterop/BUILD index 5e3cf913ce..cd785e5fc0 100644 --- a/util-core/src/main/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/main/java/com/twitter/util/javainterop/BUILD @@ -5,6 +5,7 @@ java_library( name = "util-core-java-javainterop", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", ], diff --git a/util-core/src/main/scala/com/twitter/concurrent/BUILD b/util-core/src/main/scala/com/twitter/concurrent/BUILD index b01b7650c4..19b636b184 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/main/scala/com/twitter/concurrent/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-core-concurrent", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", ], diff --git a/util-core/src/main/scala/com/twitter/conversions/BUILD b/util-core/src/main/scala/com/twitter/conversions/BUILD index a7f5379883..61cd4788da 100644 --- a/util-core/src/main/scala/com/twitter/conversions/BUILD +++ b/util-core/src/main/scala/com/twitter/conversions/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-core-conversions", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", ], diff --git a/util-core/src/main/scala/com/twitter/io/BUILD b/util-core/src/main/scala/com/twitter/io/BUILD index 76baf480e2..0e936c2914 100644 --- a/util-core/src/main/scala/com/twitter/io/BUILD +++ b/util-core/src/main/scala/com/twitter/io/BUILD @@ -6,8 +6,9 @@ scala_library( name = "util-core-io", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ - "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/concurrent", ], ) diff --git a/util-core/src/test/java/BUILD b/util-core/src/test/java/BUILD index cf8f19a631..4d9e1bb8aa 100644 --- a/util-core/src/test/java/BUILD +++ b/util-core/src/test/java/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-core/src/test/java/com/twitter/concurrent", "util/util-core/src/test/java/com/twitter/io", diff --git a/util-core/src/test/java/com/twitter/concurrent/BUILD b/util-core/src/test/java/com/twitter/concurrent/BUILD index 7df5c79937..e4c62645a0 100644 --- a/util-core/src/test/java/com/twitter/concurrent/BUILD +++ b/util-core/src/test/java/com/twitter/concurrent/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD index a85d8e909e..d7c025c52e 100644 --- a/util-core/src/test/java/com/twitter/io/BUILD +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/test/java/com/twitter/service/BUILD b/util-core/src/test/java/com/twitter/service/BUILD index ed94882638..4ad37e5f2c 100644 --- a/util-core/src/test/java/com/twitter/service/BUILD +++ b/util-core/src/test/java/com/twitter/service/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index 7a034e0a85..bcdbfdad4b 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -2,6 +2,7 @@ java_library( name = "object_size_calculator", sources = ["ObjectSizeCalculator.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/com/google/errorprone:error_prone_annotations", @@ -14,7 +15,10 @@ junit_tests( "*.java", ], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ ":object_size_calculator", "3rdparty/jvm/com/github/ben-manes/caffeine", diff --git a/util-core/src/test/java/com/twitter/util/javainterop/BUILD b/util-core/src/test/java/com/twitter/util/javainterop/BUILD index d7d64df4e1..d1c67a6836 100644 --- a/util-core/src/test/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/test/java/com/twitter/util/javainterop/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/test/resources/BUILD b/util-core/src/test/resources/BUILD index a001371a96..301b666c21 100644 --- a/util-core/src/test/resources/BUILD +++ b/util-core/src/test/resources/BUILD @@ -1,3 +1,4 @@ resources( sources = ["**/*"], + tags = ["bazel-compatible"], ) diff --git a/util-core/src/test/scala/BUILD b/util-core/src/test/scala/BUILD index 1919c2211a..5cb247c531 100644 --- a/util-core/src/test/scala/BUILD +++ b/util-core/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-core/src/test/scala/com/twitter/concurrent", "util/util-core/src/test/scala/com/twitter/conversions", diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index 10d8afc5cf..c89563ca72 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD index 049ebae433..0ff8820550 100644 --- a/util-core/src/test/scala/com/twitter/conversions/BUILD +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index da3a621a43..2a8f0b9245 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index 53c653d3e2..2cde85376f 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-function/BUILD b/util-function/BUILD index 82d3156556..d316723ed2 100644 --- a/util-function/BUILD +++ b/util-function/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-function/src/main/java", ], diff --git a/util-function/src/main/java/BUILD b/util-function/src/main/java/BUILD index 96528c9177..969c923223 100644 --- a/util-function/src/main/java/BUILD +++ b/util-function/src/main/java/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-function/src/main/java/com/twitter/function", ], diff --git a/util-function/src/main/java/com/twitter/function/BUILD b/util-function/src/main/java/com/twitter/function/BUILD index 5ca28b5f22..3e368d9187 100644 --- a/util-function/src/main/java/com/twitter/function/BUILD +++ b/util-function/src/main/java/com/twitter/function/BUILD @@ -5,4 +5,5 @@ java_library( name = "util-function-java", repo = artifactory, ), + tags = ["bazel-compatible"], ) diff --git a/util-hashing/BUILD b/util-hashing/BUILD index afce834f44..62d56876e1 100644 --- a/util-hashing/BUILD +++ b/util-hashing/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-hashing/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-hashing/src/test/scala", ], diff --git a/util-hashing/src/main/scala/BUILD b/util-hashing/src/main/scala/BUILD index 47781f687f..e55f18e86a 100644 --- a/util-hashing/src/main/scala/BUILD +++ b/util-hashing/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-hashing/src/main/scala/com/twitter/hashing", ], diff --git a/util-hashing/src/main/scala/com/twitter/hashing/BUILD b/util-hashing/src/main/scala/com/twitter/hashing/BUILD index df6e6067dd..3986ec5c74 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/main/scala/com/twitter/hashing/BUILD @@ -5,4 +5,5 @@ scala_library( name = "util-hashing", repo = artifactory, ), + tags = ["bazel-compatible"], ) diff --git a/util-hashing/src/test/resources/BUILD b/util-hashing/src/test/resources/BUILD index 45a90181cc..07015f79b6 100644 --- a/util-hashing/src/test/resources/BUILD +++ b/util-hashing/src/test/resources/BUILD @@ -4,4 +4,5 @@ resources( "!BUILD*", "**/*", ], + tags = ["bazel-compatible"], ) diff --git a/util-hashing/src/test/scala/BUILD b/util-hashing/src/test/scala/BUILD index c62ca977cc..403d5d45a0 100644 --- a/util-hashing/src/test/scala/BUILD +++ b/util-hashing/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-hashing/src/test/scala/com/twitter/hashing", ], diff --git a/util-hashing/src/test/scala/com/twitter/hashing/BUILD b/util-hashing/src/test/scala/com/twitter/hashing/BUILD index 88b50365b7..1e2a259c9f 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/test/scala/com/twitter/hashing/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", diff --git a/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD b/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD index 56acef0a01..abf6148959 100644 --- a/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD +++ b/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD @@ -7,6 +7,7 @@ java_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", ], diff --git a/util-inject/src/main/scala/com/twitter/util/inject/BUILD b/util-inject/src/main/scala/com/twitter/util/inject/BUILD index 47f79d4198..5790b5347e 100644 --- a/util-inject/src/main/scala/com/twitter/util/inject/BUILD +++ b/util-inject/src/main/scala/com/twitter/util/inject/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "//:scala-reflect", "3rdparty/jvm/com/google/inject:guice", diff --git a/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD b/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD index 22537bef23..828882d060 100644 --- a/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD +++ b/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/javax/inject:javax.inject", diff --git a/util-inject/src/test/java/com/twitter/util/inject/BUILD b/util-inject/src/test/java/com/twitter/util/inject/BUILD index 961efa2be8..3fda5202cc 100644 --- a/util-inject/src/test/java/com/twitter/util/inject/BUILD +++ b/util-inject/src/test/java/com/twitter/util/inject/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = [], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", ], diff --git a/util-inject/src/test/scala/com/twitter/util/inject/BUILD b/util-inject/src/test/scala/com/twitter/util/inject/BUILD index 07728b313a..ef10ad10fe 100644 --- a/util-inject/src/test/scala/com/twitter/util/inject/BUILD +++ b/util-inject/src/test/scala/com/twitter/util/inject/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/com/google/inject:guice", "3rdparty/jvm/com/google/inject/extensions:guice-testlib", diff --git a/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD b/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD index f3f0df4f7c..abf69ac7be 100644 --- a/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD +++ b/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD @@ -7,4 +7,5 @@ java_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], ) diff --git a/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD index f3a51ec8a5..434b79a7c1 100644 --- a/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD +++ b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD index 61ab5add42..c5ee98857a 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD index 356259ed5b..e011cf42e6 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD index ff58b07f4e..5b2663b130 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD index 4452f36502..0a5c93b812 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/BUILD index d853a7cc0b..68c104eb52 100644 --- a/util-jackson/src/test/java/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/java/com/twitter/util/jackson/BUILD @@ -2,6 +2,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], strict_deps = "True", + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", ], diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD index 47cde59828..ce8b05468f 100644 --- a/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD +++ b/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD @@ -1,5 +1,6 @@ junit_tests( sources = ["**/*.java"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson-dataformat-yaml", "3rdparty/jvm/junit", diff --git a/util-jackson/src/test/resources/BUILD b/util-jackson/src/test/resources/BUILD index a001371a96..301b666c21 100644 --- a/util-jackson/src/test/resources/BUILD +++ b/util-jackson/src/test/resources/BUILD @@ -1,3 +1,4 @@ resources( sources = ["**/*"], + tags = ["bazel-compatible"], ) diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD index b630aac594..8ba58ec7ea 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD @@ -5,6 +5,7 @@ junit_tests( "caseclass/*.scala", "serde/*.scala", ], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", diff --git a/util-jvm/BUILD b/util-jvm/BUILD index 43bac99308..f756c4066d 100644 --- a/util-jvm/BUILD +++ b/util-jvm/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-jvm/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-jvm/src/test/java", "util/util-jvm/src/test/scala", diff --git a/util-jvm/src/main/scala/BUILD b/util-jvm/src/main/scala/BUILD index 1140b99e90..0f5e600d28 100644 --- a/util-jvm/src/main/scala/BUILD +++ b/util-jvm/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-jvm/src/main/scala/com/twitter/jvm", ], diff --git a/util-jvm/src/main/scala/com/twitter/jvm/BUILD b/util-jvm/src/main/scala/com/twitter/jvm/BUILD index f45ac1032f..665c3259a2 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/main/scala/com/twitter/jvm/BUILD @@ -5,10 +5,11 @@ scala_library( name = "util-jvm", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-app/src/main/scala", - "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/conversions", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], exports = [ diff --git a/util-jvm/src/test/java/BUILD b/util-jvm/src/test/java/BUILD index d58dfaaab6..3ef7054c13 100644 --- a/util-jvm/src/test/java/BUILD +++ b/util-jvm/src/test/java/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index 7a9cd28710..b2295dac08 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -15,7 +15,10 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = COMMON_DEPS + ["3rdparty/jvm/org/scalatestplus:junit"], ) @@ -29,11 +32,13 @@ scala_library( # does not strictly enforce it. sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = COMMON_DEPS, ) jvm_binary( name = "bin", main = "com.twitter.jvm.EstimatorApp", + tags = ["bazel-compatible"], dependencies = [":estimator_app"], ) diff --git a/util-lint/BUILD b/util-lint/BUILD index 109f5946d3..2265db4d28 100644 --- a/util-lint/BUILD +++ b/util-lint/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-lint/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-lint/src/test/scala", ], diff --git a/util-lint/src/main/scala/BUILD b/util-lint/src/main/scala/BUILD index 01f00626ec..89cef8c41c 100644 --- a/util-lint/src/main/scala/BUILD +++ b/util-lint/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-lint/src/main/scala/com/twitter/util/lint", ], diff --git a/util-lint/src/main/scala/com/twitter/util/lint/BUILD b/util-lint/src/main/scala/com/twitter/util/lint/BUILD index ee4416b514..b2b903bfc7 100644 --- a/util-lint/src/main/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/main/scala/com/twitter/util/lint/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-lint", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", ], diff --git a/util-lint/src/test/scala/BUILD b/util-lint/src/test/scala/BUILD index f683daf038..549eba77b8 100644 --- a/util-lint/src/test/scala/BUILD +++ b/util-lint/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-lint/src/test/scala/com/twitter/util/lint", ], diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index 6ff2a42d8f..5856cf7ac8 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-logging/BUILD b/util-logging/BUILD index 08133e0c3a..7271a72ab1 100644 --- a/util-logging/BUILD +++ b/util-logging/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-logging/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-logging/src/test/java", "util/util-logging/src/test/scala", diff --git a/util-logging/src/main/scala/BUILD b/util-logging/src/main/scala/BUILD index 64906b48ca..3f66d5dfc9 100644 --- a/util-logging/src/main/scala/BUILD +++ b/util-logging/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-logging/src/main/scala/com/twitter/logging", ], diff --git a/util-logging/src/main/scala/com/twitter/logging/BUILD b/util-logging/src/main/scala/com/twitter/logging/BUILD index fcf8ff1edd..e59dd7b84b 100644 --- a/util-logging/src/main/scala/com/twitter/logging/BUILD +++ b/util-logging/src/main/scala/com/twitter/logging/BUILD @@ -6,11 +6,12 @@ scala_library( name = "util-logging", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-app/src/main/scala", + "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core:util-core-util", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], exports = [ diff --git a/util-logging/src/test/java/BUILD b/util-logging/src/test/java/BUILD index 397b0eee4d..af4c2a831c 100644 --- a/util-logging/src/test/java/BUILD +++ b/util-logging/src/test/java/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 9bbbd0d745..3f81913fd5 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-mock/src/main/scala/com/twitter/util/mock/BUILD b/util-mock/src/main/scala/com/twitter/util/mock/BUILD index 3e1050d2a9..9178b6c262 100644 --- a/util-mock/src/main/scala/com/twitter/util/mock/BUILD +++ b/util-mock/src/main/scala/com/twitter/util/mock/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/mockito:mockito-scala", diff --git a/util-mock/src/test/java/com/twitter/util/mock/BUILD b/util-mock/src/test/java/com/twitter/util/mock/BUILD index f81ed67772..3c58bcbf7d 100644 --- a/util-mock/src/test/java/com/twitter/util/mock/BUILD +++ b/util-mock/src/test/java/com/twitter/util/mock/BUILD @@ -2,6 +2,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], strict_deps = "True", + tags = ["bazel-compatible"], dependencies = [ ], ) diff --git a/util-mock/src/test/scala/com/twitter/util/mock/BUILD b/util-mock/src/test/scala/com/twitter/util/mock/BUILD index 8e28a51df1..fab2e74fe3 100644 --- a/util-mock/src/test/scala/com/twitter/util/mock/BUILD +++ b/util-mock/src/test/scala/com/twitter/util/mock/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/mockito:mockito-scala", diff --git a/util-reflect/BUILD b/util-reflect/BUILD index ee63e07e1c..42fc74881d 100644 --- a/util-reflect/BUILD +++ b/util-reflect/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-reflect/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ ], ) diff --git a/util-reflect/src/main/scala/BUILD b/util-reflect/src/main/scala/BUILD index b3d5909852..3282f48b4b 100644 --- a/util-reflect/src/main/scala/BUILD +++ b/util-reflect/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-reflect/src/main/scala/com/twitter/util/reflect", ], diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD index a98d2a0595..74ea3ef074 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-reflect", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", ], diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/BUILD b/util-reflect/src/test/java/com/twitter/util/reflect/BUILD index fc51b55b44..773deadb6b 100644 --- a/util-reflect/src/test/java/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/java/com/twitter/util/reflect/BUILD @@ -1,4 +1,5 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], ) diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD index 224a0035b0..7addfa0ec1 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-registry/BUILD b/util-registry/BUILD index c1361d7722..fcc4317f29 100644 --- a/util-registry/BUILD +++ b/util-registry/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-registry/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-registry/src/test/java", "util/util-registry/src/test/scala", diff --git a/util-registry/src/main/scala/BUILD b/util-registry/src/main/scala/BUILD index 6c76327ab4..f22c6d263a 100644 --- a/util-registry/src/main/scala/BUILD +++ b/util-registry/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-registry/src/main/scala/com/twitter/util/registry", ], diff --git a/util-registry/src/main/scala/com/twitter/util/registry/BUILD b/util-registry/src/main/scala/com/twitter/util/registry/BUILD index 4f4cbd55e1..62cfff778a 100644 --- a/util-registry/src/main/scala/com/twitter/util/registry/BUILD +++ b/util-registry/src/main/scala/com/twitter/util/registry/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-registry", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", ], diff --git a/util-registry/src/test/java/BUILD b/util-registry/src/test/java/BUILD index d0b5cc4eb8..f433781d70 100644 --- a/util-registry/src/test/java/BUILD +++ b/util-registry/src/test/java/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index e23721ba6f..a245b56740 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-routing/src/main/scala/com/twitter/util/routing/BUILD b/util-routing/src/main/scala/com/twitter/util/routing/BUILD index 2c7510a5f5..58ed27f316 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/main/scala/com/twitter/util/routing/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-routing", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", diff --git a/util-routing/src/test/java/com/twitter/util/routing/BUILD b/util-routing/src/test/java/com/twitter/util/routing/BUILD index 388804e4fd..b5a2cd8386 100644 --- a/util-routing/src/test/java/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/java/com/twitter/util/routing/BUILD @@ -3,7 +3,10 @@ junit_tests( "*.java", ], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index 065cec5c54..80c5daa72b 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD index 0135c0eb05..bdc9144cd6 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", "util/util-routing/src/main/scala/com/twitter/util/routing", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD index 92fbcb59a9..49ab57e12e 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", "util/util-logging/src/main/scala/com/twitter/logging", diff --git a/util-security-test-certs/src/main/resources/BUILD b/util-security-test-certs/src/main/resources/BUILD index 45a90181cc..07015f79b6 100644 --- a/util-security-test-certs/src/main/resources/BUILD +++ b/util-security-test-certs/src/main/resources/BUILD @@ -4,4 +4,5 @@ resources( "!BUILD*", "**/*", ], + tags = ["bazel-compatible"], ) diff --git a/util-security/BUILD b/util-security/BUILD index de525d1381..75d5e84a11 100644 --- a/util-security/BUILD +++ b/util-security/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-security/src/main/scala", ], diff --git a/util-security/src/main/scala/BUILD b/util-security/src/main/scala/BUILD index e337f9b286..8596e1a998 100644 --- a/util-security/src/main/scala/BUILD +++ b/util-security/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-security/src/main/scala/com/twitter/util/security", ], diff --git a/util-security/src/main/scala/com/twitter/util/security/BUILD b/util-security/src/main/scala/com/twitter/util/security/BUILD index f60689e460..ce3549f163 100644 --- a/util-security/src/main/scala/com/twitter/util/security/BUILD +++ b/util-security/src/main/scala/com/twitter/util/security/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-security", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", "util/util-logging/src/main/scala/com/twitter/logging", diff --git a/util-security/src/test/java/BUILD b/util-security/src/test/java/BUILD index 61076e5d29..da97a848c8 100644 --- a/util-security/src/test/java/BUILD +++ b/util-security/src/test/java/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-security/src/test/java/com/twitter/util/security", ], diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD index f8e3dcd324..f5d30abd13 100644 --- a/util-security/src/test/java/com/twitter/util/security/BUILD +++ b/util-security/src/test/java/com/twitter/util/security/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-security/src/test/scala/BUILD b/util-security/src/test/scala/BUILD index 3293572245..019705b791 100644 --- a/util-security/src/test/scala/BUILD +++ b/util-security/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-security/src/test/scala/com/twitter/util/security", ], diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index 11ac11f450..fcb291b7d7 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalacheck", diff --git a/util-slf4j-api/BUILD b/util-slf4j-api/BUILD index 22da9a28ad..f51e55cc98 100644 --- a/util-slf4j-api/BUILD +++ b/util-slf4j-api/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-slf4j-api/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-slf4j-api/src/test/scala", ], diff --git a/util-slf4j-api/src/main/scala/BUILD b/util-slf4j-api/src/main/scala/BUILD index dc62b38ffd..dc234d49d1 100644 --- a/util-slf4j-api/src/main/scala/BUILD +++ b/util-slf4j-api/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD index 6103f9e7d2..344c06a330 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-slf4j", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/slf4j:slf4j-api", ], diff --git a/util-slf4j-api/src/test/java/BUILD b/util-slf4j-api/src/test/java/BUILD index 8ecefa1c4b..2c3035dbb1 100644 --- a/util-slf4j-api/src/test/java/BUILD +++ b/util-slf4j-api/src/test/java/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-slf4j-api/src/test/java/com/twitter/util/logging", ], diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD index e0b2ae8b7d..698c646f0b 100644 --- a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-slf4j-api/src/test/resources/BUILD b/util-slf4j-api/src/test/resources/BUILD index ba87e2b54d..99b7ffad66 100644 --- a/util-slf4j-api/src/test/resources/BUILD +++ b/util-slf4j-api/src/test/resources/BUILD @@ -1,3 +1,4 @@ resources( sources = ["*"], + tags = ["bazel-compatible"], ) diff --git a/util-slf4j-api/src/test/scala/BUILD b/util-slf4j-api/src/test/scala/BUILD index c9645b170e..266de50be1 100644 --- a/util-slf4j-api/src/test/scala/BUILD +++ b/util-slf4j-api/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-slf4j-api/src/test/scala/com/twitter/util/logging", ], diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index 7bcb2605b0..380616d302 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -5,6 +5,7 @@ scala_library( "*.java", "*.scala", ], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/slf4j:slf4j-api", "util/util-core/src/main/scala/com/twitter/io", @@ -21,7 +22,10 @@ scala_library( junit_tests( compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ ":testutil", "3rdparty/jvm/org/mockito:mockito-scala", diff --git a/util-slf4j-jul-bridge/BUILD b/util-slf4j-jul-bridge/BUILD index db92890fe8..41a89e4ae0 100644 --- a/util-slf4j-jul-bridge/BUILD +++ b/util-slf4j-jul-bridge/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-slf4j-jul-bridge/src/main/scala", ], diff --git a/util-slf4j-jul-bridge/src/main/scala/BUILD b/util-slf4j-jul-bridge/src/main/scala/BUILD index aed7bcd15b..92db1a6284 100644 --- a/util-slf4j-jul-bridge/src/main/scala/BUILD +++ b/util-slf4j-jul-bridge/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging", ], diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD index 59aee29399..9d76fc524f 100644 --- a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-slf4j-jul-bridge", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/slf4j:jul-to-slf4j", "3rdparty/jvm/org/slf4j:slf4j-api", diff --git a/util-stats/BUILD b/util-stats/BUILD index a266afa24b..c5ae2b4be4 100644 --- a/util-stats/BUILD +++ b/util-stats/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-stats/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-stats/src/test/java", "util/util-stats/src/test/scala", diff --git a/util-stats/src/main/scala/BUILD b/util-stats/src/main/scala/BUILD index 23c6fb4ad5..fcd1b8ddc2 100644 --- a/util-stats/src/main/scala/BUILD +++ b/util-stats/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD index fbd53de60c..188e68f068 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-stats", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson:jackson-module-scala", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", diff --git a/util-stats/src/test/java/BUILD b/util-stats/src/test/java/BUILD index 00d649f2f5..b309d5dbda 100644 --- a/util-stats/src/test/java/BUILD +++ b/util-stats/src/test/java/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-stats/src/test/scala/BUILD b/util-stats/src/test/scala/BUILD index 62a6571513..8db981842a 100644 --- a/util-stats/src/test/scala/BUILD +++ b/util-stats/src/test/scala/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", diff --git a/util-test/BUILD b/util-test/BUILD index 90c6c818d6..2cd5045d38 100644 --- a/util-test/BUILD +++ b/util-test/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-test/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-test/src/test/scala", ], diff --git a/util-test/src/main/scala/BUILD b/util-test/src/main/scala/BUILD index 3a67919cff..fa0246117b 100644 --- a/util-test/src/main/scala/BUILD +++ b/util-test/src/main/scala/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-test", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-test/src/main/scala/com/twitter/logging", "util/util-test/src/main/scala/com/twitter/util/testing", diff --git a/util-test/src/main/scala/com/twitter/logging/BUILD b/util-test/src/main/scala/com/twitter/logging/BUILD index 3713dce38b..6214ce299b 100644 --- a/util-test/src/main/scala/com/twitter/logging/BUILD +++ b/util-test/src/main/scala/com/twitter/logging/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-test-logging", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scalatest", "util/util-logging/src/main/scala/com/twitter/logging", diff --git a/util-test/src/main/scala/com/twitter/util/testing/BUILD b/util-test/src/main/scala/com/twitter/util/testing/BUILD index 06e0d253b5..1fb374f8da 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/main/scala/com/twitter/util/testing/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-test-testing", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", diff --git a/util-test/src/test/scala/BUILD b/util-test/src/test/scala/BUILD index 2cfefa131f..4869e39f9f 100644 --- a/util-test/src/test/scala/BUILD +++ b/util-test/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-test/src/test/scala/com/twitter/util/testing", ], diff --git a/util-test/src/test/scala/com/twitter/util/testing/BUILD b/util-test/src/test/scala/com/twitter/util/testing/BUILD index e8f1d0def4..b007195ef9 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/test/scala/com/twitter/util/testing/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-thrift/BUILD b/util-thrift/BUILD index 3943e49d22..763f4e28d7 100644 --- a/util-thrift/BUILD +++ b/util-thrift/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-thrift/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-thrift/src/test/scala", ], diff --git a/util-thrift/src/main/scala/BUILD b/util-thrift/src/main/scala/BUILD index 129ea9ebbb..530a127a84 100644 --- a/util-thrift/src/main/scala/BUILD +++ b/util-thrift/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-thrift/src/main/scala/com/twitter/util", ], diff --git a/util-thrift/src/main/scala/com/twitter/util/BUILD b/util-thrift/src/main/scala/com/twitter/util/BUILD index ddb11f1ffe..af3920f510 100644 --- a/util-thrift/src/main/scala/com/twitter/util/BUILD +++ b/util-thrift/src/main/scala/com/twitter/util/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-thrift", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/org/apache/thrift:libthrift", diff --git a/util-thrift/src/test/java/com/twitter/util/BUILD b/util-thrift/src/test/java/com/twitter/util/BUILD index a86c35f9ff..e2b3c0c2c8 100644 --- a/util-thrift/src/test/java/com/twitter/util/BUILD +++ b/util-thrift/src/test/java/com/twitter/util/BUILD @@ -1,5 +1,6 @@ java_library( compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/apache/thrift:libthrift", ], diff --git a/util-thrift/src/test/scala/BUILD b/util-thrift/src/test/scala/BUILD index 29ff5c188e..f43bc4689c 100644 --- a/util-thrift/src/test/scala/BUILD +++ b/util-thrift/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-thrift/src/test/scala/com/twitter/util", ], diff --git a/util-thrift/src/test/scala/com/twitter/util/BUILD b/util-thrift/src/test/scala/com/twitter/util/BUILD index 053408cec9..ca918342ff 100644 --- a/util-thrift/src/test/scala/com/twitter/util/BUILD +++ b/util-thrift/src/test/scala/com/twitter/util/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatest", diff --git a/util-tunable/BUILD b/util-tunable/BUILD index bd6e06dc94..354c2a2f49 100644 --- a/util-tunable/BUILD +++ b/util-tunable/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-tunable/src/main/scala", ], diff --git a/util-tunable/src/main/scala/BUILD b/util-tunable/src/main/scala/BUILD index 32fc36f7d1..26c9ca7551 100644 --- a/util-tunable/src/main/scala/BUILD +++ b/util-tunable/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-tunable/src/main/scala/com/twitter/util/tunable", ], diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD index 8e00fbff68..512cc0b812 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD @@ -5,14 +5,15 @@ scala_library( name = "util-tunable", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "util/util-app/src/main/scala", - "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/concurrent", ], exports = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD index 69e32e255e..562f0c25dc 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD @@ -1,5 +1,6 @@ scala_library( name = "scala", + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", @@ -14,6 +15,7 @@ jvm_binary( name = "configuration-linter", main = "com.twitter.util.tunable.linter.ConfigurationLinter", compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ ":scala", ], diff --git a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD index eacf558a79..6a08d5c20d 100644 --- a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-tunable/src/test/resources/BUILD b/util-tunable/src/test/resources/BUILD index a001371a96..301b666c21 100644 --- a/util-tunable/src/test/resources/BUILD +++ b/util-tunable/src/test/resources/BUILD @@ -1,3 +1,4 @@ resources( sources = ["**/*"], + tags = ["bazel-compatible"], ) diff --git a/util-tunable/src/test/scala/BUILD b/util-tunable/src/test/scala/BUILD index c1b6083c5c..35a01a818a 100644 --- a/util-tunable/src/test/scala/BUILD +++ b/util-tunable/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-tunable/src/test/scala/com/twitter/util/tunable", ], diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD index 8622c6d62b..5ae4bc2cbf 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-validator/src/main/java/com/twitter/util/validation/BUILD b/util-validator/src/main/java/com/twitter/util/validation/BUILD index 5109e5e03d..d3641f85b5 100644 --- a/util-validator/src/main/java/com/twitter/util/validation/BUILD +++ b/util-validator/src/main/java/com/twitter/util/validation/BUILD @@ -7,6 +7,7 @@ java_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", ], diff --git a/util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD b/util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD index 91c29f7cd3..a18b26250b 100644 --- a/util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD +++ b/util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD @@ -7,6 +7,7 @@ java_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", ], diff --git a/util-validator/src/main/resources/BUILD b/util-validator/src/main/resources/BUILD index a1246ebdf9..5eb72c8354 100644 --- a/util-validator/src/main/resources/BUILD +++ b/util-validator/src/main/resources/BUILD @@ -3,4 +3,5 @@ resources( "**/*.properties", "**/*.xml", ], + tags = ["bazel-compatible"], ) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/BUILD index 0b473eeaa5..f5233ce28c 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/jakarta/validation:jakarta.validation-api", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD index d2a3af5b5b..9882de32e7 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD index 0662ef0e36..36a3c73482 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD index 9cf37ac72a..59cb1b93d1 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD index 98f85d2879..5b7dbf9347 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD index e267af8e30..b40328c9c4 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "util/util-validator/src/main/scala/com/twitter/util/validation/metadata", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD index ff39c6a888..c342934833 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD index 1479d2195f..cb8fc3ca3f 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD index d92c75574c..7095a3213b 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD index 7d135e23c6..da48717d39 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD index d0354352bf..b4f340e2e6 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD index 10a0d985d5..e690dffe82 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD index e13c3e5655..3dc3438a47 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD index 939df4b445..67650a7915 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/jakarta/validation:jakarta.validation-api", diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD b/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD index 0926cb69c2..883e09bd4b 100644 --- a/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD index a6c291af78..66abf2112f 100644 --- a/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD @@ -7,6 +7,7 @@ scala_library( repo = artifactory, ), strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", diff --git a/util-validator/src/test/java/com/twitter/util/validation/BUILD b/util-validator/src/test/java/com/twitter/util/validation/BUILD index 102b9f1969..de6fae8ad0 100644 --- a/util-validator/src/test/java/com/twitter/util/validation/BUILD +++ b/util-validator/src/test/java/com/twitter/util/validation/BUILD @@ -5,6 +5,7 @@ java_library( ], compiler_option_sets = ["fatal_warnings"], strict_deps = True, + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", ], diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD b/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD index f9b0d1d1a7..0308bb0f58 100644 --- a/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD +++ b/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD @@ -1,5 +1,6 @@ junit_tests( sources = ["**/*.java"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/junit", diff --git a/util-validator/src/test/scala/com/twitter/util/validation/BUILD b/util-validator/src/test/scala/com/twitter/util/validation/BUILD index 7e6f5d9030..4eef2dd742 100644 --- a/util-validator/src/test/scala/com/twitter/util/validation/BUILD +++ b/util-validator/src/test/scala/com/twitter/util/validation/BUILD @@ -4,6 +4,7 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", "3rdparty/jvm/com/github/ben-manes/caffeine", diff --git a/util-zk-test/BUILD b/util-zk-test/BUILD index 72454b3faf..36b2472230 100644 --- a/util-zk-test/BUILD +++ b/util-zk-test/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-zk-test/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-zk-test/src/test/java", "util/util-zk-test/src/test/scala", diff --git a/util-zk-test/src/main/scala/BUILD b/util-zk-test/src/main/scala/BUILD index 2743ea97b0..95e7cbf554 100644 --- a/util-zk-test/src/main/scala/BUILD +++ b/util-zk-test/src/main/scala/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-zk-test/src/main/scala/com/twitter/zk", ], diff --git a/util-zk-test/src/main/scala/com/twitter/zk/BUILD b/util-zk-test/src/main/scala/com/twitter/zk/BUILD index a85889c44f..9b856ddfbb 100644 --- a/util-zk-test/src/main/scala/com/twitter/zk/BUILD +++ b/util-zk-test/src/main/scala/com/twitter/zk/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-zk-test", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", ], diff --git a/util-zk-test/src/test/java/BUILD b/util-zk-test/src/test/java/BUILD index 3c3e474ba4..471b7c9d56 100644 --- a/util-zk-test/src/test/java/BUILD +++ b/util-zk-test/src/test/java/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", diff --git a/util-zk-test/src/test/scala/BUILD b/util-zk-test/src/test/scala/BUILD index a23c01ad33..fb9a61bba3 100644 --- a/util-zk-test/src/test/scala/BUILD +++ b/util-zk-test/src/test/scala/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", diff --git a/util-zk/BUILD b/util-zk/BUILD index e3ed022b4b..b251269408 100644 --- a/util-zk/BUILD +++ b/util-zk/BUILD @@ -1,4 +1,5 @@ target( + tags = ["bazel-compatible"], dependencies = [ "util/util-zk/src/main/scala", ], @@ -6,6 +7,7 @@ target( test_suite( name = "tests", + tags = ["bazel-compatible"], dependencies = [ "util/util-zk/src/test/scala", ], diff --git a/util-zk/src/main/scala/BUILD b/util-zk/src/main/scala/BUILD index 67043ece8a..19e04b8761 100644 --- a/util-zk/src/main/scala/BUILD +++ b/util-zk/src/main/scala/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-zk", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "util/util-zk/src/main/scala/com/twitter/zk", "util/util-zk/src/main/scala/com/twitter/zk/coordination", diff --git a/util-zk/src/main/scala/com/twitter/zk/BUILD b/util-zk/src/main/scala/com/twitter/zk/BUILD index 8bb00996d4..da966400d6 100644 --- a/util-zk/src/main/scala/com/twitter/zk/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/BUILD @@ -5,17 +5,18 @@ scala_library( name = "util-zk-zk", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", + "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core:util-core-util", "util/util-logging/src/main/scala/com/twitter/logging", ], exports = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", + "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", - "util/util-core:util-core-util", ], ) diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD index 470902026b..8c82b86b7f 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD @@ -5,16 +5,17 @@ scala_library( name = "util-zk-coordination", repo = artifactory, ), + tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", - "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-zk/src/main/scala/com/twitter/zk", ], exports = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", - "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-zk/src/main/scala/com/twitter/zk", ], ) diff --git a/util-zk/src/test/scala/BUILD b/util-zk/src/test/scala/BUILD index eb0e0371a3..9ef44003ba 100644 --- a/util-zk/src/test/scala/BUILD +++ b/util-zk/src/test/scala/BUILD @@ -1,4 +1,5 @@ test_suite( + tags = ["bazel-compatible"], dependencies = [ "util/util-zk/src/test/scala/com/twitter/zk", "util/util-zk/src/test/scala/com/twitter/zk/coordination", diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index ac938e70b0..5f80696f0a 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index 3c07ce463a..ebaf0e721d 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -1,7 +1,10 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - tags = ["non-exclusive"], + tags = [ + "bazel-compatible", + "non-exclusive", + ], dependencies = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", From 23212c2aa9cc83637efaadbf63e663e21cea3776 Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 8 Jul 2021 20:46:31 +0000 Subject: [PATCH 513/796] util|finagle|finatra|twitter-server|scrooge: fix typo in CHANGELOG Problem/Solution: There is a typo in the Changelogs. The ScalaCheck version was updated, not Scalatest. Fixed the typo. Differential Revision: https://phabricator.twitter.biz/D701638 --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 77c82f2a3b..bd8d5c37fa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,7 +16,7 @@ New Features Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ -* util: Update Scalatest to version 1.15.4 ``PHAB_ID=D691691`` +* util: Update ScalaCheck to version 1.15.4 ``PHAB_ID=D691691`` 21.6.0 ------ From 5b49ae061109ff35a719398c3b7a49316aa90136 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 9 Jul 2021 00:47:19 +0000 Subject: [PATCH 514/796] {util-stats,finagle-stats-core}: Use new style of metadata to construct expressions Problem We now have the new style of using metadata, but we're not using it anywhere Solution Strip the `kernel` hack from MetricBuilder, and use the new style of grabbing metadata from concrete metric instances. Differential Revision: https://phabricator.twitter.biz/D674956 --- .../finagle/stats/InMemoryStatsReceiver.scala | 36 +--------- .../twitter/finagle/stats/MetricBuilder.scala | 39 ++++------- .../finagle/stats/exp/Expression.scala | 67 +++++++++---------- .../stats/InMemoryStatsReceiverTest.scala | 20 ++---- .../finagle/stats/MetricBuilderTest.scala | 18 ----- .../stats/exp/ExpressionSchemaTest.scala | 19 ++---- 6 files changed, 57 insertions(+), 142 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index e2cd333193..14c565495d 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -1,7 +1,6 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} -import com.twitter.finagle.stats.exp.Expression.replaceExpression import com.twitter.finagle.stats.exp.ExpressionSchema import java.io.PrintStream import java.util.Locale @@ -72,19 +71,8 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { val schemas: mutable.Map[Seq[String], MetricBuilder] = new ConcurrentHashMap[Seq[String], MetricBuilder]().asScala - private val metricBuilders: ConcurrentHashMap[Int, MetricBuilder] = - new ConcurrentHashMap[Int, MetricBuilder]() - - private val preloadExpressions: ConcurrentHashMap[String, ExpressionSchema] = - new ConcurrentHashMap[String, ExpressionSchema]() - - def expressions: mutable.Map[String, ExpressionSchema] = { - preloadExpressions.replaceAll { - case (_, exprSchema) => - exprSchema.copy(expr = replaceExpression(exprSchema.expr, metricBuilders)) - } - preloadExpressions.asScala - } + val expressions: mutable.Map[String, ExpressionSchema] = + new ConcurrentHashMap[String, ExpressionSchema]().asScala override def counter(name: String*): ReadableCounter = counter(this.metricBuilder(CounterType).withName(name: _*)) @@ -103,7 +91,6 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { if (!counters.contains(metricBuilder.name)) { counters(metricBuilder.name) = 0 schemas(metricBuilder.name) = metricBuilder - storeMetricBuilder(metricBuilder) } } @@ -138,7 +125,6 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { if (!stats.contains(metricBuilder.name)) { stats(metricBuilder.name) = Nil schemas(metricBuilder.name) = metricBuilder - storeMetricBuilder(metricBuilder) } } @@ -167,11 +153,9 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { gauges += metricBuilder.name -> (() => f) schemas += metricBuilder.name -> metricBuilder verbosity += metricBuilder.name -> metricBuilder.verbosity - storeMetricBuilder(metricBuilder) def remove(): Unit = { gauges -= metricBuilder.name - removeMetricBuilder(metricBuilder) schemas -= metricBuilder.name } @@ -188,19 +172,6 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } } - private[this] def storeMetricBuilder(metricBuilder: MetricBuilder): Unit = { - metricBuilder.kernel match { - case Some(kernel) => metricBuilders.put(kernel, metricBuilder) - case None => - } - } - - private[this] def removeMetricBuilder(metricBuilder: MetricBuilder): Unit = { - metricBuilder.kernel match { - case Some(kernel) => metricBuilders.remove(kernel) - case None => - } - } override def toString: String = "InMemoryStatsReceiver" /** @@ -264,7 +235,6 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { stats.clear() gauges.clear() schemas.clear() - preloadExpressions.clear() } private[this] def toHistogramDetail(addedValues: Seq[Float]): HistogramDetail = { @@ -297,7 +267,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { case Some(serviceName) => schema.name + "_" + serviceName case None => schema.name }) + schema.namespace.mkString("_") - preloadExpressions.put(expressionId, schema) + expressions.put(expressionId, schema) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index e708b42c22..97c68425b2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -82,7 +82,6 @@ object MetricBuilder { processPath, percentiles, isCounterishGauge, - kernel = None, metricType, statsReceiver ) @@ -97,23 +96,22 @@ object MetricBuilder { case object GaugeType extends MetricType case object HistogramType extends MetricType } -object Metadata { + +sealed trait Metadata { /** - * extract the MetricBuilder from Metadata + * Extract the MetricBuilder from Metadata + * + * Will return `None` if it's `NoMetadata` */ - def getMetricBuilder(metadata: Metadata): Option[MetricBuilder] = { - metadata match { - case metricBuilder: MetricBuilder => Some(metricBuilder) - case NoMetadata => None - case MultiMetadata(schemas) => - schemas.find(_ != NoMetadata).asInstanceOf[Option[MetricBuilder]] - } + def toMetricBuilder: Option[MetricBuilder] = this match { + case metricBuilder: MetricBuilder => Some(metricBuilder) + case NoMetadata => None + case MultiMetadata(schemas) => + schemas.find(_ != NoMetadata).flatMap(_.toMetricBuilder) } } -sealed trait Metadata - case object NoMetadata extends Metadata case class MultiMetadata(schemas: Seq[Metadata]) extends Metadata @@ -136,8 +134,6 @@ class MetricBuilder private ( // Only persisted and relevant when building histograms. val percentiles: IndexedSeq[Double], val isCounterishGauge: Boolean, - // see withKernel - val kernel: Option[Int], val metricType: MetricType, val statsReceiver: StatsReceiver) extends Metadata { @@ -159,8 +155,7 @@ class MetricBuilder private ( processPath: Option[String] = this.processPath, isCounterishGauge: Boolean = this.isCounterishGauge, percentiles: IndexedSeq[Double] = this.percentiles, - metricType: MetricType = this.metricType, - kernel: Option[Int] = this.kernel + metricType: MetricType = this.metricType ): MetricBuilder = { new MetricBuilder( keyIndicator = keyIndicator, @@ -175,20 +170,10 @@ class MetricBuilder private ( percentiles = percentiles, isCounterishGauge = isCounterishGauge, statsReceiver = this.statsReceiver, - metricType = metricType, - kernel = kernel + metricType = metricType ) } - // This is a work-around for memoizing this MetricBuilder, it is the object reference - // hashCode of `this` MetricBuilder, only explicitly configured for metrics used for expressions. - // Metric use it as the hash key to find the fully hydrated MetricBuilder. - // Copying the MetricBuilder to configure other metadata must keep the kernel untouched. - private[finagle] def withKernel: MetricBuilder = { - require(this.kernel.isEmpty, "This MetricBuilder is already hashed") - this.copy(kernel = Some(System.identityHashCode(this))) - } - def withKeyIndicator(isKeyIndicator: Boolean = true): MetricBuilder = this.copy(keyIndicator = isKeyIndicator) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index 0989cf7f59..f5b7958a58 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -2,8 +2,7 @@ package com.twitter.finagle.stats.exp import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.exp.Expression.HistogramComponent -import com.twitter.finagle.stats.{MetricBuilder, StatsReceiver} -import java.util.concurrent.ConcurrentHashMap +import com.twitter.finagle.stats.{Metadata, MetricBuilder, StatsReceiver} import scala.annotation.varargs private[twitter] object Expression { @@ -24,6 +23,7 @@ private[twitter] object Expression { case MetricExpression(metricBuilder) => Set(metricBuilder.statsReceiver) case HistogramExpression(metricBuilder, _) => Set(metricBuilder.statsReceiver) case ConstantExpression(_) => Set.empty + case NoExpression => Set.empty } /** @@ -37,58 +37,46 @@ private[twitter] object Expression { * or a percentile in Double for Right. */ def apply( - metricBuilder: MetricBuilder, + metadata: Metadata, component: Either[HistogramComponent, Double] - ): HistogramExpression = { - require( - metricBuilder.metricType == HistogramType, - "this method is for creating histogram expression") - HistogramExpression(metricBuilder, component) + ): Expression = { + metadata.toMetricBuilder match { + case Some(metricBuilder) => + require( + metricBuilder.metricType == HistogramType, + "this method is for creating histogram expression") + HistogramExpression(metricBuilder, component) + case None => + NoExpression + } } /** * Create an single expression wrapping a counter or gauge. */ - def apply(metricBuilder: MetricBuilder): Expression = { - require(metricBuilder.metricType != HistogramType, "provide a component for histogram") - MetricExpression(metricBuilder) - } - - // utility methods shared by Metrics.scala and InMemoryStatsReceiver - private[stats] def replaceExpression( - expression: Expression, - metricBuilders: ConcurrentHashMap[Int, MetricBuilder] - ): Expression = { - expression match { - case f @ FunctionExpression(_, exprs) => - f.copy(exprs = exprs.map(replaceExpression(_, metricBuilders))) - case m @ MetricExpression(metricBuilder) if metricBuilder.kernel.isDefined => - val builder = metricBuilders.get(metricBuilder.kernel.get) - if (builder != null) m.copy(builder) - else m - case h @ HistogramExpression(metricBuilder, _) if metricBuilder.kernel.isDefined => - val builder = metricBuilders.get(metricBuilder.kernel.get) - if (builder != null) h.copy(builder) - else h - case otherExpression => otherExpression + def apply(metadata: Metadata): Expression = { + metadata.toMetricBuilder match { + case Some(metricBuilder) => + require(metricBuilder.metricType != HistogramType, "provide a component for histogram") + MetricExpression(metricBuilder) + case None => + NoExpression } } - - private[this] def reformSchema(builder: MetricBuilder): MetricBuilder = builder } /** * Metrics with their arithmetical(or others) calculations */ private[twitter] sealed trait Expression { - def plus(other: Expression): Expression = FunctionExpression("plus", Seq(this, other)) + final def plus(other: Expression): Expression = func("plus", other) - def minus(other: Expression): Expression = FunctionExpression("minus", Seq(this, other)) + final def minus(other: Expression): Expression = func("minus", other) - def divide(other: Expression): Expression = FunctionExpression("divide", Seq(this, other)) + final def divide(other: Expression): Expression = func("divide", other) - def multiply(other: Expression): Expression = - FunctionExpression("multiply", Seq(this, other)) + final def multiply(other: Expression): Expression = + func("multiply", other) @varargs def func(name: String, rest: Expression*): Expression = @@ -120,3 +108,8 @@ case class HistogramExpression private ( metricBuilder: MetricBuilder, component: Either[HistogramComponent, Double]) extends Expression + +case object NoExpression extends Expression { + @varargs + override def func(name: String, rest: Expression*): Expression = NoExpression +} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 9bfcea9a3d..50f1af0b49 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -367,24 +367,18 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra } } - test("expression are reloaded with fully scoped names") { + test("expressions can be hydrated from metadata that metrics use") { val sr = new InMemoryStatsReceiver - val aSchema = - MetricBuilder(name = Seq("a"), metricType = CounterType, statsReceiver = sr).withKernel - val bSchema = - MetricBuilder(name = Seq("b"), metricType = HistogramType, statsReceiver = sr).withKernel - val cSchema = - MetricBuilder(name = Seq("c"), metricType = GaugeType, statsReceiver = sr).withKernel + val aCounter = sr.scope("test").counter("a") + val bHisto = sr.scope("test").stat("b") + val cGauge = sr.scope("test").addGauge("c") { 1 } val expression = ExpressionSchema( "test_expression", - Expression(aSchema).plus(Expression(bSchema, Left(Expression.Min)).plus(Expression(cSchema)))) - .register() - - val aCounter = sr.scope("test").counter(aSchema) - val bHisto = sr.scope("test").stat(bSchema) - val cGauge = sr.scope(("test")).addGauge(cSchema) { 1 } + Expression(aCounter.metadata).plus(Expression(bHisto.metadata, Left(Expression.Min)) + .plus(Expression(cGauge.metadata))) + ).register() // what we expected as hydrated metric builders val aaSchema = diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala index 6a5e4bfd52..4c29c73e87 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala @@ -16,22 +16,4 @@ class MetricBuilderTest extends AnyFunSuite { metadataScopeSeparator.setSeparator("_aTerriblyLongStringForSomeReason_") assert(mb.toString.contains("foo_aTerriblyLongStringForSomeReason_bar")) } - - test("kernels are object reference hashCode") { - val sr = new InMemoryStatsReceiver - val mb1 = - MetricBuilder(name = Seq("a"), metricType = CounterType, statsReceiver = sr).withKernel - val mb2 = - MetricBuilder(name = Seq("a"), metricType = CounterType, statsReceiver = sr).withKernel - assert(mb2.kernel != mb1.kernel) - } - - test("kernels remain the same after copy") { - val mb1 = MetricBuilder( - name = Seq("a"), - metricType = CounterType, - statsReceiver = new InMemoryStatsReceiver).withKernel - val mb2 = mb1.withDescription("I am a description").withName("a", "b") - assert(mb1.kernel == mb2.kernel) - } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 2db57e6599..1dad70b3ed 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -13,20 +13,11 @@ class ExpressionSchemaTest extends AnyFunSuite { val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) val successMb = - MetricBuilder( - name = Seq("success"), - metricType = CounterType, - statsReceiver = clientSR).withKernel + MetricBuilder(name = Seq("success"), metricType = CounterType, statsReceiver = clientSR) val failuresMb = - MetricBuilder( - name = Seq("failures"), - metricType = CounterType, - statsReceiver = clientSR).withKernel + MetricBuilder(name = Seq("failures"), metricType = CounterType, statsReceiver = clientSR) val latencyMb = - MetricBuilder( - name = Seq("latency"), - metricType = HistogramType, - statsReceiver = clientSR).withKernel + MetricBuilder(name = Seq("latency"), metricType = HistogramType, statsReceiver = clientSR) val sum = Expression(successMb).plus(Expression(failuresMb)) def slowQuery(ctl: TimeControl, succeed: Boolean): Unit = { @@ -73,9 +64,9 @@ class ExpressionSchemaTest extends AnyFunSuite { runTheQuery(false) assert(sr.expressions("success_rate_downstream").name == successRate.name) - assert(sr.expressions("success_rate_downstream").expr != successRate.expr) + assert(sr.expressions("success_rate_downstream").expr == successRate.expr) assert(sr.expressions("latency_downstream").name == latency.name) - assert(sr.expressions("latency_downstream").expr != latency.expr) + assert(sr.expressions("latency_downstream").expr == latency.expr) assert(sr.counters(Seq("success")) == 1) assert(sr.counters(Seq("failures")) == 1) From 31849459c9987fd3c457691b385eed295a7e2d48 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 9 Jul 2021 19:29:53 +0000 Subject: [PATCH 515/796] util: make CounterishGauge its own MetricType Problem/Solution: We currently track if a metric is a counter-ish gauge via a boolean isCounterishGauge. Let's instead treat it as its own metric type. JIRA Issues: CSL-11006 Differential Revision: https://phabricator.twitter.biz/D700353 --- .../twitter/finagle/stats/MetricBuilder.scala | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 97c68425b2..2730222a8f 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -1,6 +1,12 @@ package com.twitter.finagle.stats -import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType, MetricType} +import com.twitter.finagle.stats.MetricBuilder.{ + CounterType, + CounterishGaugeType, + GaugeType, + HistogramType, + MetricType +} import java.util.function.Supplier import scala.annotation.varargs @@ -52,7 +58,6 @@ object MetricBuilder { * @param relativeName the relative metric name which will be appended to the scope of the StatsReceiver prior to long term storage * @param processPath a universal coordinate for the resource * @param percentiles used to indicate buckets for histograms, to be set by the StatsReceiver - * @param isCounterishGauge used to indicate that a gauge is being used to model something that acts like a counter * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder */ def apply( @@ -66,7 +71,6 @@ object MetricBuilder { relativeName: Seq[String] = Seq.empty, processPath: Option[String] = None, percentiles: IndexedSeq[Double] = IndexedSeq.empty, - isCounterishGauge: Boolean = false, metricType: MetricType, statsReceiver: StatsReceiver ): MetricBuilder = { @@ -81,7 +85,6 @@ object MetricBuilder { relativeName, processPath, percentiles, - isCounterishGauge, metricType, statsReceiver ) @@ -89,10 +92,14 @@ object MetricBuilder { /** * Indicate the Metric type, [[CounterType]] will create a [[Counter]], - * [[GaugeType]] will create a [[Gauge]], and [[HistogramType]] will create a [[Stat]]. + * [[GaugeType]] will create a standard [[Gauge]], and [[HistogramType]] will create a [[Stat]]. + * + * @note [[CounterishGaugeType]] will also create a [[Gauge]], but specifically one that + * models a [[Counter]]. */ sealed trait MetricType case object CounterType extends MetricType + case object CounterishGaugeType extends MetricType case object GaugeType extends MetricType case object HistogramType extends MetricType } @@ -133,7 +140,6 @@ class MetricBuilder private ( val processPath: Option[String], // Only persisted and relevant when building histograms. val percentiles: IndexedSeq[Double], - val isCounterishGauge: Boolean, val metricType: MetricType, val statsReceiver: StatsReceiver) extends Metadata { @@ -153,7 +159,6 @@ class MetricBuilder private ( name: Seq[String] = this.name, relativeName: Seq[String] = this.relativeName, processPath: Option[String] = this.processPath, - isCounterishGauge: Boolean = this.isCounterishGauge, percentiles: IndexedSeq[Double] = this.percentiles, metricType: MetricType = this.metricType ): MetricBuilder = { @@ -168,7 +173,6 @@ class MetricBuilder private ( relativeName = relativeName, processPath = processPath, percentiles = percentiles, - isCounterishGauge = isCounterishGauge, statsReceiver = this.statsReceiver, metricType = metricType ) @@ -202,9 +206,12 @@ class MetricBuilder private ( def withPercentiles(percentiles: Double*): MetricBuilder = this.copy(percentiles = percentiles.toIndexedSeq) - def withCounterishGauge: MetricBuilder = this.copy(isCounterishGauge = true) - - def withNoCounterishGauge: MetricBuilder = this.copy(isCounterishGauge = false) + def withCounterishGauge: MetricBuilder = { + require( + this.metricType == GaugeType, + "Cannot create a CounterishGauge from a Counter or Histogram") + this.copy(metricType = CounterishGaugeType) + } /** * Generates a CounterType MetricBuilder which can be used to create a counter in a StatsReceiver. @@ -229,6 +236,14 @@ class MetricBuilder private ( private[MetricBuilder] final def histogramSchema: MetricBuilder = this.copy(metricType = HistogramType) + /** + * Generates a CounterishGaugeType MetricBuilder which can be used to create a counter-ish gauge + * in a StatsReceiver. Used to test that builder class correctly propagates configured metadata. + * @return a CounterishGaugeType MetricBuilder. + */ + private[MetricBuilder] final def counterishGaugeSchema: MetricBuilder = + this.copy(metricType = CounterishGaugeType) + /** * Produce a counter as described by the builder inside the underlying StatsReceiver. * @return the counter created. From 5377fd374d2ebcb2c8a974f9945865a62a7b97fa Mon Sep 17 00:00:00 2001 From: Adam Singer Date: Fri, 16 Jul 2021 01:08:43 +0000 Subject: [PATCH 516/796] [util-core] Support files under 3 characters for TempFile Problem `File.createTempFile` does not support a `prefix` less than 3 characters. Solution Update `c.t.i.TempFile` to append "tmp" to a basename if it is less than 3 characters. JIRA Issues: DPB-12532 Differential Revision: https://phabricator.twitter.biz/D705650 --- util-core/src/main/scala/com/twitter/io/TempFile.scala | 7 ++++++- util-core/src/test/resources/1.txt | 1 + util-core/src/test/scala/com/twitter/io/TempFileTest.scala | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 util-core/src/test/resources/1.txt diff --git a/util-core/src/main/scala/com/twitter/io/TempFile.scala b/util-core/src/main/scala/com/twitter/io/TempFile.scala index b206261837..476d995e83 100644 --- a/util-core/src/main/scala/com/twitter/io/TempFile.scala +++ b/util-core/src/main/scala/com/twitter/io/TempFile.scala @@ -55,7 +55,12 @@ object TempFile { case null => throw new FileNotFoundException(path) case stream => - val (basename, ext) = parsePath(path) + val (basename, ext) = parsePath(path) match { + // The prefix parameter of File.createTempFile(prefix,suffix) is + // expected to be 3 characters long. + case (basename, ext) if basename.length < 3 => (basename + "tmp", ext) + case basenameExt => basenameExt + } val file = File.createTempFile(basename, "." + ext) file.deleteOnExit() val fos = new BufferedOutputStream(new FileOutputStream(file), 1 << 20) diff --git a/util-core/src/test/resources/1.txt b/util-core/src/test/resources/1.txt new file mode 100644 index 0000000000..dc8344c1ce --- /dev/null +++ b/util-core/src/test/resources/1.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet diff --git a/util-core/src/test/scala/com/twitter/io/TempFileTest.scala b/util-core/src/test/scala/com/twitter/io/TempFileTest.scala index e365ff3f10..e4bb3da4d2 100644 --- a/util-core/src/test/scala/com/twitter/io/TempFileTest.scala +++ b/util-core/src/test/scala/com/twitter/io/TempFileTest.scala @@ -1,6 +1,7 @@ package com.twitter.io import java.io.{ByteArrayInputStream, DataInputStream} +import java.nio.charset.StandardCharsets import java.util.Arrays import org.scalatest.wordspec.AnyWordSpec @@ -12,14 +13,18 @@ class TempFileTest extends AnyWordSpec { val f1 = TempFile.fromResourcePath("/java/lang/String.class") val f2 = TempFile.fromResourcePath(getClass, "/java/lang/String.class") val f3 = TempFile.fromSystemResourcePath("java/lang/String.class") + // Tests the basename allows basename to be less than 3 characters. + val f4 = TempFile.fromResourcePath("/1.txt") val c1 = Files.readBytes(f1) val c2 = Files.readBytes(f2) val c3 = Files.readBytes(f3) + val c4 = Files.readBytes(f4) assert(Arrays.equals(c1, c2)) assert(Arrays.equals(c2, c3)) assert(new DataInputStream(new ByteArrayInputStream(c1)).readInt == 0xcafebabe) + assert(new String(c4, StandardCharsets.UTF_8) == "Lorem ipsum dolor sit amet\n") } } From df4321c99dca592cfb1ddb1b7070d6496a048129 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Mon, 19 Jul 2021 19:53:47 +0000 Subject: [PATCH 517/796] finagle|finatra|scrooge|twitter-server|util: 3rd party upgrade cadence note Problem/Solution We agreed to upgrade Netty, Guice, Jackson, Caffeine, & sbt on a quarterly cadence. We should communicate this in our `CONTRIBUTING.md` docs. JIRA Issues: CSL-11166 Differential Revision: https://phabricator.twitter.biz/D704594 --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 136c2d41a4..8edfa2a93e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,13 @@ If you have any questions or run into any problems, please create an issue here, chat with us in [gitter](https://gitter.im/twitter/finagle), or email the Finaglers [mailing list](https://groups.google.com/forum/#!forum/finaglers). +### 3rd party upgrades + +We upgrade the following 3rd party libraries/tools at least once every 3 months: +* [sbt](https://github.com/sbt/sbt) +* [Jackson](https://github.com/FasterXML/jackson) +* [Caffeine](https://github.com/ben-manes/caffeine) + ## Workflow The workflow that we support: From 14709f4c17d71a3bd4fd9941b8418b876e9324bc Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 20 Jul 2021 01:13:00 +0000 Subject: [PATCH 518/796] finagle-core: Standard Service Metrics for thrift services Problem Currently, top-line service metrics include customized name snippets, potentially different counter types, and different separators, causing fleet queries with many challenges. Goals 1. Use internally consistent patterns. 2. Use forward slashes to separate scopes. 3. Not use latching / counter deltas. 4. Use ThriftExceptionsAsFailures to incorporate app-level failures. Solution 1. The StandardStatsReceiver is private and can only be scoped/named in the library, there are no customized labels taken in place. 2. Use a hard-coded separator for standard metrics. 3. Introduce a new MetricType - UnlatchedCounter for standard counters, need to collaborate with MetricsBackend to correctly aggregate these counters. 4. Inject the StandardStatsReceiver and a ResponseClassifier into the server stack parameters. Result Using a different ResponseClassifier (for thrift services that are not configured with it) determines that we need to have a set of new metrics instead of a StatsReceiver with a special Broadcast Counter/Stat/Gauge. Added overall RPC metrics that aggregate all servers' metrics in an application. Since Http service uses its own StatsFilter, we will address it in a separate revision. JIRA Issues: CSL-11128 Differential Revision: https://phabricator.twitter.biz/D697720 --- .../finagle/stats/CumulativeGauge.scala | 4 ++-- .../twitter/finagle/stats/MetricBuilder.scala | 20 ++++++++++++++++++- .../RelativeNameMarkingStatsReceiver.scala | 2 +- .../twitter/finagle/stats/StatsReceiver.scala | 15 ++++++++++++-- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala index 28bdce7871..dce44e1605 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CumulativeGauge.scala @@ -122,7 +122,7 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => * gauges. */ protected[this] def registerGauge(metricBuilder: MetricBuilder, f: => Float): Unit - protected[this] def deregisterGauge(name: Seq[String]): Unit + protected[this] def deregisterGauge(metricBuilder: MetricBuilder): Unit private[this] def getWhenNotPresent(metricBuilder: MetricBuilder) = whenNotPresent(metricBuilder) @@ -156,7 +156,7 @@ trait StatsReceiverWithCumulativeGauges extends StatsReceiver { self => } else { // Do the cleanup. gauges.remove(key) - self.deregisterGauge(key) + self.deregisterGauge(metricBuilder) } case c => diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 2730222a8f..764823b6e7 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -5,7 +5,8 @@ import com.twitter.finagle.stats.MetricBuilder.{ CounterishGaugeType, GaugeType, HistogramType, - MetricType + MetricType, + UnlatchedCounter } import java.util.function.Supplier import scala.annotation.varargs @@ -71,6 +72,7 @@ object MetricBuilder { relativeName: Seq[String] = Seq.empty, processPath: Option[String] = None, percentiles: IndexedSeq[Double] = IndexedSeq.empty, + isStandard: Boolean = false, metricType: MetricType, statsReceiver: StatsReceiver ): MetricBuilder = { @@ -85,6 +87,7 @@ object MetricBuilder { relativeName, processPath, percentiles, + isStandard, metricType, statsReceiver ) @@ -102,6 +105,9 @@ object MetricBuilder { case object CounterishGaugeType extends MetricType case object GaugeType extends MetricType case object HistogramType extends MetricType + + /** Counter type that will be always unlatched */ + case object UnlatchedCounter extends MetricType } sealed trait Metadata { @@ -140,6 +146,7 @@ class MetricBuilder private ( val processPath: Option[String], // Only persisted and relevant when building histograms. val percentiles: IndexedSeq[Double], + val isStandard: Boolean, val metricType: MetricType, val statsReceiver: StatsReceiver) extends Metadata { @@ -159,6 +166,7 @@ class MetricBuilder private ( name: Seq[String] = this.name, relativeName: Seq[String] = this.relativeName, processPath: Option[String] = this.processPath, + isStandard: Boolean = this.isStandard, percentiles: IndexedSeq[Double] = this.percentiles, metricType: MetricType = this.metricType ): MetricBuilder = { @@ -173,11 +181,21 @@ class MetricBuilder private ( relativeName = relativeName, processPath = processPath, percentiles = percentiles, + isStandard = isStandard, statsReceiver = this.statsReceiver, metricType = metricType ) } + private[finagle] def withStandard: MetricBuilder = { + this.copy(isStandard = true) + } + + private[finagle] def withUnlatchedCounter: MetricBuilder = { + require(this.metricType == CounterType, "unable to set a non counter to unlatched counter") + this.copy(metricType = UnlatchedCounter) + } + def withKeyIndicator(isKeyIndicator: Boolean = true): MetricBuilder = this.copy(keyIndicator = isKeyIndicator) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala index b32b4ed849..0f7bbe2102 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RelativeNameMarkingStatsReceiver.scala @@ -6,7 +6,7 @@ package com.twitter.finagle.stats * * @param self The underlying StatsReceiver to which modified schemas are passed */ -class RelativeNameMarkingStatsReceiver( +case class RelativeNameMarkingStatsReceiver( protected val self: StatsReceiver) extends StatsReceiverProxy { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 42e28875a2..bb5f08cd3f 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -1,6 +1,12 @@ package com.twitter.finagle.stats -import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType, MetricType} +import com.twitter.finagle.stats.MetricBuilder.{ + CounterType, + GaugeType, + HistogramType, + MetricType, + UnlatchedCounter +} import com.twitter.finagle.stats.exp.ExpressionSchema import java.lang.{Float => JFloat} import java.util.concurrent.Callable @@ -344,8 +350,13 @@ trait StatsReceiver { private[stats] def stat0(name: String): Stat = stat(name) protected def validateMetricType(metricBuilder: MetricBuilder, expectedType: MetricType): Unit = { + val typeMatch = expectedType match { + case CounterType => + metricBuilder.metricType == CounterType || metricBuilder.metricType == UnlatchedCounter + case _ => metricBuilder.metricType == expectedType + } require( - metricBuilder.metricType == expectedType, + typeMatch, s"creating a $expectedType using wrong MetricBuilder: ${metricBuilder.metricType}") } } From 680d39cae0c890ab7e4c2531998dad9307b346b5 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 20 Jul 2021 23:20:02 +0000 Subject: [PATCH 519/796] util-stats: StatsReceiver accepts counterish_gauge Problem counterish_gauge type can't pass through the validation when recording metrics. Solution Add the case. Differential Revision: https://phabricator.twitter.biz/D708822 --- .../twitter/finagle/stats/StatsReceiver.scala | 3 ++ .../finagle/stats/StatsReceiverTest.scala | 29 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index bb5f08cd3f..c917dbf01a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -2,6 +2,7 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.MetricBuilder.{ CounterType, + CounterishGaugeType, GaugeType, HistogramType, MetricType, @@ -353,6 +354,8 @@ trait StatsReceiver { val typeMatch = expectedType match { case CounterType => metricBuilder.metricType == CounterType || metricBuilder.metricType == UnlatchedCounter + case GaugeType => + metricBuilder.metricType == GaugeType || metricBuilder.metricType == CounterishGaugeType case _ => metricBuilder.metricType == expectedType } require( diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index a1e7f7ccea..c9144415a7 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -1,11 +1,18 @@ package com.twitter.finagle.stats import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.stats.MetricBuilder.{ + CounterType, + CounterishGaugeType, + GaugeType, + HistogramType, + UnlatchedCounter +} import com.twitter.util.{Await, Future} import java.util.concurrent.TimeUnit import org.mockito.Mockito._ -import scala.collection.mutable.ArrayBuffer import org.scalatest.funsuite.AnyFunSuite +import scala.collection.mutable.ArrayBuffer class StatsReceiverTest extends AnyFunSuite { test("RollupStatsReceiver counter/stats") { @@ -177,4 +184,24 @@ class StatsReceiverTest extends AnyFunSuite { } + test("StatsReceiver validate and record metrics") { + val sr = new InMemoryStatsReceiver() + val counter = MetricBuilder(name = Seq("a"), metricType = CounterType, statsReceiver = sr) + val counterishGauge = + MetricBuilder(name = Seq("b"), metricType = CounterishGaugeType, statsReceiver = sr) + val gauge = MetricBuilder(name = Seq("c"), metricType = GaugeType, statsReceiver = sr) + val stat = MetricBuilder(name = Seq("d"), metricType = HistogramType, statsReceiver = sr) + val unlatchedCounter = + MetricBuilder(name = Seq("e"), metricType = UnlatchedCounter, statsReceiver = sr) + + sr.addGauge(gauge)(1) + sr.addGauge(counterishGauge)(1) + sr.counter(counter) + sr.counter(unlatchedCounter) + sr.stat(stat) + + intercept[IllegalArgumentException] { + sr.counter(counterishGauge) + } + } } From cd03cf0df51de25ccbe442e9ffb5a602ce892025 Mon Sep 17 00:00:00 2001 From: Michael Doherty Date: Tue, 20 Jul 2021 23:37:49 +0000 Subject: [PATCH 520/796] util-jackson: Include null values in JsonDiff#toSortedString Problem `JsonDiff#assertDiff` writes a report that is meant to show where JSON objects differ. It shows each object using `toSortedString` so that key ordering is insignificant in the report. But `#toSortedString` currently filters null values. The result is that when objects differ only in extra keys with null values, the `JsonDiff#assertDiff` report shows identical string representations and fails to show where the objects differ. Solution Include null values in the `#toSortedString` representation. Differential Revision: https://phabricator.twitter.biz/D707033 --- CHANGELOG.rst | 7 +++ doc/src/sphinx/util-jackson/index.rst | 42 +++++++++++++--- .../com/twitter/util/jackson/JsonDiff.scala | 4 +- .../twitter/util/jackson/JsonDiffTest.scala | 49 ++++++++++++++++++- 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bd8d5c37fa..6cca8476f8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,13 @@ Runtime Behavior Changes * util: Update ScalaCheck to version 1.15.4 ``PHAB_ID=D691691`` + +Bug Fixes +~~~~~~~~~ + +* util-jackson: `JsonDiff#toSortedString` now includes null-type nodes, so that + `JsonDiff.Result#toString` shows differences in objects due to such nodes. ``PHAB_ID=D707033`` + 21.6.0 ------ diff --git a/doc/src/sphinx/util-jackson/index.rst b/doc/src/sphinx/util-jackson/index.rst index 8d70f01f83..1ec7207db6 100644 --- a/doc/src/sphinx/util-jackson/index.rst +++ b/doc/src/sphinx/util-jackson/index.rst @@ -903,16 +903,18 @@ configured with a `YAMLFactory`. `c.t.util.jackson.JsonDiff` --------------------------- -The library provides a utility for comparing JSON strings or structures which can be serialized as -JSON strings via the `c.t.util.jackson.JsonDiff `__ utility. +The library provides a utility for comparing JSON strings, or structures that can be serialized as +JSON strings, via the `c.t.util.jackson.JsonDiff `__ utility. `JsonDiff` provides two functions: `diff` and `assertDiff`. The `diff` method allows the user to decide how to handle JSON differences by returning an `Option[JsonDiff.Result]` while `assertDiff` throws an `AssertionError` when a difference is encountered. -The `JsonDiff.Result#toString` contains a textual representation meant to show when both the -`expected` and `actual` JSON strings are sorted lexicographically where the first encountered -difference occurs. When an `AssertError` is thrown, the `JsonDiff.Result#toString` is used to +The `JsonDiff.Result#toString` contains a textual representation meant to indicate where the +`expected` and `actual` differ semantically. For this representation, both `expected` and `actual` +are transformed to eliminate insigificant lexical diffences such whitespace, object key ordering, +and escape sequences. Only the first difference in this representation is indicated. +When an `AssertError` is thrown, the `JsonDiff.Result#toString` is used to populate the exception message. For example: @@ -983,10 +985,34 @@ For example: res0: Boolean = true scala> t.throwable.getMessage - res2: String = - " * + res1: String = + com.twitter.util.jackson.JsonDiff$ failure + * Expected: {"a":1,"b":2} - Actual: {"a":1,"b":3}" + Actual: {"a":1,"b":3} + + scala> val expected = """{"t1": "24\u00B0C"}""" + expected: String = {"t1": "24\u00B0C"} + + scala> val actual = """{"t1": "24°F", "t2": null}""" + actual: String = {"t1": "24°F", "t2": null} + + scala> val t = Try(JsonDiff.assertDiff(expected, actual)) // throws an AssertionError + JSON DIFF FAILED! + * + Expected: {"t1":"24°C"} + Actual: {"t1":"24°F","t2":null} + + scala> t.isThrow + res0: Boolean = true + + scala> t.throwable.getMessage + res1: String = + com.twitter.util.jackson.JsonDiff$ failure + * + Expected: {"t1":"24°C"} + Actual: {"t1":"24°F","t2":null} + Normalization ~~~~~~~~~~~~~ diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/JsonDiff.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/JsonDiff.scala index f2e8081272..cf2cf9edd5 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/JsonDiff.scala +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/JsonDiff.scala @@ -1,7 +1,8 @@ package com.twitter.util.jackson -import com.fasterxml.jackson.databind.{JsonNode, ObjectMapperCopier, SerializationFeature} +import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.databind.node.TextNode +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapperCopier, SerializationFeature} import com.twitter.util.logging.Logging import java.io.PrintStream import scala.util.control.NonFatal @@ -14,6 +15,7 @@ object JsonDiff extends Logging { private[this] lazy val sortingObjectMapper: JacksonScalaObjectMapperType = { val newMapper = ObjectMapperCopier.copy(mapper.underlying) newMapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + newMapper.setDefaultPropertyInclusion(Include.ALWAYS) newMapper } diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/JsonDiffTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/JsonDiffTest.scala index 317a9985b6..61e75f4a84 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/JsonDiffTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/JsonDiffTest.scala @@ -248,9 +248,56 @@ class JsonDiffTest } } - test("JsonDiff#generate sorted") { + // (the arbitrary tests above do not generate tests for array depth 1) + + test("JsonDiff#diff array depth 1 fail") { + val expected = """["a","b"]""" + val actual = """["b","a"]""" + JsonDiff.diff(expected, actual).isEmpty should be(false) + } + + test("JsonDiff#diff array depth 1 pass") { + val expected = """["a","b"]""" + val actual = """["a","b"]""" + JsonDiff.diff(expected, actual).isEmpty should be(true) + } + + // (the arbitrary tests above do not generate tests with null values or escapes) + + test("JsonDiff#diff with null values - fail") { + val expected = """{"a":null}""" + val actual = "{}" + JsonDiff.diff(expected, actual).isEmpty should be(false) + } + + test("JsonDiff#diff with null values - pass") { + val expected = """{"a":null}""" + val actual = """{"a": null}""" + JsonDiff.diff(expected, actual).isEmpty should be(true) + } + + test("JsonDiff#diff with unicode escape - pass") { + val expected = """{"t1": "24\u00B0F"}""" + val actual = """{"t1": "24°F"}""" + JsonDiff.diff(expected, actual).isEmpty should be(true) + } + + test("JsonDiff#toSortedString is sorted") { val before = mapper.parse[JsonNode]("""{"a":1,"c":3,"b":2}""") val expected = """{"a":1,"b":2,"c":3}""" JsonDiff.toSortedString(before) should equal(expected) } + + test("JsonDiff#toSortedString doesn't sort arrays") { + val before = mapper.parse[JsonNode]("""["b","a"]""") + val expected = """["b","a"]""" + JsonDiff.toSortedString(before) should equal(expected) + } + + test("JsonDiff#toSortedString includes null values") { + val before = mapper.parse[JsonNode]("""{"a":null}""") + val expected = """{"a":null}""" + JsonDiff.toSortedString(before) should equal(expected) + } + } From d188eb37ed3cc6f2d1fc150dc8fd73ff1b178fc6 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 28 Jul 2021 18:39:25 +0000 Subject: [PATCH 521/796] oss/github: Use codecov github action Problem/Solution Instead of curling and executing a script, integrating the codecov recommended github action. https://github.com/codecov/codecov-action JIRA Issues: CSL-11181 Differential Revision: https://phabricator.twitter.biz/D710862 --- .github/workflows/continuous-integration.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index cff19eeb5c..eab6daf38c 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -53,8 +53,6 @@ jobs: - name: test run: ${{ format('./sbt ++{0} clean coverage test coverageReport', matrix.scala) }} - name: coverage - if: ${{ success() }} run: ${{ format('./sbt ++{0} coverageAggregate', matrix.scala) }} - name: upload coverage - if: ${{ success() }} - run: unset TRAVIS; bash <(curl -s https://codecov.io/bash) + uses: codecov/codecov-action@v2 From d84bfbaaef6cea6db889a249bf756cc31f62c124 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Wed, 28 Jul 2021 21:04:58 +0000 Subject: [PATCH 522/796] util/util-core: scala 3 migration **Problem** Build util-core with Scala 3 **Solution** We first 1. util-core project settings. 2. Use Scala 3 migration flags 3. Fix incompatibilities Note: We keep the migration flags for now until we fix all warnings (mainly syntax) with the automatic compiler rewrite in a follow up diff. **Result** Code changes were made in. The area's the code changes were made in provide comments linking tickets and github issues. 1. **AsyncStream** Remove `Cons.unapply` due to dotty type parameter non-exhaustive pattern match with AsyncStream's covariant type (inclined to think this is a dotty bug). `Cons.unapply` is not used in source as `Cons` is an internal type. We can workaround this issue by making the constructor private and defining our own `apply`. 2. **Promise** `Promise#raise` is not a tailrec implementation. We make it tailrec by defining a private method `raiseImpl` that the dotty compiler recognizes as tail recursive 3. **Reader** Seq#hasDefiniteSize deprecations leading to `SeqUtil`. 4. **Broker** unreachable code when pattern matching on joined types (A | B). To be both Scala2 and Scala3 compatible, we can simply compare equality with the case object `Quiet` and split the cases, removing the need for a combined pattern match 5. **ClosableOnce** Java compilation error declaring `AbstractClosableOnce` was not overriding `close` even though the `CloseOnce` mixin is apart of the `ClosableOnce` trait. This is perhaps a dotty bug. For now, we can make the `CloseOnce` trait extend `Closable` instead of self typing with `Closable`. 6. **Spool** Remove `AbstractSpool`. `Spool` is a sealed trait in scala so it's not intended to be extended elsewhere. `AbstractSpool` is not java compatible in dotty due to type erasure and the overloaded `def ++` in Scala. Let's remove AbstractSpool since it shouldn't be used anyways to create Spools. 7. **Duration/Event** Redone pattern match to avoid compiler warnings around exhaustiveness 8. **StorageUnit** Numeric's `signum` is deprecated in favor of `sig`. Instead of splitting code, the implementation of `signum` is so simple, let's just define our own version until scala 2.12 support is dropped. Added test to make sure that method is tested. JIRA Issues: CSL-11082 Differential Revision: https://phabricator.twitter.biz/D694775 --- CHANGELOG.rst | 5 + build.sbt | 95 ++++++++++++++----- util-core/BUILD | 3 +- .../src/main/java/com/twitter/util/Vars.java | 4 +- .../com/twitter/conversions/SeqUtil.scala | 6 ++ .../com/twitter/conversions/SeqUtil.scala | 6 ++ .../com/twitter/concurrent/AsyncStream.scala | 27 +++--- .../scala/com/twitter/concurrent/Broker.scala | 77 ++++++++------- .../scala/com/twitter/concurrent/Spool.scala | 14 +-- .../src/main/scala/com/twitter/io/Pipe.scala | 6 +- .../main/scala/com/twitter/io/Reader.scala | 4 +- .../scala/com/twitter/util/Awaitable.scala | 9 +- .../scala/com/twitter/util/CloseOnce.scala | 7 +- .../scala/com/twitter/util/Duration.scala | 7 +- .../main/scala/com/twitter/util/Event.scala | 4 + .../main/scala/com/twitter/util/Promise.scala | 21 ++-- .../scala/com/twitter/util/StorageUnit.scala | 12 ++- .../src/main/scala/com/twitter/util/Var.scala | 27 +++--- .../concurrent/SpoolCompilationTest.java | 22 ----- .../util/AwaitableCompilationTest.java | 6 +- .../twitter/concurrent/AsyncStreamTest.scala | 3 +- .../com/twitter/concurrent/OfferTest.scala | 24 ++--- .../com/twitter/concurrent/SpoolTest.scala | 2 + .../com/twitter/conversions/SeqOpsTest.scala | 5 +- .../scala/com/twitter/io/ReaderTest.scala | 7 +- .../scala/com/twitter/util/ClosableTest.scala | 5 +- .../scala/com/twitter/util/DurationTest.scala | 4 +- .../twitter/util/LastWriteWinsQueueTest.scala | 2 +- .../scala/com/twitter/util/TimeTest.scala | 9 +- .../test/scala/com/twitter/util/VarTest.scala | 3 +- 30 files changed, 265 insertions(+), 161 deletions(-) create mode 100644 util-core/src/main/scala-2.12-/com/twitter/conversions/SeqUtil.scala create mode 100644 util-core/src/main/scala-2.13+/com/twitter/conversions/SeqUtil.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6cca8476f8..b6c3b4f7a9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,11 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-core: Now builds with Scala 3.0! ``PHAB_ID=D694775``. + +* util-core (BREAKING): Remove `AbstractSpool`. Java users should use `Spools` static class or + the Spool companion object to create instances of `Spool`. + * util-app: Introduce a new `Command` class which provides a `Reader` interface to the output of a shell command. ``PHAB_ID=D686134`` diff --git a/build.sbt b/build.sbt index b35c9723ed..d3aad3dbd6 100644 --- a/build.sbt +++ b/build.sbt @@ -80,9 +80,34 @@ def jdk11GcJavaOptions: Seq[String] = { ) } -val defaultProjectSettings = Seq( - scalaVersion := "2.13.6", - crossScalaVersions := Seq("2.12.12", "2.13.6") +val defaultScala3Settings = Seq( + scalaVersion := "3.0.1", + crossScalaVersions := Seq("2.12.12", "2.13.6", "3.0.1") +) +val defaultScala2Settings = Seq( + scalaVersion := "2.13.6", + crossScalaVersions := Seq("2.12.12", "2.13.6") +) + +// Our dependencies or compiler options may differ for both Scala 2 and 3. We branch here +// to account for there differences but should merge these artifacts as they are updated. +val scalaDependencies = Seq("org.scala-lang.modules" %% "scala-collection-compat" % "2.4.4") +val scala3Dependencies = scalaDependencies ++ Seq( + "org.scalatest" %% "scalatest" % "3.2.9" % "test", + "org.scalatestplus" %% "junit-4-13" % "3.2.9.0" % "test" +) +val scala2Dependencies = scalaDependencies ++ Seq( + "org.scalatest" %% "scalatest" % "3.1.2" % "test", + "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test" +) +val scala2cOptions = Seq( + "-target:jvm-1.8", + // Needs -missing-interpolator due to https://issues.scala-lang.org/browse/SI-8761 + "-Xlint:-missing-interpolator", + "-Yrangepos" +) +val scala3cOptions = Seq( + "-Xtarget:8" ) val baseSettings = Seq( @@ -91,36 +116,32 @@ val baseSettings = Seq( // Workaround for a scaladoc bug which causes it to choke on empty classpaths. Compile / unmanagedClasspath += Attributed.blank(new java.io.File("doesnotexist")), libraryDependencies ++= Seq( - "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.2", // See https://www.scala-sbt.org/0.13/docs/Testing.html#JUnit "com.novocode" % "junit-interface" % "0.11" % "test", - "org.scalatest" %% "scalatest" % "3.1.2" % "test", - "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test" ), Test / fork := true, // We have to fork to get the JavaOptions // Workaround for cross building HealthyQueue.scala, which is not compatible between - // 2.12- with 2.13+. + // 2.12-, 2.13+, and scala 3. Compile / unmanagedSourceDirectories += { val sourceDir = (Compile / sourceDirectory).value CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, n)) if n >= 13 => sourceDir / "scala-2.13+" - case _ => sourceDir / "scala-2.12-" + case Some((2, n)) if n < 13 => sourceDir / "scala-2.12-" + case _ => sourceDir / "scala-2.13+" } }, ScoverageKeys.coverageHighlighting := true, resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", scalacOptions := Seq( - "-target:jvm-1.8", "-unchecked", "-deprecation", "-feature", "-encoding", "utf8", - // Needs -missing-interpolator due to https://issues.scala-lang.org/browse/SI-8761 - "-Xlint:-missing-interpolator", - "-Yrangepos" - ), + ) ++ { + if (scalaVersion.value.startsWith("2")) scala2cOptions + else scala3cOptions + }, // Note: Use -Xlint rather than -Xlint:unchecked when TestThriftStructure // warnings are resolved javacOptions ++= Seq("-Xlint:unchecked", "-source", "1.8", "-target", "1.8"), @@ -172,7 +193,15 @@ val baseSettings = Seq( } ) -val sharedSettings = defaultProjectSettings ++ baseSettings +val sharedSettings = + baseSettings ++ + defaultScala2Settings ++ + Seq(libraryDependencies ++= scala2Dependencies) + +val sharedScala3Settings = + baseSettings ++ + defaultScala3Settings ++ + Seq(libraryDependencies ++= scala3Dependencies) lazy val noPublishSettings = Seq( publish / skip := true @@ -311,21 +340,37 @@ lazy val utilCore = Project( id = "util-core", base = file("util-core") ).settings( - sharedSettings + sharedScala3Settings ).settings( name := "util-core", // Moved some code to 'concurrent-extra' to conform to Pants' 1:1:1 principle (https://www.pantsbuild.org/build_files.html#target-granularity) // so that util-core would work better for Pants projects in IntelliJ. Compile / unmanagedSourceDirectories += baseDirectory.value / "concurrent-extra", + scalacOptions ++= { + // We're keeping in the migration flag until we use the `-rewrite` flag to have the compiler make the required syntax changes for us. + if (scalaVersion.value.startsWith("2")) Seq.empty + else Seq("-source:3.0-migration") + }, libraryDependencies ++= Seq( caffeineLib % "test", scalacheckLib, - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2", "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test", - "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" - ), + ) ++ { + if (scalaVersion.value.startsWith("2")) { + Seq( + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2", + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" + ) + } else { + Seq( + "org.scala-lang.modules" %% "scala-parser-combinators" % "2.0.0", + "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", + "org.scalatestplus" %% "mockito-3-4" % "3.2.9.0" % "test" + ) + } + }, Compile / resourceGenerators += Def.task { val projectName = name.value val file = resourceManaged.value / "com" / "twitter" / projectName / "build.properties" @@ -416,7 +461,13 @@ lazy val utilJackson = Project( "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" ) - ).dependsOn(utilCore, utilJacksonAnnotations, utilMock % Test, utilReflect, utilSlf4jApi, utilValidator) + ).dependsOn( + utilCore, + utilJacksonAnnotations, + utilMock % Test, + utilReflect, + utilSlf4jApi, + utilValidator) lazy val utilJvm = Project( id = "util-jvm", diff --git a/util-core/BUILD b/util-core/BUILD index e4143e2ea9..f1f1666ecf 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -17,11 +17,12 @@ test_suite( # (DPB-10452) This target definition lives here because it needs to accommodate util-core # code resolution in IntelliJ as well as `util-core-util` which contains packages -# com.twitter.concurrent and com.twitter.util +# com.twitter.concurrent, com.twitter.conversions, and com.twitter.util scala_library( name = "util-core-util", sources = [ "concurrent-extra/src/main/scala/com/twitter/concurrent/*.scala", + "src/main/scala-2.12-/com/twitter/conversions/*.scala", "src/main/scala-2.12-/com/twitter/util/*.scala", "src/main/scala/com/twitter/util/*.scala", ], diff --git a/util-core/src/main/java/com/twitter/util/Vars.java b/util-core/src/main/java/com/twitter/util/Vars.java index 928c18d1b9..4851666eaf 100644 --- a/util-core/src/main/java/com/twitter/util/Vars.java +++ b/util-core/src/main/java/com/twitter/util/Vars.java @@ -34,8 +34,8 @@ public static Var newVar(T init, Event event) { /** * @see com.twitter.util.Var$#value(Object) */ - public static Var newConstVar(T constant) { - return Var$.MODULE$.value(constant); + public static ConstVar newConstVar(T constant) { + return new ConstVar(constant); } /** diff --git a/util-core/src/main/scala-2.12-/com/twitter/conversions/SeqUtil.scala b/util-core/src/main/scala-2.12-/com/twitter/conversions/SeqUtil.scala new file mode 100644 index 0000000000..41765424de --- /dev/null +++ b/util-core/src/main/scala-2.12-/com/twitter/conversions/SeqUtil.scala @@ -0,0 +1,6 @@ +package com.twitter.conversions + +object SeqUtil { + def hasKnownSize[A](self: Seq[A]): Boolean = + self.hasDefiniteSize +} diff --git a/util-core/src/main/scala-2.13+/com/twitter/conversions/SeqUtil.scala b/util-core/src/main/scala-2.13+/com/twitter/conversions/SeqUtil.scala new file mode 100644 index 0000000000..a9675dd825 --- /dev/null +++ b/util-core/src/main/scala-2.13+/com/twitter/conversions/SeqUtil.scala @@ -0,0 +1,6 @@ +package com.twitter.conversions + +object SeqUtil { + def hasKnownSize[A](self: Seq[A]): Boolean = + self.knownSize != -1 +} diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 2daf0e3ecd..469b25e1ff 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -1,5 +1,6 @@ package com.twitter.concurrent +import com.twitter.conversions.SeqUtil import com.twitter.util.{Future, Return, Throw, Promise} import scala.annotation.varargs @@ -583,19 +584,21 @@ object AsyncStream { private case object Empty extends AsyncStream[Nothing] private case class Embed[A](fas: Future[AsyncStream[A]]) extends AsyncStream[A] private case class FromFuture[A](fa: Future[A]) extends AsyncStream[A] - private class Cons[A](val fa: Future[A], next: () => AsyncStream[A]) extends AsyncStream[A] { - private[this] lazy val _more: AsyncStream[A] = next() - def more(): AsyncStream[A] = _more - override def toString(): String = s"Cons($fa, $next)" - } + /** + * This is defined as a case class so that we get a generated unapply that works with + * [[AsyncStream]]s covariant type in Scala 3. See CSL-11207 or + * https://github.com/lampepfl/dotty/issues/13125 + */ + private case class Cons[A] private (fa: Future[A], next: () => AsyncStream[A]) + extends AsyncStream[A] object Cons { - def apply[A](fa: Future[A], next: () => AsyncStream[A]): AsyncStream[A] = - new Cons(fa, next) - - def unapply[A](as: Cons[A]): Option[(Future[A], () => AsyncStream[A])] = - // note: pattern match returns the memoized value - Some((as.fa, () => as.more())) + def apply[A](fut: Future[A], nextAsync: () => AsyncStream[A]): AsyncStream[A] = { + lazy val _more: AsyncStream[A] = nextAsync() + new Cons(fut, () => _more) { + override def toString(): String = s"Cons($fut, $nextAsync)" + } + } } implicit class Ops[A](tail: => AsyncStream[A]) { @@ -643,7 +646,7 @@ object AsyncStream { */ def fromSeq[A](seq: Seq[A]): AsyncStream[A] = seq match { case Nil => empty - case _ if seq.hasDefiniteSize && seq.tail.isEmpty => of(seq.head) + case _ if SeqUtil.hasKnownSize(seq) && seq.tail.isEmpty => of(seq.head) case _ => seq.head +:: fromSeq(seq.tail) } diff --git a/util-core/src/main/scala/com/twitter/concurrent/Broker.scala b/util-core/src/main/scala/com/twitter/concurrent/Broker.scala index 937c416c41..991bd422d9 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Broker.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Broker.scala @@ -73,49 +73,56 @@ class Broker[T] { Future.value(sendTx) } - case s @ (Quiet | Sending(_)) => - val p = new Promise[Tx[Unit]] - val elem: (Promise[Tx[Unit]], T) = (p, msg) - p.setInterruptHandler { - case _ => rmElem(elem) - } - val nextState = s match { - case Quiet => Sending(Queue(elem)) - case Sending(q) => Sending(q enqueue elem) - case Receiving(_) => throw new IllegalStateException() - } - - if (state.compareAndSet(s, nextState)) p else prepare() + case s @ Sending(q) => + val elem = createElem() + if (state.compareAndSet(s, Sending(q enqueue elem))) elem._1 + else prepare() + + case Quiet => + val elem = createElem() + if (state.compareAndSet(Quiet, Sending(Queue(elem)))) elem._1 + else prepare() } } + + def createElem(): (Promise[Tx[Unit]], T) = { + val p = new Promise[Tx[Unit]] + val elem = (p, msg) + p.setInterruptHandler { case _ => rmElem(elem) } + elem + } } val recv: Offer[T] = new Offer[T] { @tailrec - def prepare() = - state.get match { - case s @ Sending(sq) => - if (sq.isEmpty) throw new IllegalStateException() - val ((sendp, msg), newq) = sq.dequeue - val nextState = if (newq.isEmpty) Quiet else Sending(newq) - if (!state.compareAndSet(s, nextState)) prepare() - else { - val (sendTx, recvTx) = Tx.twoParty(msg) - sendp.setValue(sendTx) - Future.value(recvTx) - } + def prepare() = state.get match { + case s @ Sending(sq) => + if (sq.isEmpty) throw new IllegalStateException() + val ((sendp, msg), newq) = sq.dequeue + val nextState = if (newq.isEmpty) Quiet else Sending(newq) + if (!state.compareAndSet(s, nextState)) prepare() + else { + val (sendTx, recvTx) = Tx.twoParty(msg) + sendp.setValue(sendTx) + Future.value(recvTx) + } - case s @ (Quiet | Receiving(_)) => - val p = new Promise[Tx[T]] - p.setInterruptHandler { case _ => rmElem(p) } - val nextState = s match { - case Quiet => Receiving(Queue(p)) - case Receiving(q) => Receiving(q enqueue p) - case Sending(_) => throw new IllegalStateException() - } + case s @ Receiving(q) => + val p = createPromise() + if (state.compareAndSet(s, Receiving(q enqueue p))) p + else prepare() + + case Quiet => + val p = createPromise() + if (state.compareAndSet(Quiet, Receiving(Queue(p)))) p + else prepare() + } - if (state.compareAndSet(s, nextState)) p else prepare() - } + def createPromise(): Promise[Tx[T]] = { + val p = new Promise[Tx[T]] + p.setInterruptHandler { case _ => rmElem(p) } + p + } } /* Scala actor style / CSP syntax. */ diff --git a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala index c56a55c71e..aedbe9c213 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala @@ -7,7 +7,8 @@ import java.io.EOFException /** * Note: [[Spool]] is no longer the recommended asynchronous stream abstraction. - * We encourage you to use [[AsyncStream]] instead. + * We encourage you to use [[AsyncStream]] instead. See [[Spools]] for the Java + * adaptation of the [[Spool]] companion object. * * A spool is an asynchronous stream. It more or less mimics the scala * {{Stream}} collection, but with cons cells that have either eager or @@ -35,8 +36,6 @@ import java.io.EOFException * fill(rest) * firstElem *:: rest * }}} - * - * Note: There is a Java-friendly API for this trait: [[com.twitter.concurrent.AbstractSpool]]. */ sealed trait Spool[+A] { // NB: Spools are always lazy internally in order to provide the expected behavior @@ -78,12 +77,12 @@ sealed trait Spool[+A] { Future { f(Some(head)) } flatMap { _ => tail transform { case Return(s) => s.foreachElem(f) - case Throw(_: EOFException) => Future { f(None) } + case Throw(_: EOFException) => Future { f(None); () } case Throw(cause) => Future.exception(cause) } } } else { - Future { f(None) } + Future { f(None); () } } } @@ -271,11 +270,6 @@ sealed trait Spool[+A] { def force: Future[Unit] = foreach { _ => () } } -/** - * Abstract `Spool` class for Java compatibility. - */ -abstract class AbstractSpool[A] extends Spool[A] - /** * Note: [[Spool]] is no longer the recommended asynchronous stream abstraction. * We encourage you to use [[AsyncStream]] instead. diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 7f257775b6..7a2dbdc6bb 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -161,7 +161,11 @@ final class Pipe[A](timer: Timer) extends Reader[A] with Writer[A] { case State.Reading(p) => // pending reader has enough space for the full write state = State.Idle - (p, Some(buf), Future.Done) + // The Scala 3 compiler differentiates between the class type, `A` and the the defined + // type of Promise in `Reading`, due to `State` being covariant in it's type parameter + // but invariant in `Reading`. Let's explictly cast this type as we know it will be of + // `Promise[Option[A]]`. See CSL-11210 or https://github.com/lampepfl/dotty/issues/13126 + (p.asInstanceOf[Promise[Option[A]]], Some(buf), Future.Done) case State.Writing(_, _) => ( diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 7c264a2e91..0433a2dda6 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -5,7 +5,7 @@ import com.twitter.util.Promise.Detachable import com.twitter.util._ import java.io.{File, FileInputStream, InputStream} import java.util.{Collection => JCollection} -import scala.collection.JavaConverters +import scala.jdk.CollectionConverters._ /** * A reader exposes a pull-based API to model a potentially infinite stream of arbitrary elements. @@ -291,7 +291,7 @@ object Reader { * Create a new [[Reader]] from a given Java `List` */ def fromCollection[A](list: JCollection[A]): Reader[A] = fromSeq( - JavaConverters.asScalaIteratorConverter(list.iterator).asScala.toSeq + list.iterator.asScala.toSeq ) /** diff --git a/util-core/src/main/scala/com/twitter/util/Awaitable.scala b/util-core/src/main/scala/com/twitter/util/Awaitable.scala index 0f5e44b595..4461327d93 100644 --- a/util-core/src/main/scala/com/twitter/util/Awaitable.scala +++ b/util-core/src/main/scala/com/twitter/util/Awaitable.scala @@ -225,7 +225,12 @@ private[util] trait CloseAwaitably0[U <: Unit] extends Awaitable[U] { } // See https://stackoverflow.com/questions/26643045/java-interoperability-woes-with-scala-generics-and-boxing -private[util] trait CloseOnceAwaitably0[U <: Unit] extends Awaitable[U] { self: CloseOnce => +// +// NOTE: the self type must mixin [[Closable]] due to Scala 3 requirement that the self type is closed. +// See https://github.com/lampepfl/dotty/issues/2214 & +// https://github.com/lampepfl/dotty/commit/175499537c87c78d0b926d84b7a9030011e42c00 +private[util] trait CloseOnceAwaitably0[U <: Unit] extends Awaitable[U] { + self: CloseOnce with Closable => def ready(timeout: Duration)(implicit permit: Awaitable.CanAwait): this.type = { closeFuture.ready(timeout) @@ -251,7 +256,7 @@ private[util] trait CloseOnceAwaitably0[U <: Unit] extends Awaitable[U] { self: * } * }}} */ -trait CloseOnceAwaitably extends CloseOnceAwaitably0[Unit] { self: CloseOnce => } +trait CloseOnceAwaitably extends CloseOnceAwaitably0[Unit] { self: CloseOnce with Closable => } /** * A mixin to make an [[com.twitter.util.Awaitable]] out diff --git a/util-core/src/main/scala/com/twitter/util/CloseOnce.scala b/util-core/src/main/scala/com/twitter/util/CloseOnce.scala index f27ea6aca6..eb4a274e4c 100644 --- a/util-core/src/main/scala/com/twitter/util/CloseOnce.scala +++ b/util-core/src/main/scala/com/twitter/util/CloseOnce.scala @@ -12,8 +12,12 @@ import scala.util.control.NonFatal * @see [[ClosableOnce]] if you are not mixing in or extending an existing [[Closable]] * @see [[ClosableOnce.of]] for creating a proxy to a [[Closable]] * that has already been instantiated. + * + * @note this mixin extends [[Closable]] as a workaround for extending [[AbstractClosableOnce]] + * in Java. The mixin's final override is not recognized when compiled with Scala 3. See CSL-11187 + * or https://github.com/lampepfl/dotty/issues/13104. */ -trait CloseOnce { self: Closable => +trait CloseOnce extends Closable { self: Closable => // Our intrinsic lock for mutating the `closed` field private[this] val closePromise: Promise[Unit] = Promise[Unit]() @volatile private[this] var closed: Boolean = false @@ -71,5 +75,4 @@ trait CloseOnce { self: Closable => closePromise } - } diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index 3125fa086b..c52403e9ba 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -437,8 +437,11 @@ sealed class Duration private[util] (protected val nanos: Long) def %(x: Duration): Duration = x match { case Undefined => Undefined case ns if ns.isZero => Undefined - case ns if ns.isFinite => fromNanoseconds(nanos % ns.inNanoseconds) - case Top | Bottom => this + // Analogous to `case Top | Bottom =>`. + // We pattern match this way so that the compiler + // recognizes a fully exhaustive pattern match. + case ns if !ns.isFinite => this + case ns => fromNanoseconds(nanos % ns.inNanoseconds) } /** diff --git a/util-core/src/main/scala/com/twitter/util/Event.scala b/util-core/src/main/scala/com/twitter/util/Event.scala index 2a9421df64..39b9ab4591 100644 --- a/util-core/src/main/scala/com/twitter/util/Event.scala +++ b/util-core/src/main/scala/com/twitter/util/Event.scala @@ -196,6 +196,8 @@ trait Event[+T] { self => if (rest.isEmpty) state = None else state = Some(Right(Queue(rest: _*))) s.notify((t, u)) + case Some(Right(_)) => + throw new IllegalStateException("unexpected empty queue") } } } @@ -211,6 +213,8 @@ trait Event[+T] { self => if (rest.isEmpty) state = None else state = Some(Left(Queue(rest: _*))) s.notify((t, u)) + case Some(Left(_)) => + throw new IllegalStateException("unexpected empty queue") } } } diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 054306811b..9c3c579b5d 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -144,7 +144,7 @@ object Promise { * A template trait for [[com.twitter.util.Promise Promises]] that are derived * and capable of being detached from other Promises. */ - trait Detachable { _: Promise[_] => + trait Detachable { self: Promise[_] => /** * Returns true if successfully detached, will return true at most once. @@ -571,13 +571,20 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ } } - @tailrec final def raise(intr: Throwable): Unit = state match { + final def raise(intr: Throwable): Unit = raiseImpl(intr) + + // This is implemented as a private method on Promise so that the Scala 3 + // compiler recognizes this as a tail-recursive implementation. `raise` is + // a definition on `Future` instead of `Promise`, which the Scala 3 + // compiler errors on with @tailrec annocation on the `Transforming` case, + // where `s.other.raise` is invoked on a Future and not Promise. See CSL-11192. + @tailrec private def raiseImpl(intr: Throwable): Unit = state match { case waitq: WaitQueue[A] => if (!cas(waitq, new Interrupted(waitq, intr))) - raise(intr) + raiseImpl(intr) case s: Interruptible[A] => - if (!cas(s, new Interrupted(s.waitq, intr))) raise(intr) + if (!cas(s, new Interrupted(s.waitq, intr))) raiseImpl(intr) else { val current = Local.save() if (current ne s.saved) @@ -587,18 +594,18 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ } case s: Transforming[A] => - if (!cas(s, new Interrupted(s.waitq, intr))) raise(intr) + if (!cas(s, new Interrupted(s.waitq, intr))) raiseImpl(intr) else { s.other.raise(intr) } case s: Interrupted[A] => if (!cas(s, new Interrupted(s.waitq, intr))) - raise(intr) + raiseImpl(intr) case _: Try[A] /* Done */ => () // nothing to do, as its already satisfied. - case p: Promise[A] /* Linked */ => p.raise(intr) + case p: Promise[A] /* Linked */ => p.raiseImpl(intr) } @tailrec protected[Promise] final def detach(k: K[A]): Boolean = state match { diff --git a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala index ae90a150ce..f1d8cf2a8d 100644 --- a/util-core/src/main/scala/com/twitter/util/StorageUnit.scala +++ b/util-core/src/main/scala/com/twitter/util/StorageUnit.scala @@ -51,6 +51,13 @@ object StorageUnit { } } + // We define this internally since `signum` is deprecated since Scala 2.13 in favor of `sig`. + // Lets define this simple method here until support for Scala 2.12 is dropped. + private def sig(bytes: Long): Int = + if (bytes > 0) 1 + else if (bytes < 0) -1 + else 0 + /** * Note, this can cause overflows of the Long used to represent the * number of bytes. @@ -87,6 +94,8 @@ object StorageUnit { * number of bytes. */ class StorageUnit(val bytes: Long) extends Ordered[StorageUnit] { + import StorageUnit.sig + def inBytes: Long = bytes def inKilobytes: Long = bytes / 1024L def inMegabytes: Long = bytes / (1024L * 1024) @@ -139,7 +148,8 @@ class StorageUnit(val bytes: Long) extends Ordered[StorageUnit] { if (prefixIndex < 0) { "%d B".format(bytes) } else { - "%.1f %ciB".formatLocal(Locale.ENGLISH, display * bytes.signum, prefix.charAt(prefixIndex)) + "%.1f %ciB".formatLocal(Locale.ENGLISH, display * sig(bytes), prefix.charAt(prefixIndex)) } } + } diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 0dd17eb98e..2169bcba96 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -271,7 +271,7 @@ object Var { // In order to support unsubscribing from diffs when v is no longer referenced // we must avoid diffs keeping a strong reference to v. - val witness = Witness.weakReference { diff: Diff[CC, T] => + val witness = Witness.weakReference { (diff: Diff[CC, T]) => synchronized { v.update(diff.patch(v())) } @@ -282,20 +282,10 @@ object Var { v } - private case class Value[T](v: T) extends Var[T] with Extractable[T] { - protected def observe(depth: Int, obs: Observer[T]): Closable = { - obs.claim(this) - obs.publish(this, v, 0) - Closable.nop - } - - def apply(): T = v - } - /** * Create a new, constant, v-valued Var. */ - def value[T](v: T): Var[T] with Extractable[T] = Value(v) + def value[T](v: T): Var[T] with Extractable[T] = new ConstVar(v) /** * Collect a collection of Vars into a Var of collection. @@ -550,6 +540,19 @@ private[util] class UpdatableVar[T](init: T) extends Var[T] with Updatable[T] wi override def toString: String = "Var(" + state.get.value + ")@" + hashCode } +/** + * A constant [[Extractable]] [[Var]] on `v`. + */ +class ConstVar[T](v: T) extends Var[T] with Extractable[T] { + protected def observe(depth: Int, obs: Var.Observer[T]): Closable = { + obs.claim(this) + obs.publish(this, v, 0) + Closable.nop + } + + def apply(): T = v +} + /** * Java adaptation of `Var[T] with Updatable[T] with Extractable[T]`. */ diff --git a/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java b/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java index 2b89e1b9d6..86ee3e5fcd 100644 --- a/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java +++ b/util-core/src/test/java/com/twitter/concurrent/SpoolCompilationTest.java @@ -10,28 +10,6 @@ import java.util.Collection; public class SpoolCompilationTest { - private static class OwnSpool extends AbstractSpool { - @Override - public boolean isEmpty() { - return false; - } - - @Override - public Future> tail() { - return Future.value(Spools.newEmptySpool()); - } - - @Override - public String head() { - return "spool"; - } - } - @Test - public void testOwnSpool() { - Spool a = new OwnSpool(); - Assert.assertFalse(a.isEmpty()); - Assert.assertEquals("spool", a.head()); - } @Test public void testSpoolCreation() { diff --git a/util-core/src/test/java/com/twitter/util/AwaitableCompilationTest.java b/util-core/src/test/java/com/twitter/util/AwaitableCompilationTest.java index 3fdae8ea17..5182dec164 100644 --- a/util-core/src/test/java/com/twitter/util/AwaitableCompilationTest.java +++ b/util-core/src/test/java/com/twitter/util/AwaitableCompilationTest.java @@ -12,17 +12,17 @@ public class AwaitableCompilationTest { public static class MockAwaitable implements Awaitable { @Override - public Awaitable ready(Duration timeout, CanAwait permit) throws TimeoutException { + public Awaitable ready(Duration timeout, Awaitable.CanAwait permit) throws TimeoutException { return this; } @Override - public String result(Duration timeout, CanAwait permit) throws Exception { + public String result(Duration timeout, Awaitable.CanAwait permit) throws Exception { return "42"; } @Override - public boolean isReady(CanAwait permit) { + public boolean isReady(Awaitable.CanAwait permit) { return true; } } diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 60b394610b..28f5eb2440 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -5,6 +5,7 @@ import com.twitter.util._ import org.scalacheck.{Arbitrary, Gen} import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.scalatest.funsuite.AnyFunSuite +import scala.collection.compat.immutable.LazyList class AsyncStreamTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { import AsyncStream.{mk, of} @@ -114,7 +115,7 @@ class AsyncStreamTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { } test(s"$impl: fromSeq works on infinite streams") { - def ones: Stream[Int] = 1 #:: ones + def ones: LazyList[Int] = 1 #:: ones assert(toSeq(fromSeq(ones).take(3)) == Seq(1, 1, 1)) } diff --git a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala index 277404c8cd..2808dad379 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala @@ -1,20 +1,20 @@ package com.twitter.concurrent -import scala.util.Random - -import org.mockito.Mockito._ -import org.scalatestplus.mockito.MockitoSugar import com.twitter.conversions.DurationOps._ import com.twitter.util.{Await, Future, MockTimer, Promise, Return, Time} +import org.mockito.Mockito._ +import org.scalatestplus.mockito.MockitoSugar import org.scalatest.wordspec.AnyWordSpec +import scala.collection.compat.immutable.LazyList +import scala.util.Random -class SimpleOffer[T](var futures: Stream[Future[Tx[T]]]) extends Offer[T] { - def this(fut: Future[Tx[T]]) = this(Stream.continually(fut)) - def this(tx: Tx[T]) = this(Stream.continually(Future.value(tx))) +class SimpleOffer[T](var futures: LazyList[Future[Tx[T]]]) extends Offer[T] { + def this(fut: Future[Tx[T]]) = this(LazyList.continually(fut)) + def this(tx: Tx[T]) = this(LazyList.continually(Future.value(tx))) def prepare(): Future[Tx[T]] = { - val next #:: rest = futures - futures = rest + val next = futures.head + futures = futures.tail next } } @@ -176,7 +176,7 @@ class OfferTest extends AnyWordSpec with MockitoSugar { } "retry when it aborts" in { - val txps = new Promise[Tx[Int]] #:: new Promise[Tx[Int]] #:: Stream.empty + val txps = new Promise[Tx[Int]] #:: new Promise[Tx[Int]] #:: LazyList.empty val offer = spy(new SimpleOffer(txps)) val badTx = mock[Tx[Int]] val result = Future.value(Abort) @@ -262,7 +262,7 @@ class OfferTest extends AnyWordSpec with MockitoSugar { val h = new OfferOrElseHelper import h._ - val e1 = spy(new SimpleOffer(Stream.empty)) + val e1 = spy(new SimpleOffer(LazyList.empty)) val offer = e0 orElse e1 val tx = mock[Tx[Int]] val result = Future.value(Commit(321)) @@ -283,7 +283,7 @@ class OfferTest extends AnyWordSpec with MockitoSugar { val tx2 = new Promise[Tx[Int]] val e0 = spy( new SimpleOffer( - Future.value(Tx.aborted: Tx[Int]) #:: (tx2: Future[Tx[Int]]) #:: Stream.empty + Future.value(Tx.aborted: Tx[Int]) #:: (tx2: Future[Tx[Int]]) #:: LazyList.empty ) ) val offer = e0 orElse Offer.const(123) diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala index 443a224a51..38cc0d2c65 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala @@ -115,7 +115,9 @@ class SpoolTest extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { assert(x == 1) await(rest) match { case y *:: rest if y == 2 => await(rest).isEmpty + case _ => fail("expected 3 elements in spool") } + case _ => fail("expected 3 elements in spool") }) } diff --git a/util-core/src/test/scala/com/twitter/conversions/SeqOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/SeqOpsTest.scala index d92e84c498..88d1aad871 100644 --- a/util-core/src/test/scala/com/twitter/conversions/SeqOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/SeqOpsTest.scala @@ -2,6 +2,7 @@ package com.twitter.conversions import com.twitter.conversions.SeqOps._ import org.scalatest.funsuite.AnyFunSuite +import scala.collection.compat.immutable.LazyList class SeqOpsTest extends AnyFunSuite { @@ -23,8 +24,8 @@ class SeqOpsTest extends AnyFunSuite { assert(Seq(1, 2, 3).findItemAfter(5) == None) assert(Seq[Int]().findItemAfter(5) == None) - assert(Stream(1, 2, 3).findItemAfter(1) == Some(2)) - assert(Stream[Int]().findItemAfter(5) == None) + assert(LazyList(1, 2, 3).findItemAfter(1) == Some(2)) + assert(LazyList[Int]().findItemAfter(5) == None) assert(Vector(1, 2, 3).findItemAfter(1) == Some(2)) assert(Vector[Int]().findItemAfter(5) == None) diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 1773df41cd..27460ae197 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -11,9 +11,10 @@ import org.mockito.Mockito._ import org.scalacheck.{Arbitrary, Gen} import org.scalatest.concurrent.{Eventually, IntegrationPatience} import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -import scala.collection.mutable.ArrayBuffer import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import scala.collection.compat.immutable.LazyList +import scala.collection.mutable.ArrayBuffer class ReaderTest extends AnyFunSuite @@ -252,7 +253,7 @@ class ReaderTest val i2 = p2.interruptible val r2 = Reader.fromFuture(i2) - def rStreams: Stream[Reader[Any]] = r1 #:: r2 #:: rStreams + def rStreams: LazyList[Reader[Any]] = r1 #:: r2 #:: rStreams val reader = Reader.fromSeq(rStreams) val rFlat = reader.flatten @@ -270,7 +271,7 @@ class ReaderTest } test("Reader.fromSeq works on infinite streams") { - def ones: Stream[Int] = 1 #:: ones + def ones: LazyList[Int] = 1 #:: ones val reader = Reader.fromSeq(ones) assert(await(reader.read()) == Some(1)) assert(await(reader.read()) == Some(1)) diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index a485a75941..178593abf8 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -2,7 +2,6 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import org.scalatest.concurrent.{Eventually, IntegrationPatience} -import scala.language.reflectiveCalls import org.scalatest.funsuite.AnyFunSuite class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience { @@ -78,7 +77,8 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience test("Closable.all with exceptions") { val throwing = Closable.make(_ => sys.error("lolz")) - val tracking = new Closable { + + class TrackedClosable extends Closable { @volatile var calledClose = false override def close(deadline: Time): Future[Unit] = { @@ -87,6 +87,7 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience } } + val tracking = new TrackedClosable() val f = Closable.all(throwing, tracking).close() intercept[Exception] { Await.result(f, 5.seconds) diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 3342621b79..964703ebf7 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -18,8 +18,10 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import java.util.concurrent.TimeUnit +import org.scalatest.wordspec.AnyWordSpec -class DurationTest extends { val ops: Duration.type = Duration } with TimeLikeSpec[Duration] { +trait DurationOps { val ops: Duration.type = Duration } +class DurationTest extends AnyWordSpec with DurationOps with TimeLikeSpec[Duration] { "Duration" should { "*" in { diff --git a/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala b/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala index 73fab075e8..2d61bbe02d 100644 --- a/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala @@ -10,7 +10,7 @@ class LastWriteWinsQueueTest extends AnyWordSpec { assert(queue.size == 0) queue.add("1") assert(queue.size == 1) - assert(queue.remove() == "1") + assert(queue.remove == "1") assert(queue.size == 0) } diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 62a3d3a237..56ecac4a01 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -395,8 +395,13 @@ class TimeFormatTest extends AnyWordSpec { } } -class TimeTest extends { val ops: Time.type = Time } with TimeLikeSpec[Time] with Eventually -with IntegrationPatience { +trait TimeOps { val ops: Time.type = Time } +class TimeTest + extends AnyWordSpec + with TimeOps + with TimeLikeSpec[Time] + with Eventually + with IntegrationPatience { "Time" should { "work in collections" in { diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index d3eb6b8dd9..463ef8528d 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -451,7 +451,8 @@ class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { // in order to avoid extremely long runtimes, we limit the max size // of the generated Seqs and Sets. A limit of 50 keeps runtime below // 1 second on my crufty old laptop. - implicit val generatorDrivenConfig = PropertyCheckConfiguration(sizeRange = 50) + implicit val generatorDrivenConfig: PropertyCheckConfiguration = + PropertyCheckConfiguration(sizeRange = 50) forAll(arbitrary[Seq[Set[Int]]].suchThat(_.nonEmpty)) { sets => val v = Var(sets.head) From cb4beb96913716b1e36280d45ea14949277a5ad3 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Thu, 29 Jul 2021 00:57:20 +0000 Subject: [PATCH 523/796] util/util-core: Scala 3 compiler migration rewrite Problem After fixing incompatibilities in D694775, the scala compiler can resolve warnings by itself. Both syntax and deprecation. Solution Run the `-rewrite` compiler option to resolve warnings. Result Remove the `source:3.0-migration` flag. Differential Revision: https://phabricator.twitter.biz/D710778 --- build.sbt | 5 -- .../com/twitter/concurrent/AsyncMeter.scala | 2 +- .../com/twitter/concurrent/Serialized.scala | 5 +- .../src/main/scala/com/twitter/io/Buf.scala | 12 ++--- .../scala/com/twitter/io/TempFolder.scala | 6 +-- .../scala/com/twitter/util/Disposable.scala | 2 +- .../main/scala/com/twitter/util/Promise.scala | 6 +-- .../main/scala/com/twitter/util/Signal.scala | 2 +- .../twitter/concurrent/AsyncStreamTest.scala | 8 ++-- .../twitter/conversions/StringOpsTest.scala | 29 +++++------ .../com/twitter/conversions/U64OpsTest.scala | 4 +- .../com/twitter/io/BufByteWriterTest.scala | 36 +++++++------- .../scala/com/twitter/io/BufReaderTest.scala | 6 +-- .../test/scala/com/twitter/io/BufTest.scala | 2 +- .../scala/com/twitter/io/ByteReaderTest.scala | 48 +++++++++---------- .../com/twitter/io/IteratorReaderTest.scala | 4 +- .../scala/com/twitter/io/ReaderTest.scala | 24 +++++----- .../scala/com/twitter/util/ActivityTest.scala | 18 +++---- .../com/twitter/util/Base64LongTest.scala | 4 +- .../com/twitter/util/CancellableTest.scala | 12 ++--- .../scala/com/twitter/util/FutureTest.scala | 4 +- .../scala/com/twitter/util/MemoizeTest.scala | 4 +- .../com/twitter/util/StorageUnitTest.scala | 12 ++--- .../scala/com/twitter/util/TimeTest.scala | 6 +-- .../test/scala/com/twitter/util/VarTest.scala | 2 +- 25 files changed, 131 insertions(+), 132 deletions(-) diff --git a/build.sbt b/build.sbt index d3aad3dbd6..c8fdfe0642 100644 --- a/build.sbt +++ b/build.sbt @@ -346,11 +346,6 @@ lazy val utilCore = Project( // Moved some code to 'concurrent-extra' to conform to Pants' 1:1:1 principle (https://www.pantsbuild.org/build_files.html#target-granularity) // so that util-core would work better for Pants projects in IntelliJ. Compile / unmanagedSourceDirectories += baseDirectory.value / "concurrent-extra", - scalacOptions ++= { - // We're keeping in the migration flag until we use the `-rewrite` flag to have the compiler make the required syntax changes for us. - if (scalaVersion.value.startsWith("2")) Seq.empty - else Seq("-source:3.0-migration") - }, libraryDependencies ++= Seq( caffeineLib % "test", scalacheckLib, diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala index 24b46d63ff..59e61c4d24 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncMeter.scala @@ -138,7 +138,7 @@ object AsyncMeter { val seqWithoutLast: Seq[Future[Unit]] = (0 until num).map(_ => meter.await(meter.burstSize)) val seq = if (last == 0) seqWithoutLast else seqWithoutLast :+ meter.await(last) val result = Future.join(seq) - result.onFailure { exc => seq.foreach { f: Future[Unit] => f.raise(exc) } } + result.onFailure { exc => seq.foreach { (f: Future[Unit]) => f.raise(exc) } } result } else meter.await(permits) } diff --git a/util-core/src/main/scala/com/twitter/concurrent/Serialized.scala b/util-core/src/main/scala/com/twitter/concurrent/Serialized.scala index 4849e9b987..1dd27ec135 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Serialized.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Serialized.scala @@ -28,9 +28,10 @@ trait Serialized { serializedQueue add { Job(result, () => f) } if (nwaiters.getAndIncrement() == 0) { - do { + Try { serializedQueue.remove()() } + while (nwaiters.decrementAndGet() > 0) { Try { serializedQueue.remove()() } - } while (nwaiters.decrementAndGet() > 0) + } } result diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index cc54e985f1..eee08d65cd 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -429,7 +429,7 @@ object Buf { def slice(from: Int, until: Int): Buf = { checkSliceArgs(from, until) - if (isSliceEmpty(from, until)) return Buf.Empty + if (this.isSliceEmpty(from, until)) return Buf.Empty else if (isSliceIdentity(from, until)) return this var begin = from @@ -503,7 +503,7 @@ object Buf { def process(from: Int, until: Int, processor: Processor): Int = { checkSliceArgs(from, until) - if (isSliceEmpty(from, until)) return -1 + if (this.isSliceEmpty(from, until)) return -1 var i = 0 var bufIdx = 0 var continue = true @@ -620,7 +620,7 @@ object Buf { def process(from: Int, until: Int, processor: Processor): Int = { checkSliceArgs(from, until) - if (isSliceEmpty(from, until)) return -1 + if (this.isSliceEmpty(from, until)) return -1 var i = from var continue = true val endAt = math.min(until, length) @@ -647,7 +647,7 @@ object Buf { def slice(from: Int, until: Int): Buf = { checkSliceArgs(from, until) - if (isSliceEmpty(from, until)) Buf.Empty + if (this.isSliceEmpty(from, until)) Buf.Empty else if (isSliceIdentity(from, until)) this else { val cap = math.min(until, length) @@ -826,7 +826,7 @@ object Buf { def process(from: Int, until: Int, processor: Processor): Int = { checkSliceArgs(from, until) - if (isSliceEmpty(from, until)) return -1 + if (this.isSliceEmpty(from, until)) return -1 val pos = underlying.position() var i = from var continue = true @@ -856,7 +856,7 @@ object Buf { def slice(from: Int, until: Int): Buf = { checkSliceArgs(from, until) - if (isSliceEmpty(from, until)) Buf.Empty + if (this.isSliceEmpty(from, until)) Buf.Empty else if (isSliceIdentity(from, until)) this else { val dup = underlying.duplicate() diff --git a/util-core/src/main/scala/com/twitter/io/TempFolder.scala b/util-core/src/main/scala/com/twitter/io/TempFolder.scala index 25b1fd492c..36eeed4fb9 100644 --- a/util-core/src/main/scala/com/twitter/io/TempFolder.scala +++ b/util-core/src/main/scala/com/twitter/io/TempFolder.scala @@ -38,10 +38,10 @@ trait TempFolder { val tempFolder = System.getProperty("java.io.tmpdir") // Note: If we were willing to have a dependency on Guava in util-core // we could just use `com.google.common.io.Files.createTempDir()` - var folder: File = null - do { + var folder: File = new File(tempFolder, "scala-test-" + System.currentTimeMillis) + while (!folder.mkdir()) { folder = new File(tempFolder, "scala-test-" + System.currentTimeMillis) - } while (!folder.mkdir()) + } _folderName.set(folder) try { diff --git a/util-core/src/main/scala/com/twitter/util/Disposable.scala b/util-core/src/main/scala/com/twitter/util/Disposable.scala index d2b502a0f0..78d29bae7f 100644 --- a/util-core/src/main/scala/com/twitter/util/Disposable.scala +++ b/util-core/src/main/scala/com/twitter/util/Disposable.scala @@ -112,7 +112,7 @@ trait Managed[+T] { selfT => u.dispose(deadline) transform { case Return(_) => t.dispose(deadline) case Throw(outer) => - t.dispose transform { + t.dispose() transform { case Throw(inner) => Future.exception(new DoubleTrouble(outer, inner)) case Return(_) => Future.exception(outer) } diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 9c3c579b5d..615aae8499 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -470,17 +470,17 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ @volatile private[this] var state: Any = WaitQueue.empty[A] private def theState(): Any = state - private[util] def this(forwardInterrupts: Future[_]) { + private[util] def this(forwardInterrupts: Future[_]) = { this() this.state = new Transforming[A](WaitQueue.empty, forwardInterrupts) } - def this(handleInterrupt: PartialFunction[Throwable, Unit]) { + def this(handleInterrupt: PartialFunction[Throwable, Unit]) = { this() this.state = new Interruptible[A](WaitQueue.empty, handleInterrupt, Local.save()) } - def this(result: Try[A]) { + def this(result: Try[A]) = { this() this.state = result } diff --git a/util-core/src/main/scala/com/twitter/util/Signal.scala b/util-core/src/main/scala/com/twitter/util/Signal.scala index 333af1588d..2a945361c1 100644 --- a/util-core/src/main/scala/com/twitter/util/Signal.scala +++ b/util-core/src/main/scala/com/twitter/util/Signal.scala @@ -23,7 +23,7 @@ import scala.collection.{Map, Set, mutable} object SignalHandlerFactory { def apply(): Option[SunSignalHandler] = { // only one actual implementation for now - SunSignalHandler.instantiate + SunSignalHandler.instantiate() } } diff --git a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala index 28f5eb2440..d24f1fcb7f 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/AsyncStreamTest.scala @@ -632,15 +632,15 @@ class AsyncStreamTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { } test(s"$impl: sum") { - forAll { xs: List[Int] => assert(xs.sum == await(fromSeq(xs).sum)) } + forAll { (xs: List[Int]) => assert(xs.sum == await(fromSeq(xs).sum)) } } test(s"$impl: size") { - forAll { xs: List[Int] => assert(xs.size == await(fromSeq(xs).size)) } + forAll { (xs: List[Int]) => assert(xs.size == await(fromSeq(xs).size)) } } test(s"$impl: force") { - forAll { xs: List[Int] => + forAll { (xs: List[Int]) => val p = new Promise[Unit] // The promise will be defined iff the tail is forced. val s = fromSeq(xs) ++ { p.setDone(); AsyncStream.empty } @@ -746,7 +746,7 @@ class AsyncStreamTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { val stream = tailRecM(0) { i => AsyncStream.of(if (i < n) Left(i + 1) else Right(i)) } - assert(Await.result(stream.toSeq) == Seq(n)) + assert(Await.result(stream.toSeq()) == Seq(n)) } } diff --git a/util-core/src/test/scala/com/twitter/conversions/StringOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/StringOpsTest.scala index f00df0459d..a7857e1011 100644 --- a/util-core/src/test/scala/com/twitter/conversions/StringOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/StringOpsTest.scala @@ -22,23 +22,24 @@ import org.scalatest.funsuite.AnyFunSuite class StringOpsTest extends AnyFunSuite { test("string#quoteC") { - assert("nothing".quoteC == "nothing") + assert("nothing".quoteC() == "nothing") assert( - "name\tvalue\t\u20acb\u00fcllet?\u20ac".quoteC == "name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac" + "name\tvalue\t\u20acb\u00fcllet?\u20ac".quoteC() == "name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac" ) - assert("she said \"hello\"".quoteC == "she said \\\"hello\\\"") - assert("\\backslash".quoteC == "\\\\backslash") + assert("she said \"hello\"".quoteC() == "she said \\\"hello\\\"") + assert("\\backslash".quoteC() == "\\\\backslash") } test("string#unquoteC") { - assert("nothing".unquoteC == "nothing") + assert("nothing".unquoteC() == "nothing") assert( - "name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac".unquoteC == "name\tvalue\t\u20acb\u00fcllet?\u20ac" + "name\\tvalue\\t\\u20acb\\xfcllet?\\u20ac" + .unquoteC() == "name\tvalue\t\u20acb\u00fcllet?\u20ac" ) - assert("she said \\\"hello\\\"".unquoteC == "she said \"hello\"") - assert("\\\\backslash".unquoteC == "\\backslash") - assert("real\\$dollar".unquoteC == "real\\$dollar") - assert("silly\\/quote".unquoteC == "silly/quote") + assert("she said \\\"hello\\\"".unquoteC() == "she said \"hello\"") + assert("\\\\backslash".unquoteC() == "\\backslash") + assert("real\\$dollar".unquoteC() == "real\\$dollar") + assert("silly\\/quote".unquoteC() == "silly/quote") } test("string#hexlify") { @@ -47,10 +48,10 @@ class StringOpsTest extends AnyFunSuite { } test("string#unhexlify") { - assert("656c6c".unhexlify.toList == "hello".getBytes.slice(1, 4).toList) - assert("68656c6c6f".unhexlify.toList == "hello".getBytes.toList) - "5".unhexlify - assert("5".unhexlify.hexlify.toInt == 5) + assert("656c6c".unhexlify().toList == "hello".getBytes.slice(1, 4).toList) + assert("68656c6c6f".unhexlify().toList == "hello".getBytes.toList) + "5".unhexlify() + assert("5".unhexlify().hexlify.toInt == 5) } test("string#toCamelCase") { diff --git a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala index a25f4cf076..942c41bb8c 100644 --- a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala @@ -7,11 +7,11 @@ class U64OpsTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { import com.twitter.conversions.U64Ops._ test("toU64HextString") { - forAll { l: Long => assert(l.toU64HexString == "%016x".format(l)) } + forAll { (l: Long) => assert(l.toU64HexString == "%016x".format(l)) } } test("toU64Long") { - forAll { l: Long => + forAll { (l: Long) => val s = "%016x".format(l) assert(s.toU64Long == java.lang.Long.parseUnsignedLong(s, 16)) } diff --git a/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala b/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala index c6bc755597..7b835ba818 100644 --- a/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufByteWriterTest.scala @@ -33,7 +33,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC } def testWriteByte(name: String, bwFactory: () => BufByteWriter, overflowOK: Boolean): Unit = - test(s"$name: writeByte")(forAll { byte: Byte => + test(s"$name: writeByte")(forAll { (byte: Byte) => val bw = bwFactory() val buf = bw.writeByte(byte).owned() @@ -43,7 +43,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC }) def testWriteShort(name: String, bwFactory: () => BufByteWriter, overflowOK: Boolean): Unit = - test(s"$name: writeShort{BE,LE}")(forAll { s: Short => + test(s"$name: writeShort{BE,LE}")(forAll { (s: Short) => val be = bwFactory().writeShortBE(s) val le = bwFactory().writeShortLE(s) @@ -64,7 +64,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC }) def testWriteMedium(name: String, bwFactory: () => BufByteWriter, overflowOK: Boolean): Unit = - test(s"$name: writeMedium{BE,LE}")(forAll { m: Int => + test(s"$name: writeMedium{BE,LE}")(forAll { (m: Int) => val be = bwFactory().writeMediumBE(m) val le = bwFactory().writeMediumLE(m) @@ -86,7 +86,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC }) def testWriteInt(name: String, bwFactory: () => BufByteWriter, overflowOK: Boolean): Unit = - test(s"$name: writeInt{BE,LE}")(forAll { i: Int => + test(s"$name: writeInt{BE,LE}")(forAll { (i: Int) => val be = bwFactory().writeIntBE(i) val le = bwFactory().writeIntLE(i) @@ -109,7 +109,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC }) def testWriteLong(name: String, bwFactory: () => BufByteWriter, overflowOK: Boolean): Unit = - test(s"$name: writeLong{BE,LE}")(forAll { l: Long => + test(s"$name: writeLong{BE,LE}")(forAll { (l: Long) => val be = bwFactory().writeLongBE(l) val le = bwFactory().writeLongLE(l) @@ -136,7 +136,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC }) def testWriteFloat(name: String, bwFactory: () => BufByteWriter, overflowOK: Boolean): Unit = - test(s"$name: writeFloat{BE,LE}")(forAll { f: Float => + test(s"$name: writeFloat{BE,LE}")(forAll { (f: Float) => val be = bwFactory().writeFloatBE(f) val le = bwFactory().writeFloatLE(f) @@ -161,7 +161,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC }) def testWriteDouble(name: String, bwFactory: () => BufByteWriter, overflowOK: Boolean): Unit = - test(s"$name: writeDouble{BE,LE}")(forAll { d: Double => + test(s"$name: writeDouble{BE,LE}")(forAll { (d: Double) => val be = bwFactory().writeDoubleBE(d) val le = bwFactory().writeDoubleLE(d) @@ -195,9 +195,9 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC } test("trims unwritten bytes") { - assert(BufByteWriter.fixed(5).owned.length == 0) - assert(BufByteWriter.fixed(5).writeIntBE(1).owned.length == 4) - assert(BufByteWriter.fixed(4).writeIntBE(1).owned.length == 4) + assert(BufByteWriter.fixed(5).owned().length == 0) + assert(BufByteWriter.fixed(5).writeIntBE(1).owned().length == 4) + assert(BufByteWriter.fixed(4).writeIntBE(1).owned().length == 4) } testWriteString("fixed", size => BufByteWriter.fixed(size), overflowOK = false) @@ -209,7 +209,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC testWriteFloat("fixed", () => BufByteWriter.fixed(4), overflowOK = false) testWriteDouble("fixed", () => BufByteWriter.fixed(8), overflowOK = false) - test("fixed: writeBytes(Array[Byte])")(forAll { bytes: Array[Byte] => + test("fixed: writeBytes(Array[Byte])")(forAll { (bytes: Array[Byte]) => val bw = BufByteWriter.fixed(bytes.length) val buf = bw.writeBytes(bytes).owned() intercept[OverflowException] { bw.writeByte(0xff) } @@ -217,7 +217,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC assertIndex(bw, bytes.length) }) - test("fixed: writeBytes(Array[Byte]) 2 times")(forAll { bytes: Array[Byte] => + test("fixed: writeBytes(Array[Byte]) 2 times")(forAll { (bytes: Array[Byte]) => val bw = BufByteWriter.fixed(bytes.length * 2) val buf = bw.writeBytes(bytes).writeBytes(bytes).owned() intercept[OverflowException] { bw.writeByte(0xff) } @@ -226,7 +226,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC assertIndex(bw, bytes.length * 2) }) - test("fixed: writeBytes(Buf)")(forAll { arr: Array[Byte] => + test("fixed: writeBytes(Buf)")(forAll { (arr: Array[Byte]) => val bytes = Buf.ByteArray.Owned(arr) val bw = BufByteWriter.fixed(bytes.length) val buf = bw.writeBytes(bytes).owned() @@ -235,7 +235,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC assertIndex(bw, bytes.length) }) - test("fixed: writeBytes(Buf) 2 times")(forAll { arr: Array[Byte] => + test("fixed: writeBytes(Buf) 2 times")(forAll { (arr: Array[Byte]) => val bytes = Buf.ByteArray.Owned(arr) val bw = BufByteWriter.fixed(bytes.length * 2) val buf = bw.writeBytes(bytes).writeBytes(bytes).owned() @@ -263,20 +263,20 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC overflowOK = true ) - test("dynamic: writeBytes(Array[Byte])")(forAll { bytes: Array[Byte] => + test("dynamic: writeBytes(Array[Byte])")(forAll { (bytes: Array[Byte]) => val bw = BufByteWriter.dynamic() val buf = bw.writeBytes(bytes).owned() assert(buf == Buf.ByteArray.Owned(bytes)) }) - test("dynamic: writeBytes(Buf)")(forAll { arr: Array[Byte] => + test("dynamic: writeBytes(Buf)")(forAll { (arr: Array[Byte]) => val bytes = Buf.ByteArray.Owned(arr) val bw = BufByteWriter.dynamic() val buf = bw.writeBytes(bytes).owned() assert(buf == bytes) }) - test("dynamic: writeBytes(Array[Byte]) 3 times")(forAll { bytes: Array[Byte] => + test("dynamic: writeBytes(Array[Byte]) 3 times")(forAll { (bytes: Array[Byte]) => val bw = BufByteWriter.dynamic() val buf = bw .writeBytes(bytes) @@ -286,7 +286,7 @@ final class BufByteWriterTest extends AnyFunSuite with ScalaCheckDrivenPropertyC assert(buf == Buf.ByteArray.Owned(bytes ++ bytes ++ bytes)) }) - test("dynamic: writeBytes(Buf) 3 times")(forAll { arr: Array[Byte] => + test("dynamic: writeBytes(Buf) 3 times")(forAll { (arr: Array[Byte]) => val bytes = Buf.ByteArray.Owned(arr) val bw = BufByteWriter.dynamic() val buf = bw diff --git a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala index c7bb9a4393..ccdca2b0aa 100644 --- a/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufReaderTest.scala @@ -13,7 +13,7 @@ class BufReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) test("BufReader - apply(buf)") { - forAll { bytes: String => + forAll { (bytes: String) => val buf = Buf.Utf8(bytes) val r = BufReader(buf) assert(await(BufReader.readAll(r)) == buf) @@ -35,7 +35,7 @@ class BufReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { } test("BufReader - readAll") { - forAll { bytes: String => + forAll { (bytes: String) => val r = Reader.fromBuf(Buf.Utf8(bytes)) assert(await(BufReader.readAll(r)) == Buf.Utf8(bytes)) } @@ -85,7 +85,7 @@ class BufReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { } yield Buf.ByteArray.Owned(bytes) ) - forAll(getByteArrays) { buffers: Seq[Buf] => + forAll(getByteArrays) { (buffers: Seq[Buf]) => val buffersWithLength = buffers.map(buf => Buf.U32BE(buf.length).concat(buf)) val r = BufReader.framed(BufReader(Buf(buffersWithLength)), new BufReaderTest.U32BEFramer()) diff --git a/util-core/src/test/scala/com/twitter/io/BufTest.scala b/util-core/src/test/scala/com/twitter/io/BufTest.scala index c5156a3e3e..948c3b1293 100644 --- a/util-core/src/test/scala/com/twitter/io/BufTest.scala +++ b/util-core/src/test/scala/com/twitter/io/BufTest.scala @@ -1035,7 +1035,7 @@ class BufTest buf <- arbBuf.arbitrary } yield Buf(Seq.fill(n)(buf)) - forAll(Gen.listOf(bufGen)) { bufs: List[Buf] => + forAll(Gen.listOf(bufGen)) { (bufs: List[Buf]) => val concatLeft = bufs.foldLeft(Buf.Empty) { (l, r) => l.concat(r) } val concatRight = bufs.foldRight(Buf.Empty) { (l, r) => l.concat(r) } val constructor = Buf(bufs) diff --git a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala index 1239a160df..9b0618fcc2 100644 --- a/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ByteReaderTest.scala @@ -33,13 +33,13 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readByte")(forAll { byte: Byte => + test("readByte")(forAll { (byte: Byte) => val br = readerWith(byte) assert(br.readByte() == byte) intercept[UnderflowException] { br.readByte() } }) - test("readShortBE")(forAll { s: Short => + test("readShortBE")(forAll { (s: Short) => val br = readerWith( ((s >> 8) & 0xff).toByte, ((s) & 0xff).toByte @@ -50,7 +50,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readShortLE")(forAll { s: Short => + test("readShortLE")(forAll { (s: Short) => val br = readerWith( ((s) & 0xff).toByte, ((s >> 8) & 0xff).toByte @@ -62,7 +62,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readUnsignedMediumBE")(forAll { m: Int => + test("readUnsignedMediumBE")(forAll { (m: Int) => val br = readerWith( ((m >> 16) & 0xff).toByte, ((m >> 8) & 0xff).toByte, @@ -72,7 +72,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readUnsignedMediumLE")(forAll { m: Int => + test("readUnsignedMediumLE")(forAll { (m: Int) => val br = readerWith( ((m) & 0xff).toByte, ((m >> 8) & 0xff).toByte, @@ -82,7 +82,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readIntBE")(forAll { i: Int => + test("readIntBE")(forAll { (i: Int) => val br = readerWith( ((i >> 24) & 0xff).toByte, ((i >> 16) & 0xff).toByte, @@ -93,7 +93,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readIntLE")(forAll { i: Int => + test("readIntLE")(forAll { (i: Int) => val br = readerWith( ((i) & 0xff).toByte, ((i >> 8) & 0xff).toByte, @@ -104,7 +104,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readLongBE")(forAll { l: Long => + test("readLongBE")(forAll { (l: Long) => val br = readerWith( ((l >> 56) & 0xff).toByte, ((l >> 48) & 0xff).toByte, @@ -119,7 +119,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readLongLE")(forAll { l: Long => + test("readLongLE")(forAll { (l: Long) => val br = readerWith( ((l) & 0xff).toByte, ((l >> 8) & 0xff).toByte, @@ -134,41 +134,41 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readUnsignedByte")(forAll { b: Byte => + test("readUnsignedByte")(forAll { (b: Byte) => val br = new ByteReaderImpl(Buf.Empty) { override def readByte() = b } assert(br.readUnsignedByte() == (b & 0xff)) }) - test("readUnsignedShortBE")(forAll { s: Short => + test("readUnsignedShortBE")(forAll { (s: Short) => val br = new ByteReaderImpl(Buf.Empty) { override def readShortBE() = s } assert(br.readUnsignedShortBE() == (s & 0xffff)) }) - test("readUnsignedShortLE")(forAll { s: Short => + test("readUnsignedShortLE")(forAll { (s: Short) => val br = new ByteReaderImpl(Buf.Empty) { override def readShortLE() = s } assert(br.readUnsignedShortLE() == (s & 0xffff)) }) - test("readMediumBE")(forAll { i: Int => + test("readMediumBE")(forAll { (i: Int) => val m = maskMedium(i) val br = new ByteReaderImpl(Buf.Empty) { override def readUnsignedMediumBE() = m } val expected = if (m > SignedMediumMax) m | 0xff000000 else m assert(br.readMediumBE() == expected) }) - test("readMediumLE")(forAll { i: Int => + test("readMediumLE")(forAll { (i: Int) => val m = maskMedium(i) val br = new ByteReaderImpl(Buf.Empty) { override def readUnsignedMediumLE() = m } val expected = if (m > SignedMediumMax) m | 0xff000000 else m assert(br.readMediumLE() == expected) }) - test("readUnsignedIntBE")(forAll { i: Int => + test("readUnsignedIntBE")(forAll { (i: Int) => val br = new ByteReaderImpl(Buf.Empty) { override def readIntBE() = i } assert(br.readUnsignedIntBE() == (i & 0xffffffffL)) }) - test("readUnsignedIntLE")(forAll { i: Int => + test("readUnsignedIntLE")(forAll { (i: Int) => val br = new ByteReaderImpl(Buf.Empty) { override def readIntLE() = i } assert(br.readUnsignedIntLE() == (i & 0xffffffffL)) }) @@ -177,7 +177,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { .chooseNum(Long.MinValue, Long.MaxValue) .map(x => BigInt(x) + BigInt(2).pow(63)) - test("readUnsignedLongBE")(forAll(uInt64s) { bi: BigInt => + test("readUnsignedLongBE")(forAll(uInt64s) { (bi: BigInt) => val br = readerWith( ((bi >> 56) & 0xff).toByte, ((bi >> 48) & 0xff).toByte, @@ -192,7 +192,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { intercept[UnderflowException] { br.readByte() } }) - test("readUnsignedLongLE")(forAll(uInt64s) { bi1: BigInt => + test("readUnsignedLongLE")(forAll(uInt64s) { (bi1: BigInt) => val bi = bi1.abs val br = readerWith( ((bi) & 0xff).toByte, @@ -209,27 +209,27 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { }) // .equals is required to handle NaN - test("readFloatBE")(forAll { i: Int => + test("readFloatBE")(forAll { (i: Int) => val br = new ByteReaderImpl(Buf.Empty) { override def readIntBE() = i } assert(br.readFloatBE().equals(JFloat.intBitsToFloat(i))) }) - test("readFloatLE")(forAll { i: Int => + test("readFloatLE")(forAll { (i: Int) => val br = new ByteReaderImpl(Buf.Empty) { override def readIntLE() = i } assert(br.readFloatLE().equals(JFloat.intBitsToFloat(i))) }) - test("readDoubleBE")(forAll { l: Long => + test("readDoubleBE")(forAll { (l: Long) => val br = new ByteReaderImpl(Buf.Empty) { override def readLongBE() = l } assert(br.readDoubleBE().equals(JDouble.longBitsToDouble(l))) }) - test("readDoubleLE")(forAll { l: Long => + test("readDoubleLE")(forAll { (l: Long) => val br = new ByteReaderImpl(Buf.Empty) { override def readLongLE() = l } assert(br.readDoubleLE().equals(JDouble.longBitsToDouble(l))) }) - test("readBytes")(forAll { bytes: Array[Byte] => + test("readBytes")(forAll { (bytes: Array[Byte]) => val buf = Buf.ByteArray.Owned(bytes ++ bytes) val br = ByteReader(buf) intercept[IllegalArgumentException] { br.readBytes(-1) } @@ -238,7 +238,7 @@ class ByteReaderTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { assert(br.readBytes(1) == Buf.Empty) }) - test("readAll")(forAll { bytes: Array[Byte] => + test("readAll")(forAll { (bytes: Array[Byte]) => val buf = Buf.ByteArray.Owned(bytes ++ bytes) val br = ByteReader(buf) assert(br.readAll() == Buf.ByteArray.Owned(bytes ++ bytes)) diff --git a/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala b/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala index c40715fbc4..2f059a27d5 100644 --- a/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/IteratorReaderTest.scala @@ -10,14 +10,14 @@ class IteratorReaderTest extends AnyFunSuite with Checkers { private def await[A](f: Future[A]): A = Await.result(f, 5.seconds) test("IteratorReader") { - check { list: List[String] => + check { (list: List[String]) => val r = Reader.fromIterator(list.iterator) await(Reader.readAllItems(r)).mkString == list.mkString } } test("IteratorReader - discard") { - check { list: List[String] => + check { (list: List[String]) => val r = Reader.fromIterator(list.iterator) r.discard() diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 27460ae197..376e73843a 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -63,7 +63,7 @@ class ReaderTest } test("Reader.concat collection of readers") { - forAll { ss: List[String] => + forAll { (ss: List[String]) => val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) val concatedReaders = Reader.concat(readers) val values = Reader.readAllItems(concatedReaders) @@ -72,7 +72,7 @@ class ReaderTest } test("Reader.concat from a collection won't force the stream") { - forAll { ss: List[String] => + forAll { (ss: List[String]) => val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) val head = Reader.value("hmm") Reader.concat(head +: readers) @@ -81,7 +81,7 @@ class ReaderTest } test("Reader.concat from a stream of readers") { - forAll { ss: List[String] => + forAll { (ss: List[String]) => val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) val concatedReaders = Reader.concat(AsyncStream.fromSeq(readers)) val values = Reader.readAllItems(concatedReaders) @@ -90,7 +90,7 @@ class ReaderTest } test("Reader.concat from a stream of readers forces the stream") { - forAll { ss: List[String] => + forAll { (ss: List[String]) => val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) val head = Reader.value("hmm") Reader.concat(AsyncStream.fromSeq(head +: readers)) @@ -155,7 +155,7 @@ class ReaderTest } test("Reader.flatten") { - forAll { ss: List[String] => + forAll { (ss: List[String]) => val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) val value = Reader.readAllItems(Reader.flatten(Reader.fromSeq(readers))) assert(await(value) == ss) @@ -163,7 +163,7 @@ class ReaderTest } test("Reader#flatten") { - forAll { ss: List[String] => + forAll { (ss: List[String]) => val readers: List[Reader[String]] = ss.map(s => Reader.value(s)) val buf = Reader.readAllItems((Reader.fromSeq(readers).flatten)) assert(await(buf) == ss) @@ -250,7 +250,7 @@ class ReaderTest val r1 = Reader.value(1) val p2 = new Promise - val i2 = p2.interruptible + val i2 = p2.interruptible() val r2 = Reader.fromFuture(i2) def rStreams: LazyList[Reader[Any]] = r1 #:: r2 #:: rStreams @@ -332,7 +332,7 @@ class ReaderTest } test("Reader.toAsyncStream") { - forAll { l: List[Byte] => + forAll { (l: List[Byte]) => val buf = Buf.ByteArray.Owned(l.toArray) val as = Reader.toAsyncStream(Reader.fromBuf(buf, 1)) @@ -341,7 +341,7 @@ class ReaderTest } test("Reader.flatMap") { - forAll { s: String => + forAll { (s: String) => val reader1 = Reader.fromBuf(Buf.Utf8(s), 8) val reader2 = reader1.flatMap { buf => val pipe = new Pipe[Buf] @@ -430,7 +430,7 @@ class ReaderTest } test("Reader.value") { - forAll { a: AnyVal => + forAll { (a: AnyVal) => val r = Reader.value(a) assert(await(r.read()) == Some(a)) assert(r.onClose.isDefined == false) @@ -440,7 +440,7 @@ class ReaderTest } test("Reader.exception") { - forAll { ex: Exception => + forAll { (ex: Exception) => val r = Reader.exception(ex) val exr = intercept[Exception] { await(r.read()) @@ -468,7 +468,7 @@ class ReaderTest } test("Reader.map") { - forAll { l: List[Int] => + forAll { (l: List[Int]) => val pipe = new Pipe[Int] writeLoop(l, pipe) val reader2 = pipe.map(_.toString) diff --git a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala index 44693c17ac..4bec850cb5 100644 --- a/util-core/src/test/scala/com/twitter/util/ActivityTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ActivityTest.scala @@ -85,24 +85,24 @@ class ActivityTest extends AnyFunSuite { } test("Activity.future: produce an initially-pending Activity") { - assert(Activity.future(Future.never).run.sample == Activity.Pending) + assert(Activity.future(Future.never).run.sample() == Activity.Pending) } test("Activity.future: produce an Activity that completes on success of the original Future") { val p = new Promise[Int] val act = Activity.future(p) - assert(act.run.sample == Activity.Pending) + assert(act.run.sample() == Activity.Pending) p.setValue(4) - assert(act.run.sample == Activity.Ok(4)) + assert(act.run.sample() == Activity.Ok(4)) } test("Activity.future: produce an Activity that fails on failure of the original Future") { val p = new Promise[Unit] val e = new Exception("gooby pls") val act = Activity.future(p) - assert(act.run.sample == Activity.Pending) + assert(act.run.sample() == Activity.Pending) p.setException(e) - assert(act.run.sample == Activity.Failed(e)) + assert(act.run.sample() == Activity.Failed(e)) } test( @@ -279,7 +279,8 @@ class ActivityTest extends AnyFunSuite { assert(a.sample() == 1) } - test("Activity#stabilize will ignore new values until it is observed again after deregistration") { + test( + "Activity#stabilize will ignore new values until it is observed again after deregistration") { val ex = new Exception val v = Var[Activity.State[Int]](Activity.Ok(1)) val a = Activity(v).stabilize @@ -302,7 +303,8 @@ class ActivityTest extends AnyFunSuite { assert(a.sample() == 1) } - test("Activity#stabilize will see new valid values immediately on observation after deregistration") { + test( + "Activity#stabilize will see new valid values immediately on observation after deregistration") { val ex = new Exception val v = Var[Activity.State[Int]](Activity.Ok(1)) val a = Activity(v).stabilize @@ -326,7 +328,7 @@ class ActivityTest extends AnyFunSuite { def register(witness: Witness[Activity.State[Int]]): Closable = { count += 1 w = witness - Closable.make { deadline: Time => + Closable.make { (deadline: Time) => count -= 1 w = null Future.Done diff --git a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala index e90007845a..a8c3422153 100644 --- a/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala +++ b/util-core/src/test/scala/com/twitter/util/Base64LongTest.scala @@ -65,7 +65,7 @@ class Base64LongTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { assert(toBase64(0) == "A") val b = new StringBuilder - forAll { a: Alphabet => + forAll { (a: Alphabet) => b.setLength(0) toBase64(b, 0, a) assert(b.result == a(0).toString) @@ -77,7 +77,7 @@ class Base64LongTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { } test("toBase64 uses the expected number of digits") { - BoundaryValues.foreach { n: Long => assert(toBase64(n).length == expectedLength(n)) } + BoundaryValues.foreach { (n: Long) => assert(toBase64(n).length == expectedLength(n)) } forAll((n: Long) => assert(toBase64(n).length == expectedLength(n))) val b = new StringBuilder diff --git a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala index 14d150218d..8b23f172f9 100644 --- a/util-core/src/test/scala/com/twitter/util/CancellableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/CancellableTest.scala @@ -21,7 +21,7 @@ class CancellableTest extends AnyFunSuite { test("CancellableSink cancel normal case") { var count = 2 def square(): Unit = count *= count - val s = new CancellableSink(square) + val s = new CancellableSink(square()) assert(count == 2) s.cancel() assert(count == 4) @@ -30,7 +30,7 @@ class CancellableTest extends AnyFunSuite { test("CancellableSink cancel once") { var count = 0 def increment(): Unit = count += 1 - val s = new CancellableSink(increment) + val s = new CancellableSink(increment()) s.cancel() assert(count == 1) s.cancel() @@ -40,7 +40,7 @@ class CancellableTest extends AnyFunSuite { test("CancellableSink has not cancelled") { var count = 0 def increment(): Unit = count += 1 - val s = new CancellableSink(increment) + val s = new CancellableSink(increment()) assert(!s.isCancelled) assert(count == 0) } @@ -48,7 +48,7 @@ class CancellableTest extends AnyFunSuite { test("CancellableSink confirm cancelled") { var count = 0 def increment(): Unit = count += 1 - val s = new CancellableSink(increment) + val s = new CancellableSink(increment()) s.cancel() assert(s.isCancelled) assert(count == 1) @@ -57,8 +57,8 @@ class CancellableTest extends AnyFunSuite { test("CancellableSink not support linking") { var count = 1 def multiply(): Unit = count *= 2 - val s1 = new CancellableSink(multiply) - val s2 = new CancellableSink(multiply) + val s1 = new CancellableSink(multiply()) + val s2 = new CancellableSink(multiply()) assertThrows[Exception] { s1.linkTo(s2) } diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index 954ee649d1..f981eb7722 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -901,8 +901,8 @@ class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenProp } } - testJavaFuture("toJavaFuture", { f: Future[Int] => f.toJavaFuture }) - testJavaFuture("toCompletableFuture", { f: Future[Int] => f.toCompletableFuture }) + testJavaFuture("toJavaFuture", { (f: Future[Int]) => f.toJavaFuture }) + testJavaFuture("toCompletableFuture", { (f: Future[Int]) => f.toCompletableFuture }) "monitored" should { trait MonitoredHelper { diff --git a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala index 2294a8fbab..c84ca34614 100644 --- a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala @@ -47,7 +47,7 @@ class MemoizeTest extends AnyFunSuite { val callCount = new AtomicInteger(0) val startUpLatch = new JavaCountDownLatch(1) - val memoizer = Memoize { i: Int => + val memoizer = Memoize { (i: Int) => // Wait for all of the threads to be started before // continuing. This gives races a chance to happen. startUpLatch.await() @@ -85,7 +85,7 @@ class MemoizeTest extends AnyFunSuite { // A computation that should fail the first time, and then // succeed for all subsequent attempts. - val memo = Memoize { i: Int => + val memo = Memoize { (i: Int) => // Ensure that all of the callers have been started startUpLatch.await(200, TimeUnit.MILLISECONDS) // This effect should happen once per exception plus once for diff --git a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala index 3a42c01e27..a057a6ae1d 100644 --- a/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StorageUnitTest.scala @@ -22,11 +22,11 @@ class StorageUnitTest extends AnyFunSuite { } test("StorageUnit: should confer an essential humanity") { - assert(900.bytes.toHuman == "900 B") - assert(1.kilobyte.toHuman == "1024 B") - assert(2.kilobytes.toHuman == "2.0 KiB") - assert(Int.MaxValue.bytes.toHuman == "2.0 GiB") - assert(Long.MaxValue.bytes.toHuman == "8.0 EiB") + assert(900.bytes.toHuman() == "900 B") + assert(1.kilobyte.toHuman() == "1024 B") + assert(2.kilobytes.toHuman() == "2.0 KiB") + assert(Int.MaxValue.bytes.toHuman() == "2.0 GiB") + assert(Long.MaxValue.bytes.toHuman() == "8.0 EiB") } test("StorageUnit: should handle Long value") { @@ -51,7 +51,7 @@ class StorageUnitTest extends AnyFunSuite { test("StorageUnit: should deal with negative values") { assert(-123.bytes.inBytes == -123) - assert(-2.kilobytes.toHuman == "-2.0 KiB") + assert(-2.kilobytes.toHuman() == "-2.0 KiB") } test("StorageUnit: should min properly") { diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 56ecac4a01..0bd395311b 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -605,13 +605,13 @@ class TimeTest "fromFractionalSeconds" in { val tolerance = 2.microseconds // we permit 1us slop - forAll { i: Int => + forAll { (i: Int) => assert( Time.fromSeconds(i).moreOrLessEquals(Time.fromFractionalSeconds(i.toDouble), tolerance) ) } - forAll { d: Double => + forAll { (d: Double) => val magic = 9223372036854775L // cribbed from Time.fromMicroseconds val microseconds = d * 1.second.inMicroseconds whenever(microseconds > -magic && microseconds < magic) { @@ -623,7 +623,7 @@ class TimeTest } } - forAll { l: Long => + forAll { (l: Long) => val seconds: Double = l.toDouble / 1.second.inNanoseconds assert( Time.fromFractionalSeconds(seconds).moreOrLessEquals(Time.fromNanoseconds(l), tolerance) diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 463ef8528d..9330e4d973 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -225,7 +225,7 @@ class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { } test("Var.collect: empty") { - assert(Var.collect(List.empty[Var[Int]]).sample == Nil) + assert(Var.collect(List.empty[Var[Int]]).sample() == Nil) } test("Var.collect[Seq]") { From 474c384c1c7b798aa658493cad0bee7652e3e33e Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 30 Jul 2021 20:52:22 +0000 Subject: [PATCH 524/796] Twitter-oss: Prepare OSS libraries for release 21.7.0 Problem We want to release the next versions of our Twitter OSS libraries 21.7.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D713326 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b6c3b4f7a9..529eb95fe7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.7.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 8f58fb8aef..d796959569 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.6.0" +val utilCore = "com.twitter" %% "util-core" % "21.7.0" ``` # Units diff --git a/build.sbt b/build.sbt index c8fdfe0642..ef540cdde3 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.7.0-SNAPSHOT" +val releaseVersion = "21.7.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 5b960dd148ed0f23e75731259088cf3ae0d6b378 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 30 Jul 2021 21:38:11 +0000 Subject: [PATCH 525/796] util/util-lint: scala 3 migration **Problem** Build util-lint with Scala 3 **Solution** # Change util-lint project settings. # Compile project + test with Scala 3 migration flags # Remove flags # Fix incompatibilities # Run cross-building tests **Result** Code changes were made in **GlobalRulesTest** to resolve a deprecation warning since `iterable.seq` returns the iterable itself. JIRA Issues: CSL-11087 Differential Revision: https://phabricator.twitter.biz/D698954 --- CHANGELOG.rst | 7 ++++--- build.sbt | 2 +- .../test/scala/com/twitter/util/lint/GlobalRulesTest.scala | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 529eb95fe7..e0e84af727 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,20 +13,21 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-app: Introduce a new `Command` class which provides a `Reader` interface to the output + of a shell command. ``PHAB_ID=D686134`` + * util-core: Now builds with Scala 3.0! ``PHAB_ID=D694775``. * util-core (BREAKING): Remove `AbstractSpool`. Java users should use `Spools` static class or the Spool companion object to create instances of `Spool`. -* util-app: Introduce a new `Command` class which provides a `Reader` interface to the output - of a shell command. ``PHAB_ID=D686134`` +* util-lint: Now builds with Scala 3.0! ``PHAB_ID=D698954`` Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ * util: Update ScalaCheck to version 1.15.4 ``PHAB_ID=D691691`` - Bug Fixes ~~~~~~~~~ diff --git a/build.sbt b/build.sbt index ef540cdde3..1e9cf501fa 100644 --- a/build.sbt +++ b/build.sbt @@ -481,7 +481,7 @@ lazy val utilLint = Project( id = "util-lint", base = file("util-lint") ).settings( - sharedSettings + sharedScala3Settings ).settings( name := "util-lint" ).dependsOn(utilCore) diff --git a/util-lint/src/test/scala/com/twitter/util/lint/GlobalRulesTest.scala b/util-lint/src/test/scala/com/twitter/util/lint/GlobalRulesTest.scala index 4ff541ff0f..f9a0ed6105 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/GlobalRulesTest.scala +++ b/util-lint/src/test/scala/com/twitter/util/lint/GlobalRulesTest.scala @@ -32,7 +32,7 @@ class GlobalRulesTest extends AnyFunSuite { // alwaysRule should not be present assert(GlobalRules.get.iterable.size == 1) // should just be neverRule - assert(GlobalRules.get.iterable.seq.head == neverRule) + assert(GlobalRules.get.iterable.head == neverRule) } } From 895ff00055a203e43b28f928490bf1fcc543c095 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 30 Jul 2021 21:57:03 +0000 Subject: [PATCH 526/796] Twitter-oss: update OSS libraries post-release to 21.8.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.8.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D715563 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d796959569..85ebac626d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 1e9cf501fa..f684458137 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.7.0" +val releaseVersion = "21.8.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 94541fad54e3ca5195cb98988ce3850b8378b292 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 30 Jul 2021 22:04:26 +0000 Subject: [PATCH 527/796] util-stats,finagle,twitter-server: Labels want to be free Problem Right now, it's difficult to add new labels to metrics, because they're hardcoded in a case class. Solution Make labels just a simple Map[String, String], or a dictionary when we export it. JIRA Issues: CSL-11163 Differential Revision: https://phabricator.twitter.biz/D704479 --- .../finagle/stats/InMemoryStatsReceiver.scala | 12 ++++------- .../finagle/stats/exp/ExpressionLabels.scala | 19 ----------------- .../finagle/stats/exp/ExpressionSchema.scala | 21 ++++++++++++------- .../stats/InMemoryStatsReceiverTest.scala | 7 ++++--- .../stats/exp/ExpressionSchemaTest.scala | 20 +++++++++++++----- 5 files changed, 37 insertions(+), 42 deletions(-) delete mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 14c565495d..1ec8316dc1 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -1,7 +1,7 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} -import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} import java.io.PrintStream import java.util.Locale import java.util.concurrent.ConcurrentHashMap @@ -71,8 +71,8 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { val schemas: mutable.Map[Seq[String], MetricBuilder] = new ConcurrentHashMap[Seq[String], MetricBuilder]().asScala - val expressions: mutable.Map[String, ExpressionSchema] = - new ConcurrentHashMap[String, ExpressionSchema]().asScala + val expressions: mutable.Map[ExpressionSchemaKey, ExpressionSchema] = + new ConcurrentHashMap[ExpressionSchemaKey, ExpressionSchema]().asScala override def counter(name: String*): ReadableCounter = counter(this.metricBuilder(CounterType).withName(name: _*)) @@ -263,11 +263,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { * Designed to match the behavior of Metrics::registerExpression(). */ override protected[finagle] def registerExpression(schema: ExpressionSchema): Unit = { - val expressionId = (schema.labels.serviceName match { - case Some(serviceName) => schema.name + "_" + serviceName - case None => schema.name - }) + schema.namespace.mkString("_") - expressions.put(expressionId, schema) + expressions.put(schema.schemaKey, schema) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala deleted file mode 100644 index 5185a093a3..0000000000 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionLabels.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.finagle.stats.exp - -import com.twitter.finagle.stats.{NoRoleSpecified, SourceRole} - -private[exp] object ExpressionLabels { - val empty = ExpressionLabels(None, None, NoRoleSpecified) -} - -/** - * ExpressionLabels provide service-related information - * - * @param processPath a universal coordinate for the resource - * @param serviceName a client label or a server label depends on the `role` - * @param role whether the service is playing the part of client or server regarding this metric - */ -private[exp] case class ExpressionLabels( - processPath: Option[String], - serviceName: Option[String], - role: SourceRole) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index 258a463b89..8a7ce39e38 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -6,7 +6,7 @@ import com.twitter.finagle.stats.{MetricUnit, SourceRole, StatsReceiver, Unspeci * ExpressionSchema is builder class that construct an expression with its metadata. * * @param name this is going to be an important query key when fetching expressions - * @param labels service related information, see [[ExpressionLabels]] + * @param labels service related information * @param namespace a list of namespaces the expression belongs to, this is usually * used to indicate a tenant in a multi-tenancy systems or similar concepts. * For standalone services, this should be empty. @@ -18,7 +18,7 @@ import com.twitter.finagle.stats.{MetricUnit, SourceRole, StatsReceiver, Unspeci */ case class ExpressionSchema private ( name: String, - labels: ExpressionLabels, + labels: Map[String, String], expr: Expression, namespace: Seq[String], bounds: Bounds, @@ -39,11 +39,14 @@ case class ExpressionSchema private ( def withNamespace(name: String*): ExpressionSchema = copy(namespace = this.namespace ++ name) + def withLabel(labelName: String, labelValue: String): ExpressionSchema = + copy(labels = labels + (labelName -> labelValue)) + private[finagle] def withRole(role: SourceRole): ExpressionSchema = - copy(labels = labels.copy(role = role)) + withLabel(ExpressionSchema.Role, role.toString) private[finagle] def withServiceName(name: String): ExpressionSchema = - copy(labels = labels.copy(serviceName = Some(name))) + withLabel(ExpressionSchema.ServiceName, name) def register(): Unit = { Expression.getStatsReceivers(expr).toSeq match { @@ -54,7 +57,7 @@ case class ExpressionSchema private ( } def schemaKey(): ExpressionSchemaKey = { - ExpressionSchemaKey(name, labels.serviceName, namespace) + ExpressionSchemaKey(name, labels, namespace) } } @@ -67,15 +70,19 @@ case class ExpressionSchema private ( */ case class ExpressionSchemaKey( name: String, - serviceName: Option[String], + labels: Map[String, String], namespaces: Seq[String]) // expose for testing in twitter-server private[twitter] object ExpressionSchema { + val Role: String = "role" + val ServiceName: String = "service_name" + val ProcessPath: String = "process_path" + def apply(name: String, expr: Expression): ExpressionSchema = ExpressionSchema( name = name, - labels = ExpressionLabels.empty, + labels = Map.empty, namespace = Seq.empty, expr = expr, bounds = Unbounded.get, diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 50f1af0b49..f78b47a2d4 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -1,7 +1,7 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} -import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema} +import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema, ExpressionSchemaKey} import java.io.{ByteArrayOutputStream, PrintStream} import java.nio.charset.StandardCharsets import org.scalatest.concurrent.{Eventually, IntegrationPatience} @@ -127,7 +127,7 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra "a", Expression( MetricBuilder(name = Seq("counter"), metricType = CounterType, statsReceiver = stats))) - stats.expressions.contains("a") + stats.expressions.contains(ExpressionSchemaKey("a", Map(), Nil)) } test("print") { @@ -393,6 +393,7 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra Expression(aaSchema).plus( Expression(bbSchema, Left(Expression.Min)).plus(Expression(ccSchema)))) - assert(sr.expressions.get("test_expression").get.expr == expected_expression.expr) + assert(sr.expressions + .get(ExpressionSchemaKey("test_expression", Map(), Nil)).get.expr == expected_expression.expr) } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 1dad70b3ed..5f4a037662 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -63,16 +63,26 @@ class ExpressionSchemaTest extends AnyFunSuite { runTheQuery(true) runTheQuery(false) - assert(sr.expressions("success_rate_downstream").name == successRate.name) - assert(sr.expressions("success_rate_downstream").expr == successRate.expr) - assert(sr.expressions("latency_downstream").name == latency.name) - assert(sr.expressions("latency_downstream").expr == latency.expr) + val downstreamLabel = + Map(ExpressionSchema.Role -> Client.toString, ExpressionSchema.ServiceName -> "downstream") + assert( + sr.expressions( + ExpressionSchemaKey("success_rate", downstreamLabel, Nil)).name == successRate.name) + assert( + sr.expressions( + ExpressionSchemaKey("success_rate", downstreamLabel, Nil)).expr == successRate.expr) + assert( + sr.expressions(ExpressionSchemaKey("latency", downstreamLabel, Nil)).name == latency.name) + assert( + sr.expressions(ExpressionSchemaKey("latency", downstreamLabel, Nil)).expr == latency.expr) assert(sr.counters(Seq("success")) == 1) assert(sr.counters(Seq("failures")) == 1) assert(sr.stats(Seq("latency")) == Seq(50, 50)) - assert(sr.expressions("success_rate_downstream").labels.role == Client) + assert( + sr.expressions(ExpressionSchemaKey("success_rate", downstreamLabel, Nil)).labels( + ExpressionSchema.Role) == Client.toString) } } From 39b0355db188fa24bac09bc6a6661f462c9301b4 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Wed, 4 Aug 2021 16:01:35 +0000 Subject: [PATCH 528/796] util/util-registry: scala 3 migration **Problem** Build util-registry with Scala 3 **Solution** # Change util-registry project settings. # Compile project + test with Scala 3 migration flags # Remove flags # Fix incompatibilities # Run cross-building tests Note: scalatestplus is split to use mockito 3-3 or 3-4 depending on Scala version until we universally migrate to mockito 3-4, which we will look into in this ticket: https://jira.twitter.biz/browse/CSL-11233. **Result** Roster test: Resolve a deprecation warning as method `anyObject` is deprecated, but its alias `any` is not. JIRA Issues: CSL-11082, CSL-11088 Differential Revision: https://phabricator.twitter.biz/D716019 --- CHANGELOG.rst | 2 ++ build.sbt | 17 +++++++++++++---- .../com/twitter/util/registry/RosterTest.scala | 6 +++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0e84af727..8cdc52a23c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -23,6 +23,8 @@ New Features * util-lint: Now builds with Scala 3.0! ``PHAB_ID=D698954`` +* util-registry: Now builds with Scala 3.0! ``PHAB_ID=D716019`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index f684458137..eb8cc2ad9f 100644 --- a/build.sbt +++ b/build.sbt @@ -513,13 +513,22 @@ lazy val utilRegistry = Project( id = "util-registry", base = file("util-registry") ).settings( - sharedSettings + sharedScala3Settings ).settings( name := "util-registry", libraryDependencies ++= Seq( - "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" - ) + "org.mockito" % "mockito-core" % mockitoVersion % "test", + ) ++ { + if (scalaVersion.value.startsWith("2")) { + Seq( + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test", + ) + } else { + Seq( + "org.scalatestplus" %% "mockito-3-4" % "3.2.9.0" % "test" + ) + } + } ).dependsOn(utilCore) lazy val utilRouting = Project( diff --git a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala index 46a864bc3d..1bfda82cc8 100644 --- a/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala +++ b/util-registry/src/test/scala/com/twitter/util/registry/RosterTest.scala @@ -2,7 +2,7 @@ package com.twitter.util.registry import java.util.logging.Logger import org.mockito.Mockito.{never, verify} -import org.mockito.ArgumentMatchers.anyObject +import org.mockito.ArgumentMatchers.any import org.scalatestplus.mockito.MockitoSugar import org.scalatest.funsuite.AnyFunSuite @@ -24,7 +24,7 @@ class RosterTest extends AnyFunSuite with MockitoSugar { Entry(Seq("foo", "bar"), "qux") ) assert(GlobalRegistry.get.toSet == expected) - verify(log, never).warning(anyObject[String]) + verify(log, never).warning(any[String]) } } @@ -35,7 +35,7 @@ class RosterTest extends AnyFunSuite with MockitoSugar { Entry(Seq("foo", "bar"), "fantastic") ) assert(GlobalRegistry.get.toSet == expected) - verify(log, never).warning(anyObject[String]) + verify(log, never).warning(any[String]) } } From 35b00359cf397c3f070f2ed2ba5c1affbd9a06b2 Mon Sep 17 00:00:00 2001 From: Natalia Margarita Gonzalez Herrera Date: Wed, 4 Aug 2021 18:24:44 +0000 Subject: [PATCH 529/796] util/util-codec: scala3 migration Problem We're migrating util-codec to scala3. Dotty has new syntax rules that need to be addressed. Solution Use migration flags to solve syntax warnings. In this case there was no need for any changes other that the sharedScala3Settings JIRA Issues: CSL-11107 Differential Revision: https://phabricator.twitter.biz/D715114 --- CHANGELOG.rst | 1 + build.sbt | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8cdc52a23c..e30f4fdf7a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,7 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-codec: Now builds with Scala 3.0! ``PHAB_ID=D715114``. * util-app: Introduce a new `Command` class which provides a `Reader` interface to the output of a shell command. ``PHAB_ID=D686134`` diff --git a/build.sbt b/build.sbt index eb8cc2ad9f..f253c21b55 100644 --- a/build.sbt +++ b/build.sbt @@ -81,12 +81,12 @@ def jdk11GcJavaOptions: Seq[String] = { } val defaultScala3Settings = Seq( - scalaVersion := "3.0.1", - crossScalaVersions := Seq("2.12.12", "2.13.6", "3.0.1") + scalaVersion := "3.0.1", + crossScalaVersions := Seq("2.12.12", "2.13.6", "3.0.1") ) val defaultScala2Settings = Seq( - scalaVersion := "2.13.6", - crossScalaVersions := Seq("2.12.12", "2.13.6") + scalaVersion := "2.13.6", + crossScalaVersions := Seq("2.12.12", "2.13.6") ) // Our dependencies or compiler options may differ for both Scala 2 and 3. We branch here @@ -101,13 +101,13 @@ val scala2Dependencies = scalaDependencies ++ Seq( "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test" ) val scala2cOptions = Seq( - "-target:jvm-1.8", - // Needs -missing-interpolator due to https://issues.scala-lang.org/browse/SI-8761 - "-Xlint:-missing-interpolator", - "-Yrangepos" + "-target:jvm-1.8", + // Needs -missing-interpolator due to https://issues.scala-lang.org/browse/SI-8761 + "-Xlint:-missing-interpolator", + "-Yrangepos" ) val scala3cOptions = Seq( - "-Xtarget:8" + "-Xtarget:8" ) val baseSettings = Seq( @@ -139,8 +139,8 @@ val baseSettings = Seq( "-encoding", "utf8", ) ++ { - if (scalaVersion.value.startsWith("2")) scala2cOptions - else scala3cOptions + if (scalaVersion.value.startsWith("2")) scala2cOptions + else scala3cOptions }, // Note: Use -Xlint rather than -Xlint:unchecked when TestThriftStructure // warnings are resolved @@ -331,7 +331,7 @@ lazy val utilCodec = Project( id = "util-codec", base = file("util-codec") ).settings( - sharedSettings + sharedScala3Settings ).settings( name := "util-codec" ).dependsOn(utilCore) From 5f67cddb366e0fe906930740652bae64c486dc37 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Wed, 4 Aug 2021 18:24:58 +0000 Subject: [PATCH 530/796] util/util-app-lifecycle: scala 3 migration **Problem** Build util-app-lifecycle with Scala 3 **Solution** # Change util-app-lifecycle project settings. # Compile project + test with Scala 3 migration flags # Remove flags # Fix incompatibilities # Run cross-building tests JIRA Issues: CSL-11082, CSL-11089 Differential Revision: https://phabricator.twitter.biz/D716444 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e30f4fdf7a..cf955aa1e5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,8 @@ New Features * util-app: Introduce a new `Command` class which provides a `Reader` interface to the output of a shell command. ``PHAB_ID=D686134`` +* util-app-lifecycle: Now builds with Scala 3.0! ``PHAB_ID=D716444`` + * util-core: Now builds with Scala 3.0! ``PHAB_ID=D694775``. * util-core (BREAKING): Remove `AbstractSpool`. Java users should use `Spools` static class or diff --git a/build.sbt b/build.sbt index f253c21b55..1aefda67b7 100644 --- a/build.sbt +++ b/build.sbt @@ -269,7 +269,7 @@ lazy val utilAppLifecycle = Project( id = "util-app-lifecycle", base = file("util-app-lifecycle") ).settings( - sharedSettings + sharedScala3Settings ).settings( name := "util-app-lifecycle" ).dependsOn(utilCore) From b1608b5e165b3457bd73d9f9e2658890628442d4 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Wed, 4 Aug 2021 22:42:53 +0000 Subject: [PATCH 531/796] util/util-core: fix scala3 runtime problems Problem: Scala 3 will optimize away `private[this]` members that are not used within the class. This breaks the `DetachedPromise` implementation causing the tests and runtime usages of Promise to fail. Solution: Fix this by adding a private method that accesses this field so the compiler will not remove the field. Result: There's also a compiler bug in the 3.0.1 release when pattern matching on private inner classes. Let's fix a test that uses this pattern until the next Scala 3 release JIRA Issues: CSL-11232 Differential Revision: https://phabricator.twitter.biz/D718074 --- .../src/main/scala/com/twitter/util/Promise.scala | 5 +++++ .../src/test/scala/com/twitter/util/TimerTest.scala | 11 +++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 615aae8499..9f3ba788c7 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -187,6 +187,11 @@ object Promise { @volatile private[this] var alreadyDetached: Int = 0 + // We add this method so that the Scala 3 compiler doesn't optimize + // away the `alreadyDetached` field which isn't explictly used within + // the scope of this class + private[this] def detached: Boolean = alreadyDetached == 1 + def detach(): Boolean = unsafe.compareAndSwapInt(this, detachedFutureOffset, 0, 1) diff --git a/util-core/src/test/scala/com/twitter/util/TimerTest.scala b/util-core/src/test/scala/com/twitter/util/TimerTest.scala index aeb97c0026..6954a1190f 100644 --- a/util-core/src/test/scala/com/twitter/util/TimerTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimerTest.scala @@ -353,9 +353,16 @@ class TimerTest extends AnyFunSuite with MockitoSugar with Eventually with Integ assert(mockTimerLocalPropagation(timer, 99) == 99) } - private class SomeEx extends Exception - private def testTimerUsesLocalMonitor(timer: Timer): Unit = { + // Let's define this exception within this private method since + // the Scala 3.0.1 compiler removes the accessor method to the + // outer instance, `TimerTest`. This is needed when pattern matching + // on the exception in `Monitor.mk`. + // + // See CSL-11237 or https://github.com/lampepfl/dotty/issues/13096. + // The next Scala 3 release will fix this bug. + class SomeEx extends Exception + val seen = new AtomicInteger(0) val monitor = Monitor.mk { case _: SomeEx => From 73e3ab679eda24093c0c6806808e6ed6fe8fc8ae Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Thu, 5 Aug 2021 01:48:40 +0000 Subject: [PATCH 532/796] util: only cross build scala 3 Problem: Scala 2.13 libs may not play nicely with Scala 3 jars. Solution / Result: We should only cross build our libraries with scala 3 but still keep the scalaVersion of projects as 2.13.6 until we successfully crossbuild all of our libs with scala 3 JIRA Issues: CSL-11235 Differential Revision: https://phabricator.twitter.biz/D718313 --- build.sbt | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/build.sbt b/build.sbt index 1aefda67b7..3287ada7e6 100644 --- a/build.sbt +++ b/build.sbt @@ -80,13 +80,16 @@ def jdk11GcJavaOptions: Seq[String] = { ) } -val defaultScala3Settings = Seq( - scalaVersion := "3.0.1", - crossScalaVersions := Seq("2.12.12", "2.13.6", "3.0.1") +val _scalaVersion = "2.13.6" +val _crossScalaVersions = Seq("2.12.12", "2.13.6") + +val defaultScalaSettings = Seq( + scalaVersion := _scalaVersion, + crossScalaVersions := _crossScalaVersions ) -val defaultScala2Settings = Seq( - scalaVersion := "2.13.6", - crossScalaVersions := Seq("2.12.12", "2.13.6") +val defaultScala3EnabledSettings = Seq( + scalaVersion := _scalaVersion, + crossScalaVersions := _crossScalaVersions ++ Seq("3.0.1") ) // Our dependencies or compiler options may differ for both Scala 2 and 3. We branch here @@ -195,12 +198,12 @@ val baseSettings = Seq( val sharedSettings = baseSettings ++ - defaultScala2Settings ++ + defaultScalaSettings ++ Seq(libraryDependencies ++= scala2Dependencies) -val sharedScala3Settings = +val sharedScala3EnabledSettings = baseSettings ++ - defaultScala3Settings ++ + defaultScala3EnabledSettings ++ Seq(libraryDependencies ++= scala3Dependencies) lazy val noPublishSettings = Seq( @@ -269,7 +272,7 @@ lazy val utilAppLifecycle = Project( id = "util-app-lifecycle", base = file("util-app-lifecycle") ).settings( - sharedScala3Settings + sharedScala3EnabledSettings ).settings( name := "util-app-lifecycle" ).dependsOn(utilCore) @@ -331,7 +334,7 @@ lazy val utilCodec = Project( id = "util-codec", base = file("util-codec") ).settings( - sharedScala3Settings + sharedScala3EnabledSettings ).settings( name := "util-codec" ).dependsOn(utilCore) @@ -340,7 +343,7 @@ lazy val utilCore = Project( id = "util-core", base = file("util-core") ).settings( - sharedScala3Settings + sharedScala3EnabledSettings ).settings( name := "util-core", // Moved some code to 'concurrent-extra' to conform to Pants' 1:1:1 principle (https://www.pantsbuild.org/build_files.html#target-granularity) @@ -481,7 +484,7 @@ lazy val utilLint = Project( id = "util-lint", base = file("util-lint") ).settings( - sharedScala3Settings + sharedScala3EnabledSettings ).settings( name := "util-lint" ).dependsOn(utilCore) @@ -513,7 +516,7 @@ lazy val utilRegistry = Project( id = "util-registry", base = file("util-registry") ).settings( - sharedScala3Settings + sharedScala3EnabledSettings ).settings( name := "util-registry", libraryDependencies ++= Seq( From 3a93c8c78a9085ecbc92022602dfc1aa95109a82 Mon Sep 17 00:00:00 2001 From: Natalia Margarita Gonzalez Herrera Date: Thu, 5 Aug 2021 16:37:00 +0000 Subject: [PATCH 533/796] util/util-thrift: scala3 migration Problem We're migrating util-codec to scala3. Dotty has new syntax rules that need to be addressed. Solution Use migration flags to solve syntax warnings, in this case there was no need for any syntax changes. While Manifest has not been dropped in scala3, it does not work the same way as it did in scala2, for this reason we are using ClassTag now JIRA Issues: CSL-11108 Differential Revision: https://phabricator.twitter.biz/D715129 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- .../main/scala/com/twitter/util/ThriftCodec.scala | 13 +++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf955aa1e5..5e61fada3e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,11 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-thrift: Now builds with Scala 3.0! ``PHAB_ID=D715129``. + +* util-thrift (BREAKING): ThriftCodec now uses ClassTag instead of Manifest. In scala3 Manifest is + intended for use by the compiler and should not be used in client code. ``PHAB_ID=D715129`` + * util-codec: Now builds with Scala 3.0! ``PHAB_ID=D715114``. * util-app: Introduce a new `Command` class which provides a `Reader` interface to the output diff --git a/build.sbt b/build.sbt index 3287ada7e6..8c0ef198ff 100644 --- a/build.sbt +++ b/build.sbt @@ -635,7 +635,7 @@ lazy val utilThrift = Project( id = "util-thrift", base = file("util-thrift") ).settings( - sharedSettings + sharedScala3Settings ).settings( name := "util-thrift", libraryDependencies ++= Seq( diff --git a/util-thrift/src/main/scala/com/twitter/util/ThriftCodec.scala b/util-thrift/src/main/scala/com/twitter/util/ThriftCodec.scala index 80e97c7ce0..6fc3d4ad12 100644 --- a/util-thrift/src/main/scala/com/twitter/util/ThriftCodec.scala +++ b/util-thrift/src/main/scala/com/twitter/util/ThriftCodec.scala @@ -2,21 +2,22 @@ package com.twitter.util import org.apache.thrift.TBase import org.apache.thrift.protocol.{TBinaryProtocol, TCompactProtocol, TProtocolFactory} +import scala.reflect.{ClassTag, classTag} object ThriftCodec { - def apply[T <: TBase[_, _]: Manifest, P <: TProtocolFactory: Manifest]: ThriftCodec[T, P] = + def apply[T <: TBase[_, _]: ClassTag, P <: TProtocolFactory: ClassTag]: ThriftCodec[T, P] = new ThriftCodec[T, P] } -class ThriftCodec[T <: TBase[_, _]: Manifest, P <: TProtocolFactory: Manifest] +class ThriftCodec[T <: TBase[_, _]: ClassTag, P <: TProtocolFactory: ClassTag] extends Codec[T, Array[Byte]] with ThriftSerializer { protected lazy val prototype: T = - manifest[T].runtimeClass.asInstanceOf[Class[T]].newInstance + classTag[T].runtimeClass.asInstanceOf[Class[T]].newInstance lazy val protocolFactory: TProtocolFactory = - manifest[P].runtimeClass.asInstanceOf[Class[P]].newInstance + classTag[P].runtimeClass.asInstanceOf[Class[P]].newInstance def encode(item: T): Array[Byte] = toBytes(item) @@ -27,7 +28,7 @@ class ThriftCodec[T <: TBase[_, _]: Manifest, P <: TProtocolFactory: Manifest] } } -class BinaryThriftCodec[T <: TBase[_, _]: Manifest] extends ThriftCodec[T, TBinaryProtocol.Factory] +class BinaryThriftCodec[T <: TBase[_, _]: ClassTag] extends ThriftCodec[T, TBinaryProtocol.Factory] -class CompactThriftCodec[T <: TBase[_, _]: Manifest] +class CompactThriftCodec[T <: TBase[_, _]: ClassTag] extends ThriftCodec[T, TCompactProtocol.Factory] From 11a917dc0538a32f71665893d83293894179af38 Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Thu, 5 Aug 2021 18:40:03 +0000 Subject: [PATCH 534/796] util: fix missed settings update in build.sbt Problem/Solution One of the Scala 3 settings values was not updated and the build is broken as a result. Let's fix it. Differential Revision: https://phabricator.twitter.biz/D718902 --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 8c0ef198ff..ab8e4b5641 100644 --- a/build.sbt +++ b/build.sbt @@ -520,7 +520,7 @@ lazy val utilRegistry = Project( ).settings( name := "util-registry", libraryDependencies ++= Seq( - "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.mockito" % "mockito-core" % mockitoVersion % "test", ) ++ { if (scalaVersion.value.startsWith("2")) { Seq( @@ -635,7 +635,7 @@ lazy val utilThrift = Project( id = "util-thrift", base = file("util-thrift") ).settings( - sharedScala3Settings + sharedScala3EnabledSettings ).settings( name := "util-thrift", libraryDependencies ++= Seq( From b9ef5d06153c7f4459cdf57ebec8852ec3a0a59c Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 6 Aug 2021 21:36:23 +0000 Subject: [PATCH 535/796] util/util-hashing: scala 3 migration **Problem** Build util-hashing with Scala 3 **Solution** # Change util-hashing project settings. # Compile project + test with Scala 3 migration flags # Remove flags # Fix incompatibilities # Run cross-building tests **Result** //Auto-rewrite fixes:// Hashable.scala: - Scala 3 requires parentheses around the parameter of a lambda ConsistentHashingDistributorTest.scala: - Scala 3 requires parentheses around the parameter of a lambda - Do/while loops need to be rewritten HashableTest.scala: - Scala 3 requires parentheses around the parameter of a lambda //Other fixes:// KeyHasherTest.scala: - Scala3 requires explicit empty argument list to invoke method JIRA Issues: CSL-11082, CSL-11090 Differential Revision: https://phabricator.twitter.biz/D718914 --- CHANGELOG.rst | 2 ++ build.sbt | 17 ++++++++++++---- .../scala/com/twitter/hashing/Hashable.scala | 2 +- .../ConsistentHashingDistributorTest.scala | 20 +++++++++---------- .../com/twitter/hashing/HashableTest.scala | 2 +- .../com/twitter/hashing/KeyHasherTest.scala | 2 +- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5e61fada3e..d3816b3418 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -29,6 +29,8 @@ New Features * util-core (BREAKING): Remove `AbstractSpool`. Java users should use `Spools` static class or the Spool companion object to create instances of `Spool`. +* util-hashing: Now builds with Scala 3.0! ``PHAB_ID=D718914`` + * util-lint: Now builds with Scala 3.0! ``PHAB_ID=D698954`` * util-registry: Now builds with Scala 3.0! ``PHAB_ID=D716019`` diff --git a/build.sbt b/build.sbt index ab8e4b5641..c10ff3162f 100644 --- a/build.sbt +++ b/build.sbt @@ -421,13 +421,22 @@ lazy val utilHashing = Project( id = "util-hashing", base = file("util-hashing") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-hashing", libraryDependencies ++= Seq( - scalacheckLib, - "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" - ) + scalacheckLib + ) ++ { + if (scalaVersion.value.startsWith("2")) { + Seq( + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", + ) + } else { + Seq( + "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", + ) + } + }, ).dependsOn(utilCore % "test") lazy val utilJacksonAnnotations = Project( diff --git a/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala b/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala index 233e713fb8..9ffc61f392 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/Hashable.scala @@ -21,7 +21,7 @@ trait LowPriorityHashable { // Get the UTF-8 bytes of a string to hash it implicit def fromString[T](implicit h: Hashable[Array[Byte], T]): Hashable[String, T] = - h.compose { s: String => s.getBytes } + h.compose { (s: String) => s.getBytes } } object Hashable extends LowPriorityHashable { diff --git a/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala index 3d4868aa5a..bed6f207f3 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala @@ -33,15 +33,13 @@ class ConsistentHashingDistributorTest extends AnyWordSpec with ScalaCheckDriven val stream = getClass.getClassLoader.getResourceAsStream("ketama_results") val reader = new BufferedReader(new InputStreamReader(stream)) val expected = new mutable.ListBuffer[Array[String]] - var line: String = null - do { - line = reader.readLine - if (line != null) { - val segments = line.split(" ") - assert(segments.length == 4) - expected += segments - } - } while (line != null) + var line: String = reader.readLine() + while (line != null) { + val segments = line.split(" ") + assert(segments.length == 4) + expected += segments + line = reader.readLine() + } assert(expected.size == 99) // Test that ketamaClient.clientOf(key) == expected IP @@ -86,7 +84,7 @@ class ConsistentHashingDistributorTest extends AnyWordSpec with ScalaCheckDriven "byteArrayToLE" in { val ketama = new ConsistentHashingDistributor[Unit](Seq.empty, 0, false) - forAll { s: String => + forAll { (s: String) => val ba = MessageDigest.getInstance("MD5").digest(s.getBytes("UTF-8")) val bb = ByteBuffer.wrap(ba).order(ByteOrder.LITTLE_ENDIAN) @@ -110,7 +108,7 @@ class ConsistentHashingDistributorTest extends AnyWordSpec with ScalaCheckDriven assert(pid1 == pid2) } - forAll { s: String => + forAll { (s: String) => val hash = keyHasher.hashKey(s.getBytes("UTF-8")) val (pid1, _) = ketamaDistributor.entryForHash(hash) val pid2 = ketamaDistributor.partitionIdForHash(hash) diff --git a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala index 75af619d4c..2a76d0196f 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/HashableTest.scala @@ -18,7 +18,7 @@ class HashableTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { ) def testConsistency[T](algo: Hashable[Array[Byte], T]): Unit = { - forAll { input: Array[Byte] => assert(algo(input) == algo(input)) } + forAll { (input: Array[Byte]) => assert(algo(input) == algo(input)) } } algorithms foreach { algo => diff --git a/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala index f334302983..2aff9d868d 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala @@ -10,7 +10,7 @@ class KeyHasherTest extends AnyWordSpec { def readResource(name: String) = { var lines = new ListBuffer[String]() val src = scala.io.Source.fromFile(TempFile.fromResourcePath(getClass, "/" + name)) - src.getLines + src.getLines() } val base64 = Base64.getDecoder From 0955e5c3e6e6075062e33510fc80c534108bfcd8 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 9 Aug 2021 15:32:16 +0000 Subject: [PATCH 536/796] util/util-core: resolve small deprecation warnings Problem: util-core has a few deprecation warnings around not providing the empty argument list explicitly to invoke the `result` method, Solution: Append '()' where the `result` method is used JIRA Issues: CSL-11220 Differential Revision: https://phabricator.twitter.biz/D719983 --- .../scala/com/twitter/concurrent/AsyncStream.scala | 10 +++++----- .../src/main/scala/com/twitter/concurrent/Spool.scala | 2 +- .../main/scala/com/twitter/conversions/MapOps.scala | 2 +- .../src/main/scala/com/twitter/util/Activity.scala | 2 +- util-core/src/main/scala/com/twitter/util/Future.scala | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala index 469b25e1ff..a26def1ace 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/AsyncStream.scala @@ -476,8 +476,8 @@ sealed abstract class AsyncStream[+A] { } go(this).transform { - case Throw(exc) => Future.value(buf.result -> Some(exc)) - case Return(_) => Future.value(buf.result -> None) + case Throw(exc) => Future.value(buf.result() -> Some(exc)) + case Return(_) => Future.value(buf.result() -> None) } } @@ -498,15 +498,15 @@ sealed abstract class AsyncStream[+A] { )( s: => AsyncStream[A] ): Future[(Seq[A], () => AsyncStream[A])] = - if (sizeRemaining < 1) Future.value((buffer.result, () => s)) + if (sizeRemaining < 1) Future.value((buffer.result(), () => s)) else s match { - case Empty => Future.value((buffer.result, () => s)) + case Empty => Future.value((buffer.result(), () => s)) case FromFuture(fa) => fa.flatMap { a => buffer += a - Future.value((buffer.result, () => empty)) + Future.value((buffer.result(), () => empty)) } case Cons(fa, more) => diff --git a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala index aedbe9c213..90caec8c0c 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/Spool.scala +++ b/util-core/src/main/scala/com/twitter/concurrent/Spool.scala @@ -260,7 +260,7 @@ sealed trait Spool[+A] { val as = Vector.newBuilder[A] foreach { a => as += a - } map { _ => as.result } + } map { _ => as.result() } } /** diff --git a/util-core/src/main/scala/com/twitter/conversions/MapOps.scala b/util-core/src/main/scala/com/twitter/conversions/MapOps.scala index 55f2c69f64..fdf562cffb 100644 --- a/util-core/src/main/scala/com/twitter/conversions/MapOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/MapOps.scala @@ -67,7 +67,7 @@ object MapOps { val invertedMap = immutable.Map.newBuilder[V, Seq[K]] for ((k, valueBuilder) <- invertedMapWithBuilderValues) { - invertedMap += (k -> valueBuilder.result) + invertedMap += (k -> valueBuilder.result()) } invertedMap.result() diff --git a/util-core/src/main/scala/com/twitter/util/Activity.scala b/util-core/src/main/scala/com/twitter/util/Activity.scala index b4671dc2af..75675b4285 100644 --- a/util-core/src/main/scala/com/twitter/util/Activity.scala +++ b/util-core/src/main/scala/com/twitter/util/Activity.scala @@ -256,7 +256,7 @@ object Activity { case _ => assert(false) } - Ok(ts.result) + Ok(ts.result()) } Activity(stateVar map flip) diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 301ca3ec35..f6a620a968 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1184,7 +1184,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( buf += h.liftToTry buf += iterator.next().liftToTry while (iterator.hasNext) buf += iterator.next().liftToTry - Future.collect(buf.result) + Future.collect(buf.result()) } else { Future.collect(List(h.liftToTry)) } @@ -1378,7 +1378,7 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( buf += f i += 1 } - buf.result + buf.result() } /** From 94f1fa3738fdbb5866571e83d303c65bd507de49 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 9 Aug 2021 20:57:02 +0000 Subject: [PATCH 537/796] util/util-app: scala 3 migration **Problem** Build util-app with Scala 3 **Solution** # Change util-app project settings. # Compile project + test with Scala 3 migration flags # Remove flags # Fix incompatibilities # Run cross-building tests **Result** //Auto-rewrite fixes:// Scala 3 requires parentheses around the parameter of a lambda: - ClassPath.scala - Flaggable.scala //Other fixes:// Scala3 requires explicit empty argument list to invoke method: - ClassPath.scala - AppTest.scala - LoadServiceTest.scala Manifest is no longer available in Scala 3, use ClassTag instead: - Flags.scala - GlobalFlag.scala JavaConverters is deprecated, use CollectionConverters instead: - CommandExecutor.scala - RealCommandTest.scala: JIRA Issues: CSL-11082, CSL-11091 Differential Revision: https://phabricator.twitter.biz/D714780 --- CHANGELOG.rst | 8 ++++++++ build.sbt | 15 ++++++++++++--- .../main/scala/com/twitter/app/ClassPath.scala | 4 ++-- .../main/scala/com/twitter/app/Flaggable.scala | 8 ++++---- .../src/main/scala/com/twitter/app/Flags.scala | 3 ++- .../main/scala/com/twitter/app/GlobalFlag.scala | 3 ++- .../com/twitter/app/command/CommandExecutor.scala | 2 +- .../src/test/scala/com/twitter/app/AppTest.scala | 2 +- .../scala/com/twitter/app/LoadServiceTest.scala | 4 ++-- .../com/twitter/app/command/RealCommandTest.scala | 2 +- 10 files changed, 35 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d3816b3418..9a35487b31 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,14 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ +* util-app: Now builds with Scala 3.0! ``PHAB_ID=D714780`` + +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ +* util-app: Flags and GlobalFlag now use ClassTag instead of Manifest. ``PHAB_ID=D714780`` + 21.7.0 ------ diff --git a/build.sbt b/build.sbt index c10ff3162f..411fea6752 100644 --- a/build.sbt +++ b/build.sbt @@ -259,13 +259,22 @@ lazy val utilApp = Project( id = "util-app", base = file("util-app") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-app", libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" - ) + ) ++ { + if (scalaVersion.value.startsWith("2")) { + Seq( + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" + ) + } else { + Seq( + "org.scalatestplus" %% "mockito-3-4" % "3.2.9.0" % "test" + ) + } + }, ).dependsOn(utilAppLifecycle, utilCore, utilRegistry) lazy val utilAppLifecycle = Project( diff --git a/util-app/src/main/scala/com/twitter/app/ClassPath.scala b/util-app/src/main/scala/com/twitter/app/ClassPath.scala index 261cfd4129..9dbf20e39c 100644 --- a/util-app/src/main/scala/com/twitter/app/ClassPath.scala +++ b/util-app/src/main/scala/com/twitter/app/ClassPath.scala @@ -60,7 +60,7 @@ private[app] sealed abstract class ClassPath[CpInfo <: ClassPath.Info] { browseUri0(uri, loader, buf, seenUris) seenUris += uri } - buf.result + buf.result() } // Note: In JDK 9+ URLClassLoader is no longer the default ClassLoader. @@ -71,7 +71,7 @@ private[app] sealed abstract class ClassPath[CpInfo <: ClassPath.Info] { // TODO - add suppport for the ModulePath after dropping JDK 8 support. private[this] def urlsFromClasspath(): Array[URL] = { val classpath: String = System.getProperty("java.class.path") - classpath.split(File.pathSeparator).map { pathEntry: String => + classpath.split(File.pathSeparator).map { (pathEntry: String) => Paths.get(pathEntry).toAbsolutePath().toUri().toURL } } diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index 43646031fd..d84696c59c 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -132,19 +132,19 @@ object Flaggable { implicit val ofInt: Flaggable[Int] = mandatory(_.toInt) implicit val ofJavaInteger: Flaggable[JInteger] = - mandatory { s: String => JInteger.valueOf(s.toInt) } + mandatory { (s: String) => JInteger.valueOf(s.toInt) } implicit val ofLong: Flaggable[Long] = mandatory(_.toLong) implicit val ofJavaLong: Flaggable[JLong] = - mandatory { s: String => JLong.valueOf(s.toLong) } + mandatory { (s: String) => JLong.valueOf(s.toLong) } implicit val ofFloat: Flaggable[Float] = mandatory(_.toFloat) implicit val ofJavaFloat: Flaggable[JFloat] = - mandatory { s: String => JFloat.valueOf(s.toFloat) } + mandatory { (s: String) => JFloat.valueOf(s.toFloat) } implicit val ofDouble: Flaggable[Double] = mandatory(_.toDouble) implicit val ofJavaDouble: Flaggable[JDouble] = - mandatory { s: String => JDouble.valueOf(s.toDouble) } + mandatory { (s: String) => JDouble.valueOf(s.toDouble) } // Conversions for common non-primitive types and collections. implicit val ofDuration: Flaggable[Duration] = mandatory(Duration.parse(_)) diff --git a/util-app/src/main/scala/com/twitter/app/Flags.scala b/util-app/src/main/scala/com/twitter/app/Flags.scala index f92faf104a..7fc4f87667 100644 --- a/util-app/src/main/scala/com/twitter/app/Flags.scala +++ b/util-app/src/main/scala/com/twitter/app/Flags.scala @@ -4,6 +4,7 @@ import scala.collection.immutable.TreeSet import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ +import scala.reflect.ClassTag import scala.util.control.NonFatal /** @@ -277,7 +278,7 @@ class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) * @param name The name of the flag. * @param help The help string of the flag. */ - def apply[T](name: String, help: String)(implicit _f: Flaggable[T], m: Manifest[T]): Flag[T] = { + def apply[T](name: String, help: String)(implicit _f: Flaggable[T], m: ClassTag[T]): Flag[T] = { val f = new Flag[T](name, help, m.toString, failFastUntilParsed) add(f) f diff --git a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala index 89c537a65e..bc57d68ab5 100644 --- a/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala +++ b/util-app/src/main/scala/com/twitter/app/GlobalFlag.scala @@ -2,6 +2,7 @@ package com.twitter.app import java.lang.reflect.Modifier import scala.util.control.NonFatal +import scala.reflect.ClassTag /** * Subclasses of GlobalFlag (that are defined in libraries) are "global" in the @@ -85,7 +86,7 @@ abstract class GlobalFlag[T] private[app] ( * * @param help documentation regarding usage of this [[Flag]]. */ - def this(help: String)(implicit _f: Flaggable[T], m: Manifest[T]) = + def this(help: String)(implicit _f: Flaggable[T], m: ClassTag[T]) = this(Right(m.toString), help) /** diff --git a/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala b/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala index a33452f46d..ee6c91cab7 100644 --- a/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala +++ b/util-app/src/main/scala/com/twitter/app/command/CommandExecutor.scala @@ -1,7 +1,7 @@ package com.twitter.app.command import java.io.File -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ /** * CommandExecutor is a private trait used for testing so that the actual forking of the diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index ea66b2fd1b..98e61256e6 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -205,7 +205,7 @@ class AppTest extends AnyFunSuite { assert(n2 == 0) assert(!f.isDefined) - p.setDone + p.setDone() assert(n2 == 1) assert(f.isDefined) } diff --git a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala index 5495e9bea5..d4aa02f791 100644 --- a/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala +++ b/util-app/src/test/scala/com/twitter/app/LoadServiceTest.scala @@ -109,7 +109,7 @@ class LoadServiceTest extends AnyFunSuite with MockitoSugar { f.setReadable(false) new LoadServiceClassPath().browseUri(f.toURI, loader, buf) - assert(buf.result.isEmpty) + assert(buf.result().isEmpty) f.delete() } } @@ -128,7 +128,7 @@ class LoadServiceTest extends AnyFunSuite with MockitoSugar { subDir.setReadable(false) new LoadServiceClassPath().browseUri(f.toURI, loader, buf) - assert(buf.result.isEmpty) + assert(buf.result().isEmpty) subDir.delete() f.delete() diff --git a/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala index ae03c0c4ab..72fd8ea805 100644 --- a/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala +++ b/util-app/src/test/scala/com/twitter/app/command/RealCommandTest.scala @@ -8,7 +8,7 @@ import java.nio.file.attribute.PosixFilePermission import java.nio.file.Files import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import scala.collection.JavaConverters._ +import scala.jdk.CollectionConverters._ object RealCommandTest { // Bazel provides the internal dependency as a JAR, and consequently, From 07cd74d70b5b9d4b032f2e0f060eb8fc8ccdb038 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Tue, 10 Aug 2021 16:37:59 +0000 Subject: [PATCH 538/796] util: Update changelog **Problem** The more recent util migration changes have been put under version 21.7.0, but they have not been released yet. **Solution** Move recent changes to unreleased section. JIRA Issues: CSL-11247 Differential Revision: https://phabricator.twitter.biz/D720565 --- CHANGELOG.rst | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9a35487b31..f1e5f6bca8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,39 +9,43 @@ Unreleased New Features ~~~~~~~~~~~~ -* util-app: Now builds with Scala 3.0! ``PHAB_ID=D714780`` + +* util-app: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D714780`` + +* util-app-lifecycle: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D716444`` + +* util-codec: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D715114`` + +* util-hashing: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D718914`` + +* util-lint: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D698954`` + +* util-registry: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D716019`` + +* util-thrift: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D715129`` Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ * util-app: Flags and GlobalFlag now use ClassTag instead of Manifest. ``PHAB_ID=D714780`` +* util-thrift: ThriftCodec now uses ClassTag instead of Manifest. In + scala3 Manifest is intended for use by the compiler and should not be used in + client code. ``PHAB_ID=D715129`` + 21.7.0 ------ New Features ~~~~~~~~~~~~ -* util-thrift: Now builds with Scala 3.0! ``PHAB_ID=D715129``. - -* util-thrift (BREAKING): ThriftCodec now uses ClassTag instead of Manifest. In scala3 Manifest is - intended for use by the compiler and should not be used in client code. ``PHAB_ID=D715129`` - -* util-codec: Now builds with Scala 3.0! ``PHAB_ID=D715114``. * util-app: Introduce a new `Command` class which provides a `Reader` interface to the output of a shell command. ``PHAB_ID=D686134`` -* util-app-lifecycle: Now builds with Scala 3.0! ``PHAB_ID=D716444`` - -* util-core: Now builds with Scala 3.0! ``PHAB_ID=D694775``. +* util-core: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D694775`` * util-core (BREAKING): Remove `AbstractSpool`. Java users should use `Spools` static class or - the Spool companion object to create instances of `Spool`. - -* util-hashing: Now builds with Scala 3.0! ``PHAB_ID=D718914`` - -* util-lint: Now builds with Scala 3.0! ``PHAB_ID=D698954`` + the Spool companion object to create instances of `Spool`. -* util-registry: Now builds with Scala 3.0! ``PHAB_ID=D716019`` Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ From 68fec217365543decb3e5bd4a907f2f3cf0b0c37 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 10 Aug 2021 20:01:53 +0000 Subject: [PATCH 539/796] util: crossbuild with the Scala 3.0.2-RC1 Problem: The Scala 3.0.1 compiler includes bugs that are patched in the 3.0.2-RC1 release. An example of this is the broken Java APIs in JDK 11 for our `TimeLikeOps` classes because of https://github.com/lampepfl/dotty/issues/13039. Result / Solution: Using the 3.0.2-RC1 release doesn't break any tests and our projects compile fine. Let's update to this to make use of the bug fixes until the 3.0.2 is officially released. The new Dotty scaladoc requires network access due to a bug https://github.com/lampepfl/dotty/issues/13272. For now let's disable scaladoc for scala 3. JIRA Issues: CSL-11235 Differential Revision: https://phabricator.twitter.biz/D720887 --- CHANGELOG.rst | 3 +-- build.sbt | 12 +++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f1e5f6bca8..55b8be1e1e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -44,8 +44,7 @@ New Features * util-core: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D694775`` * util-core (BREAKING): Remove `AbstractSpool`. Java users should use `Spools` static class or - the Spool companion object to create instances of `Spool`. - + the Spool companion object to create instances of `Spool`. ``PHAB_ID=D694775`` Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index 411fea6752..e405cbd24d 100644 --- a/build.sbt +++ b/build.sbt @@ -89,7 +89,7 @@ val defaultScalaSettings = Seq( ) val defaultScala3EnabledSettings = Seq( scalaVersion := _scalaVersion, - crossScalaVersions := _crossScalaVersions ++ Seq("3.0.1") + crossScalaVersions := _crossScalaVersions ++ Seq("3.0.2-RC1") ) // Our dependencies or compiler options may differ for both Scala 2 and 3. We branch here @@ -132,6 +132,16 @@ val baseSettings = Seq( case _ => sourceDir / "scala-2.13+" } }, + // Let's skip compiling docs for Scala 3 due a Dotty Scaladoc bug where generating the docs + // requires network access. See https://github.com/lampepfl/dotty/issues/13272 or CSL-11251. + Compile / doc / sources := { + if (scalaVersion.value.startsWith("3")) Seq.empty + else (Compile / doc / sources).value + }, + Compile / packageDoc / publishArtifact := { + if (scalaVersion.value.startsWith("3")) false + else (Compile / packageDoc / publishArtifact).value + }, ScoverageKeys.coverageHighlighting := true, resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", From d13072bba4bf634fa1d54f5e481f197e3e8e3ced Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Wed, 11 Aug 2021 19:43:42 +0000 Subject: [PATCH 540/796] util-jvm: export JVM expressions Problem/Solution: Lets add JVM expressions to the expressions endpoint JIRA Issues: CSL-11133 Differential Revision: https://phabricator.twitter.biz/D710867 --- .../main/scala/com/twitter/jvm/JvmStats.scala | 56 +++++++++++++++---- .../scala/com/twitter/jvm/JvmStatsTest.scala | 34 +++++++++++ 2 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 6254f007bb..7bb3bca842 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -2,7 +2,8 @@ package com.twitter.jvm import com.twitter.conversions.StringOps._ import com.twitter.finagle.stats.MetricBuilder.GaugeType -import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.stats.{Milliseconds, StatsReceiver} +import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema} import java.lang.management.{BufferPoolMXBean, ManagementFactory} import scala.collection.mutable import scala.jdk.CollectionConverters._ @@ -38,8 +39,9 @@ object JvmStats { gauges.add(threadStats.addGauge("peak_count") { threads.getPeakThreadCount().toLong }) val runtime = ManagementFactory.getRuntimeMXBean() + val uptime = stats.addGauge("uptime") { runtime.getUptime() } + gauges.add(uptime) gauges.add(stats.addGauge("start_time") { runtime.getStartTime() }) - gauges.add(stats.addGauge("uptime") { runtime.getUptime() }) val os = ManagementFactory.getOperatingSystemMXBean() gauges.add(stats.addGauge("num_cpus") { os.getAvailableProcessors().toLong }) @@ -126,18 +128,52 @@ object JvmStats { val gcStats = stats.scope("gc") gcPool.foreach { gc => val name = gc.getName.regexSub("""[^\w]""".r) { m => "_" } - gauges.add(gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "cycles") { - gc.getCollectionCount - }) - gauges.add(gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "msec") { + val poolCycles = + gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "cycles") { + gc.getCollectionCount + } + val poolMsec = gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "msec") { gc.getCollectionTime - }) + } + + ExpressionSchema(s"gc_cycles", Expression(poolCycles.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withLabel("gc_pool", name) + .withDescription( + s"The total number of collections that have occurred for the $name gc pool") + .register() + ExpressionSchema(s"gc_latency", Expression(poolMsec.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withLabel("gc_pool", name) + .withUnit(Milliseconds) + .withDescription(s"The total elapsed time spent doing collections for the $name gc pool") + .register() + + gauges.add(poolCycles) + gauges.add(poolMsec) } // note, these could be -1 if the collector doesn't have support for it. - gauges.add(gcStats.addGauge("cycles") { gcPool.map(_.getCollectionCount).filter(_ > 0).sum }) - gauges.add(gcStats.addGauge("msec") { gcPool.map(_.getCollectionTime).filter(_ > 0).sum }) - + val cycles = gcStats.addGauge("cycles") { gcPool.map(_.getCollectionCount).filter(_ > 0).sum } + val msec = gcStats.addGauge("msec") { gcPool.map(_.getCollectionTime).filter(_ > 0).sum } + + ExpressionSchema("jvm_uptime", Expression(uptime.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withUnit(Milliseconds) + .withDescription("The uptime of the JVM in MS") + .register() + ExpressionSchema("gc_cycles", Expression(cycles.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withDescription("The total number of collections that have occurred") + .register() + ExpressionSchema("gc_latency", Expression(msec.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withUnit(Milliseconds) + .withDescription("The total elapsed time spent doing collections") + .register() + + gauges.add(cycles) + gauges.add(msec) allocations = new Allocations(gcStats) allocations.start() if (allocations.trackingEden) { diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala new file mode 100644 index 0000000000..1b2e249843 --- /dev/null +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala @@ -0,0 +1,34 @@ +package com.twitter.jvm + +import com.twitter.finagle.stats.InMemoryStatsReceiver +import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey, MetricExpression} +import org.scalatest.funsuite.AnyFunSuite + +class JvmStatsTest extends AnyFunSuite { + + def assertMetric(expressionSchema: ExpressionSchema): Unit = { + assert(expressionSchema.expr.isInstanceOf[MetricExpression]) + } + + test("expressions are instrumented") { + val receiver = new InMemoryStatsReceiver() + JvmStats.register(receiver) + + val role = Map((ExpressionSchema.Role, "jvm")) + val markSweepLabel = role ++ Map(("gc_pool", "PS_MarkSweep")) + val scavengeLabel = role ++ Map(("gc_pool", "PS_Scavenge")) + + val uptimeKey = ExpressionSchemaKey("jvm_uptime", role, Seq()) + val cyclesKey = ExpressionSchemaKey("gc_cycles", role, Seq()) + val latencyKey = ExpressionSchemaKey("gc_latency", role, Seq()) + + assertMetric(receiver.expressions(uptimeKey)) + assertMetric(receiver.expressions(cyclesKey)) + assertMetric(receiver.expressions(latencyKey)) + + assertMetric(receiver.expressions(ExpressionSchemaKey("gc_cycles", markSweepLabel, Seq()))) + assertMetric(receiver.expressions(ExpressionSchemaKey("gc_latency", markSweepLabel, Seq()))) + assertMetric(receiver.expressions(ExpressionSchemaKey("gc_cycles", scavengeLabel, Seq()))) + assertMetric(receiver.expressions(ExpressionSchemaKey("gc_latency", scavengeLabel, Seq()))) + } +} From 3acc0880fd9eb7934b6e1827778bdb61bc3dbc61 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Wed, 11 Aug 2021 20:31:06 +0000 Subject: [PATCH 541/796] Twitter-oss: Prepare OSS libraries for release 21.8.0 Problem We want to release the next versions of our Twitter OSS libraries 21.8.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D722632 --- CHANGELOG.rst | 7 +++++-- README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 55b8be1e1e..0a64fe1415 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.8.0 +------ + New Features ~~~~~~~~~~~~ @@ -32,8 +35,8 @@ Breaking API Changes scala3 Manifest is intended for use by the compiler and should not be used in client code. ``PHAB_ID=D715129`` -21.7.0 ------- +21.7.0 (No Release) +------------------- New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 85ebac626d..baa015fb9d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -35,7 +35,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.7.0" +val utilCore = "com.twitter" %% "util-core" % "21.8.0" ``` # Units diff --git a/build.sbt b/build.sbt index e405cbd24d..9ff3ee7509 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.8.0-SNAPSHOT" +val releaseVersion = "21.8.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 29602592800306e0ff34aa79fd6d17c03162ee8b Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Thu, 12 Aug 2021 23:44:04 +0000 Subject: [PATCH 542/796] util-stats: add getAllExpressionsWithLabel utility Problem/Solution: We currently need an entire ExpressionSchemaKey to query expressions. Lets add a utility to get a list of all expressions with a particular label Differential Revision: https://phabricator.twitter.biz/D722942 --- CHANGELOG.rst | 2 ++ .../scala/com/twitter/jvm/JvmStatsTest.scala | 19 ++++---------- .../finagle/stats/InMemoryStatsReceiver.scala | 14 +++++++++- .../stats/InMemoryStatsReceiverTest.scala | 26 +++++++++++++++++++ 4 files changed, 46 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0a64fe1415..922fdea8da 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-stats: add getAllExpressionsWithLabel utility to InMemoryStatsReceiver. +``PHAB_ID=D722942`` * util-app: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D714780`` diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala index 1b2e249843..0e4aaaf9e8 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala @@ -14,21 +14,12 @@ class JvmStatsTest extends AnyFunSuite { val receiver = new InMemoryStatsReceiver() JvmStats.register(receiver) - val role = Map((ExpressionSchema.Role, "jvm")) - val markSweepLabel = role ++ Map(("gc_pool", "PS_MarkSweep")) - val scavengeLabel = role ++ Map(("gc_pool", "PS_Scavenge")) + val expressions = receiver.getAllExpressionsWithLabel(ExpressionSchema.Role, "jvm") - val uptimeKey = ExpressionSchemaKey("jvm_uptime", role, Seq()) - val cyclesKey = ExpressionSchemaKey("gc_cycles", role, Seq()) - val latencyKey = ExpressionSchemaKey("gc_latency", role, Seq()) + assert(expressions.size == 7) - assertMetric(receiver.expressions(uptimeKey)) - assertMetric(receiver.expressions(cyclesKey)) - assertMetric(receiver.expressions(latencyKey)) - - assertMetric(receiver.expressions(ExpressionSchemaKey("gc_cycles", markSweepLabel, Seq()))) - assertMetric(receiver.expressions(ExpressionSchemaKey("gc_latency", markSweepLabel, Seq()))) - assertMetric(receiver.expressions(ExpressionSchemaKey("gc_cycles", scavengeLabel, Seq()))) - assertMetric(receiver.expressions(ExpressionSchemaKey("gc_latency", scavengeLabel, Seq()))) + expressions.foreach { x: (ExpressionSchemaKey, ExpressionSchema) => + assertMetric(x._2) + } } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 1ec8316dc1..49a2963be2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -5,7 +5,8 @@ import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} import java.io.PrintStream import java.util.Locale import java.util.concurrent.ConcurrentHashMap -import scala.collection.{SortedMap, mutable} +import scala.collection.{SortedMap, mutable, Map => scalaMap} +import scala.collection.compat._ import scala.jdk.CollectionConverters._ object InMemoryStatsReceiver { @@ -266,6 +267,17 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { expressions.put(schema.schemaKey, schema) } + /** + * Retrieves all expressions with a given label key and value + * + * @return a Map of expressions ([[ExpressionSchemaKey]] -> [[ExpressionSchema]]) + */ + def getAllExpressionsWithLabel( + key: String, + value: String + ): scalaMap[ExpressionSchemaKey, ExpressionSchema] = + expressions.view.filterKeys(k => k.labels.get(key) == Some(value)).toMap + } /** diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index f78b47a2d4..c8ad0b0a77 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -396,4 +396,30 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(sr.expressions .get(ExpressionSchemaKey("test_expression", Map(), Nil)).get.expr == expected_expression.expr) } + + test("getExpressionsWithLabel") { + val sr = new InMemoryStatsReceiver + + val aCounter = sr.scope("test").counter("a") + + ExpressionSchema( + "test_expression_0", + Expression(aCounter.metadata) + ).withLabel(ExpressionSchema.Role, "test").register() + + ExpressionSchema( + "test_expression_1", + Expression(aCounter.metadata) + ).withLabel(ExpressionSchema.Role, "dont appear").register() + + val withLabel = sr.getAllExpressionsWithLabel(ExpressionSchema.Role, "test") + assert(withLabel.size == 1) + + assert( + !withLabel.contains( + ExpressionSchemaKey("test_expression_1", Map((ExpressionSchema.Role, "dont appear")), Nil))) + + assert(sr.getAllExpressionsWithLabel("nonexistent key", "nonexistent value").isEmpty) + } + } From 0a3165b380a91893c25500e8a72236d4714f4484 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 13 Aug 2021 00:36:16 +0000 Subject: [PATCH 543/796] Twitter-oss: Prepare OSS libraries for release 21.8.0 Problem We want to release the next versions of our Twitter OSS libraries 21.8.0 util scrooge finagle twitter-server finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D723532 --- CHANGELOG.rst | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 922fdea8da..24e87dc492 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,8 +7,8 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -21.8.0 ------- +21.8.0 (No 21.7.0 Release) +-------------------------- New Features ~~~~~~~~~~~~ @@ -29,25 +29,20 @@ New Features * util-thrift: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D715129`` +* util-app: Introduce a new `Command` class which provides a `Reader` interface to the output + of a shell command. ``PHAB_ID=D686134`` + +* util-core: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D694775`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ + * util-app: Flags and GlobalFlag now use ClassTag instead of Manifest. ``PHAB_ID=D714780`` * util-thrift: ThriftCodec now uses ClassTag instead of Manifest. In scala3 Manifest is intended for use by the compiler and should not be used in client code. ``PHAB_ID=D715129`` -21.7.0 (No Release) -------------------- - -New Features -~~~~~~~~~~~~ - -* util-app: Introduce a new `Command` class which provides a `Reader` interface to the output - of a shell command. ``PHAB_ID=D686134`` - -* util-core: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D694775`` - * util-core (BREAKING): Remove `AbstractSpool`. Java users should use `Spools` static class or the Spool companion object to create instances of `Spool`. ``PHAB_ID=D694775`` @@ -56,9 +51,6 @@ Runtime Behavior Changes * util: Update ScalaCheck to version 1.15.4 ``PHAB_ID=D691691`` -Bug Fixes -~~~~~~~~~ - * util-jackson: `JsonDiff#toSortedString` now includes null-type nodes, so that `JsonDiff.Result#toString` shows differences in objects due to such nodes. ``PHAB_ID=D707033`` From 75f6c89e99363d603ea21ff37a72cefb93b81cf3 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 13 Aug 2021 00:54:10 +0000 Subject: [PATCH 544/796] Twitter-oss: update OSS libraries post-release to 21.9.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D722646 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index baa015fb9d..9bf746a747 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 9ff3ee7509..5829aee98b 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.8.0" +val releaseVersion = "21.9.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 1028127ef54d1c7ea39edc03bc3cecb327e4ccf5 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 13 Aug 2021 14:58:35 +0000 Subject: [PATCH 545/796] util: publish empty javadoc for scala 3 Problem: Sonatype expects a compiled javadoc JAR during the release and errors otherwise. Solution / Result: Allow sbt to publish an empty javadoc artifact JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D724170 --- build.sbt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build.sbt b/build.sbt index 5829aee98b..906228f09b 100644 --- a/build.sbt +++ b/build.sbt @@ -138,10 +138,6 @@ val baseSettings = Seq( if (scalaVersion.value.startsWith("3")) Seq.empty else (Compile / doc / sources).value }, - Compile / packageDoc / publishArtifact := { - if (scalaVersion.value.startsWith("3")) false - else (Compile / packageDoc / publishArtifact).value - }, ScoverageKeys.coverageHighlighting := true, resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", From 4c579633c00640fa61592fbaafc243a17f438c82 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 13 Aug 2021 16:38:26 +0000 Subject: [PATCH 546/796] Revert "Twitter-oss: update OSS libraries post-release to 21.9.0" Problem: Sonatype requires a javadoc jar, even if empty to release. Solution / Result: Revert the post-release commit to act as the new SHA to which will be released. This SHA will come after D724170 which doesn't skip the creation of the empty util javadoc JAR for Scala 3. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D724302 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9bf746a747..baa015fb9d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 906228f09b..42dcc6e58e 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.9.0-SNAPSHOT" +val releaseVersion = "21.8.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From bab1bf5792dbab49b7afecb2ba5d891869920d9f Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Fri, 13 Aug 2021 17:53:30 +0000 Subject: [PATCH 547/796] Twitter-oss: update OSS libraries post-release to 21.9.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11162 Differential Revision: https://phabricator.twitter.biz/D724396 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index baa015fb9d..9bf746a747 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 42dcc6e58e..906228f09b 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.8.0" +val releaseVersion = "21.9.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.2" From 22977aa0ff20fe5fc1226cb9a1d75fe38424280c Mon Sep 17 00:00:00 2001 From: Natalia Margarita Gonzalez Herrera Date: Fri, 13 Aug 2021 23:26:49 +0000 Subject: [PATCH 548/796] util/util-cache: scala3 migration **Problem** We're migrating util-cache to scala3. Dotty has new syntax rules that need to be addressed. **Solution** Use migration flags to solve syntax warnings. Add parentheses to make sure method call and definition match. JIRA Issues: CSL-11106 Differential Revision: https://phabricator.twitter.biz/D714304 --- build.sbt | 34 ++++++++++++------- project/plugins.sbt | 1 + .../twitter/cache/ConcurrentMapCache.scala | 2 +- .../scala/com/twitter/cache/FutureCache.scala | 2 +- .../cache/caffeine/CaffeineCache.scala | 2 +- .../com/twitter/cache/EvictingCacheTest.scala | 8 ++--- .../twitter/cache/KeyEncodingCacheTest.scala | 2 +- .../caffeine/LoadingFutureCacheTest.scala | 2 +- .../com/twitter/conversions/SeqOps.scala | 4 +++ .../src/main/scala/com/twitter/io/Pipe.scala | 2 +- .../main/scala/com/twitter/util/Promise.scala | 2 +- 11 files changed, 38 insertions(+), 23 deletions(-) diff --git a/build.sbt b/build.sbt index 906228f09b..b58491e5ed 100644 --- a/build.sbt +++ b/build.sbt @@ -95,14 +95,6 @@ val defaultScala3EnabledSettings = Seq( // Our dependencies or compiler options may differ for both Scala 2 and 3. We branch here // to account for there differences but should merge these artifacts as they are updated. val scalaDependencies = Seq("org.scala-lang.modules" %% "scala-collection-compat" % "2.4.4") -val scala3Dependencies = scalaDependencies ++ Seq( - "org.scalatest" %% "scalatest" % "3.2.9" % "test", - "org.scalatestplus" %% "junit-4-13" % "3.2.9.0" % "test" -) -val scala2Dependencies = scalaDependencies ++ Seq( - "org.scalatest" %% "scalatest" % "3.1.2" % "test", - "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test" -) val scala2cOptions = Seq( "-target:jvm-1.8", // Needs -missing-interpolator due to https://issues.scala-lang.org/browse/SI-8761 @@ -113,6 +105,15 @@ val scala3cOptions = Seq( "-Xtarget:8" ) +val scala3Dependencies = scalaDependencies ++ Seq( + "org.scalatest" %% "scalatest" % "3.2.9" % "test", + "org.scalatestplus" %% "junit-4-13" % "3.2.9.0" % "test" +) +val scala2Dependencies = scalaDependencies ++ Seq( + "org.scalatest" %% "scalatest" % "3.1.2" % "test", + "org.scalatestplus" %% "junit-4-12" % "3.1.2.0" % "test" +) + val baseSettings = Seq( version := releaseVersion, organization := "com.twitter", @@ -265,7 +266,7 @@ lazy val utilApp = Project( id = "util-app", base = file("util-app") ).settings( - sharedScala3EnabledSettings + sharedScala3EnabledSettings ).settings( name := "util-app", libraryDependencies ++= Seq( @@ -315,15 +316,24 @@ lazy val utilCache = Project( id = "util-cache", base = file("util-cache") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-cache", libraryDependencies ++= Seq( caffeineLib, jsr305Lib, "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" - ) + ) ++ { + if (scalaVersion.value.startsWith("2")) { + Seq( + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" + ) + } else { + Seq( + "org.scalatestplus" %% "mockito-3-4" % "3.2.9.0" % "test" + ) + } + } ).dependsOn(utilCore) lazy val utilCacheGuava = Project( diff --git a/project/plugins.sbt b/project/plugins.sbt index e2cefb38da..b14aeff542 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,3 +4,4 @@ resolvers += Resolver.sonatypeRepo("snapshots") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.0") +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.29") diff --git a/util-cache/src/main/scala/com/twitter/cache/ConcurrentMapCache.scala b/util-cache/src/main/scala/com/twitter/cache/ConcurrentMapCache.scala index 9103966d22..e1f3c7806c 100644 --- a/util-cache/src/main/scala/com/twitter/cache/ConcurrentMapCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/ConcurrentMapCache.scala @@ -20,7 +20,7 @@ class ConcurrentMapCache[K, V](underlying: ConcurrentMap[K, Future[V]]) extends def set(key: K, value: Future[V]): Unit = underlying.put(key, value) def getOrElseUpdate(key: K)(compute: => Future[V]): Future[V] = { - val p = Promise[V] + val p = Promise[V]() underlying.putIfAbsent(key, p) match { case null => p.become(compute) diff --git a/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala b/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala index 5e370904c5..83e74116d2 100644 --- a/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/FutureCache.scala @@ -107,7 +107,7 @@ object FutureCache { * @see [[standard]] for the equivalent Java API. */ def default[K, V](fn: K => Future[V], cache: FutureCache[K, V]): K => Future[V] = - AsyncMemoize(fn, new EvictingCache(cache)) andThen { f: Future[V] => f.interruptible() } + AsyncMemoize(fn, new EvictingCache(cache)) andThen { (f: Future[V]) => f.interruptible() } /** * Alias for [[default]] which can be called from Java. diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala b/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala index f8a2ece734..0a853d4949 100644 --- a/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/CaffeineCache.scala @@ -54,7 +54,7 @@ object CaffeineCache { */ def fromLoadingCache[K, V](cache: LoadingCache[K, Future[V]]): K => Future[V] = { val evicting = EvictingCache.lazily(new LoadingFutureCache(cache)); - { key: K => evicting.get(key).get.interruptible() } + { (key: K) => evicting.get(key).get.interruptible() } } /** diff --git a/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala index 1868542aed..c7cf5f3ccb 100644 --- a/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/EvictingCacheTest.scala @@ -10,7 +10,7 @@ class EvictingCacheTest extends AnyFunSuite with MockitoSugar { test("EvictingCache should evict on failed futures for set") { val cache = mock[FutureCache[String, String]] val fCache = new EvictingCache(cache) - val p = Promise[String] + val p = Promise[String]() fCache.set("key", p) verify(cache).set("key", p) p.setException(new Exception) @@ -20,7 +20,7 @@ class EvictingCacheTest extends AnyFunSuite with MockitoSugar { test("EvictingCache should keep satisfied futures for set") { val cache = mock[FutureCache[String, String]] val fCache = new EvictingCache(cache) - val p = Promise[String] + val p = Promise[String]() fCache.set("key", p) verify(cache).set("key", p) p.setValue("value") @@ -31,7 +31,7 @@ class EvictingCacheTest extends AnyFunSuite with MockitoSugar { val map = new ConcurrentHashMap[String, Future[String]]() val cache = new ConcurrentMapCache(map) val fCache = new EvictingCache(cache) - val p = Promise[String] + val p = Promise[String]() assert(fCache.getOrElseUpdate("key")(p).poll == p.poll) p.setException(new Exception) assert(fCache.get("key") == None) @@ -41,7 +41,7 @@ class EvictingCacheTest extends AnyFunSuite with MockitoSugar { val map = new ConcurrentHashMap[String, Future[String]]() val cache = new ConcurrentMapCache(map) val fCache = new EvictingCache(cache) - val p = Promise[String] + val p = Promise[String]() assert(fCache.getOrElseUpdate("key")(p).poll == p.poll) p.setValue("value") assert(fCache.get("key").map(_.poll) == Some(p.poll)) diff --git a/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala index 98bf953b2b..3f7a296aef 100644 --- a/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/KeyEncodingCacheTest.scala @@ -10,6 +10,6 @@ class KeyEncodingCacheTest extends AbstractFutureCacheTest { val underlyingMap: ConcurrentHashMap[Int, Future[String]] = new ConcurrentHashMap() val underlyingCache: FutureCache[Int, String] = new ConcurrentMapCache(underlyingMap) val cache: FutureCache[String, String] = - new KeyEncodingCache({ num: String => num.hashCode }, underlyingCache) + new KeyEncodingCache({ (num: String) => num.hashCode }, underlyingCache) } } diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/LoadingFutureCacheTest.scala b/util-cache/src/test/scala/com/twitter/cache/caffeine/LoadingFutureCacheTest.scala index 65b66ad9d9..efe2223c15 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/LoadingFutureCacheTest.scala +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/LoadingFutureCacheTest.scala @@ -12,7 +12,7 @@ class LoadingFutureCacheTest extends AbstractLoadingFutureCacheTest { // loading cache semantics are sufficiently unique // to merit distinct tests. - def mkCtx: Ctx = new Ctx { + def mkCtx(): Ctx = new Ctx { val cache = new LoadingFutureCache( Caffeine .newBuilder() diff --git a/util-core/src/main/scala/com/twitter/conversions/SeqOps.scala b/util-core/src/main/scala/com/twitter/conversions/SeqOps.scala index 29e0727fe0..d9f337d8ea 100644 --- a/util-core/src/main/scala/com/twitter/conversions/SeqOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/SeqOps.scala @@ -2,6 +2,10 @@ package com.twitter.conversions import scala.annotation.tailrec +abstract class SeqUtilCompanion { + def hasKnownSize[A](self: Seq[A]): Boolean +} + /** * Implicits for converting [[Seq]]s. * diff --git a/util-core/src/main/scala/com/twitter/io/Pipe.scala b/util-core/src/main/scala/com/twitter/io/Pipe.scala index 7a2dbdc6bb..9a80c38490 100644 --- a/util-core/src/main/scala/com/twitter/io/Pipe.scala +++ b/util-core/src/main/scala/com/twitter/io/Pipe.scala @@ -158,7 +158,7 @@ final class Pipe[A](timer: Timer) extends Reader[A] with Writer[A] { state = State.Writing(buf, p) (null, null, p) - case State.Reading(p) => + case State.Reading(p: Promise[Option[A]]) => // pending reader has enough space for the full write state = State.Idle // The Scala 3 compiler differentiates between the class type, `A` and the the defined diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 9f3ba788c7..24521a06a5 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -492,7 +492,7 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ override def toString: String = { val theState = state match { - case waitq: WaitQueue[A] => s"Waiting($waitq)" + case waitq: Promise.WaitQueue[A] => s"Waiting($waitq)" case s: Interruptible[A] => s"Interruptible(${s.waitq},${s.handler})" case s: Transforming[A] => s"Transforming(${s.waitq},${s.other})" case s: Interrupted[A] => s"Interrupted(${s.waitq},${s.signal})" From 86df882b076c2c8df8eef3bd2e5e467a2089486b Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Mon, 16 Aug 2021 17:39:52 +0000 Subject: [PATCH 549/796] util/util-zk-test: scala 3 migration **Problem** Build util-zk-test with Scala 3 **Solution** # Change util-zk-test project settings. # Compile project + test with Scala 3 migration flags # Remove flags # Fix incompatibilities # Run cross-building tests JIRA Issues: CSL-11093 Differential Revision: https://phabricator.twitter.biz/D720603 --- CHANGELOG.rst | 6 ++++++ build.sbt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 24e87dc492..5e3eb960d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,11 +7,17 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-zk-test: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D720603`` + 21.8.0 (No 21.7.0 Release) -------------------------- New Features ~~~~~~~~~~~~ + * util-stats: add getAllExpressionsWithLabel utility to InMemoryStatsReceiver. ``PHAB_ID=D722942`` diff --git a/build.sbt b/build.sbt index b58491e5ed..484dea9395 100644 --- a/build.sbt +++ b/build.sbt @@ -732,7 +732,7 @@ lazy val utilZkTest = Project( id = "util-zk-test", base = file("util-zk-test") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-zk-test", libraryDependencies += zkDependency From 6131382e273326d14e4784463f7612f388008600 Mon Sep 17 00:00:00 2001 From: Natalia Margarita Gonzalez Herrera Date: Mon, 16 Aug 2021 22:53:33 +0000 Subject: [PATCH 550/796] util/util-cache-guava: scala3 migration Problem We're migrating util-cache-guava to scala3. Dotty has new syntax rules that need to be addressed. Solution Use migration flags to solve syntax warnings. Add parentheses to make sure method call and definition match. JIRA Issues: CSL-11109 Differential Revision: https://phabricator.twitter.biz/D716101 --- CHANGELOG.rst | 3 +++ build.sbt | 26 ++++++++++--------- .../com/twitter/cache/guava/GuavaCache.scala | 2 +- .../cache/guava/LoadingFutureCacheTest.scala | 2 +- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5e3eb960d6..a1389977ed 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* util-cache-guava: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D716101``. + +* util-cache: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D714304``. New Features ~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index 484dea9395..af266f5894 100644 --- a/build.sbt +++ b/build.sbt @@ -217,6 +217,18 @@ lazy val noPublishSettings = Seq( publish / skip := true ) +def scalatestMockitoVersionedDep(scalaVersion: String) = { + if (scalaVersion.startsWith("2")) { + Seq( + "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" + ) + } else { + Seq( + "org.scalatestplus" %% "mockito-3-4" % "3.2.9.0" % "test" + ) + } +} + lazy val util = Project( id = "util", base = file(".") @@ -323,24 +335,14 @@ lazy val utilCache = Project( caffeineLib, jsr305Lib, "org.mockito" % "mockito-core" % mockitoVersion % "test", - ) ++ { - if (scalaVersion.value.startsWith("2")) { - Seq( - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" - ) - } else { - Seq( - "org.scalatestplus" %% "mockito-3-4" % "3.2.9.0" % "test" - ) - } - } + ) ++ scalatestMockitoVersionedDep(scalaVersion.value) ).dependsOn(utilCore) lazy val utilCacheGuava = Project( id = "util-cache-guava", base = file("util-cache-guava") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-cache-guava", libraryDependencies ++= Seq(guavaLib, jsr305Lib) diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala b/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala index 6dc3678dac..4a0f468c1f 100644 --- a/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/GuavaCache.scala @@ -53,7 +53,7 @@ object GuavaCache { */ def fromLoadingCache[K, V](cache: LoadingCache[K, Future[V]]): K => Future[V] = { val evicting = EvictingCache.lazily(new LoadingFutureCache(cache)); - { key: K => evicting.get(key).get.interruptible() } + { (key: K) => evicting.get(key).get.interruptible() } } /** diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/LoadingFutureCacheTest.scala b/util-cache-guava/src/test/scala/com/twitter/cache/guava/LoadingFutureCacheTest.scala index 9cdefcf3ee..4e1c7a5371 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/LoadingFutureCacheTest.scala +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/LoadingFutureCacheTest.scala @@ -12,7 +12,7 @@ class LoadingFutureCacheTest extends AbstractLoadingFutureCacheTest { // loading cache semantics are sufficiently unique // to merit distinct tests. - def mkCtx: Ctx = new Ctx { + def mkCtx(): Ctx = new Ctx { val cache = new LoadingFutureCache( CacheBuilder .newBuilder() From e23687809145b8366046deaa8a7639e97c1152d6 Mon Sep 17 00:00:00 2001 From: Natalia Margarita Gonzalez Herrera Date: Tue, 17 Aug 2021 19:08:41 +0000 Subject: [PATCH 551/796] util/util-slf4j-api: scala3 migration **Problem** We're migrating util-slf4j-api to scala3. Dotty has new syntax rules that need to be addressed. This project also depends on util-mock and therefore on mockito-scala, which is incompatible with scala3. **Solution** Use migration flags to solve syntax warnings. Remove dependency on mockito-scala and use mockito-core instead in LoggingTest and MarkerLoggingTest. JIRA Issues: CSL-11110 Differential Revision: https://phabricator.twitter.biz/D725491 --- CHANGELOG.rst | 3 +- build.sbt | 10 +- .../test/scala/com/twitter/util/logging/BUILD | 5 + .../twitter/util/logging/LoggingTest.scala | 190 +++++++++--------- .../util/logging/MarkerLoggingTest.scala | 169 ++++++++-------- .../com/twitter/util/logging/Stock.scala | 2 +- 6 files changed, 193 insertions(+), 186 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a1389977ed..aa3963a3ff 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,13 +6,14 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* util-sl4j-api: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D725491`` + * util-cache-guava: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D716101``. * util-cache: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D714304``. New Features ~~~~~~~~~~~~ - * util-zk-test: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D720603`` 21.8.0 (No 21.7.0 Release) diff --git a/build.sbt b/build.sbt index af266f5894..b1e77f4f9c 100644 --- a/build.sbt +++ b/build.sbt @@ -583,15 +583,15 @@ lazy val utilSlf4jApi = Project( id = "util-slf4j-api", base = file("util-slf4j-api") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-slf4j-api", libraryDependencies ++= Seq( slf4jApi, - "org.mockito" %% "mockito-scala" % mockitoScalaVersion % "test", - "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" - ) - ).dependsOn(utilCore % "test", utilMock % "test") + "org.mockito" % "mockito-core" % mockitoVersion % "test", + "org.slf4j" % "slf4j-simple" % slf4jVersion % "test", + ) ++ scalatestMockitoVersionedDep(scalaVersion.value) + ).dependsOn(utilCore % "test") lazy val utilSlf4jJulBridge = Project( id = "util-slf4j-jul-bridge", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index 380616d302..f09e317844 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -7,6 +7,11 @@ scala_library( ], tags = ["bazel-compatible"], dependencies = [ + "3rdparty/jvm/org/mockito:mockito-core", + "3rdparty/jvm/org/scalatest", + "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:scalacheck", "3rdparty/jvm/org/slf4j:slf4j-api", "util/util-core/src/main/scala/com/twitter/io", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala index f35af4d1e6..f4bb5e3b85 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala @@ -1,271 +1,272 @@ package com.twitter.util.logging -import com.twitter.util.mock.Mockito +import org.mockito.Mockito._ import org.mockito.Mockito.when +import org.mockito.ArgumentMatchers.any import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import scala.language.reflectiveCalls +import org.scalatestplus.mockito.MockitoSugar -class LoggingTest extends AnyFunSuite with Matchers with Mockito { +class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { /* Trace */ test("Logging#trace enabled") { - val f = fixture(_.isTraceEnabled, isEnabled = true) + val f = Fixture(_.isTraceEnabled, isEnabled = true) f.logger.trace(f.message) - f.underlying.trace(f.message) was called + verify(f.underlying).trace(f.message) } test("Logging#trace not enabled") { - val f = fixture(_.isTraceEnabled, isEnabled = false) + val f = Fixture(_.isTraceEnabled, isEnabled = false) f.logger.trace(f.message) - f.underlying.trace(any[String]) wasNever called + verify(f.underlying, never()).trace(any[String]) } test("Logging#trace enabled with message and cause") { - val f = fixture(_.isTraceEnabled, isEnabled = true) + val f = Fixture(_.isTraceEnabled, isEnabled = true) f.logger.trace(f.message, f.cause) - f.underlying.trace(f.message, f.cause) was called + verify(f.underlying).trace(f.message, f.cause) } test("Logging#trace not enabled with message and cause") { - val f = fixture(_.isTraceEnabled, isEnabled = false) + val f = Fixture(_.isTraceEnabled, isEnabled = false) f.logger.trace(f.message, f.cause) - f.underlying.trace(any[String], any) wasNever called + verify(f.underlying, never()).trace(any[String], any) } test("Logging#trace enabled with parameters") { - val f = fixture(_.isTraceEnabled, isEnabled = true) + val f = Fixture(_.isTraceEnabled, isEnabled = true) f.logger.traceWith(f.message, f.arg1) - f.underlying.trace(f.message, Seq(f.arg1): _*) was called + verify(f.underlying).trace(f.message, Seq(f.arg1): _*) f.logger.traceWith(f.message, f.arg1, f.arg2) - f.underlying.trace(f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).trace(f.message, Seq(f.arg1, f.arg2): _*) f.logger.traceWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.trace(f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).trace(f.message, f.arg1, f.arg2, f.arg3) } test("Logging#trace not enabled with parameters") { - val f = fixture(_.isTraceEnabled, isEnabled = false) + val f = Fixture(_.isTraceEnabled, isEnabled = false) f.logger.traceWith(f.message, f.arg1) - f.underlying.trace(f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).trace(f.message, Seq(f.arg1): _*) f.logger.traceWith(f.message, f.arg1, f.arg2) - f.underlying.trace(f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).trace(f.message, Seq(f.arg1, f.arg2): _*) f.logger.traceWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.trace(f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).trace(f.message, f.arg1, f.arg2, f.arg3) } /* Debug */ test("Logging#debug enabled") { - val f = fixture(_.isDebugEnabled, isEnabled = true) + val f = Fixture(_.isDebugEnabled, isEnabled = true) f.logger.debug(f.message) - f.underlying.debug(f.message) was called + verify(f.underlying).debug(f.message) } test("Logging#debug not enabled") { - val f = fixture(_.isDebugEnabled, isEnabled = false) + val f = Fixture(_.isDebugEnabled, isEnabled = false) f.logger.debug(f.message) - f.underlying.debug(any[String]) wasNever called + verify(f.underlying, never()).debug(any[String]) } test("Logging#debug enabled with message and cause") { - val f = fixture(_.isDebugEnabled, isEnabled = true) + val f = Fixture(_.isDebugEnabled, isEnabled = true) f.logger.debug(f.message, f.cause) - f.underlying.debug(f.message, f.cause) was called + verify(f.underlying).debug(f.message, f.cause) } test("Logging#debug not enabled with message and cause") { - val f = fixture(_.isDebugEnabled, isEnabled = false) + val f = Fixture(_.isDebugEnabled, isEnabled = false) f.logger.debug(f.message, f.cause) - f.underlying.debug(any[String], any) wasNever called + verify(f.underlying, never()).debug(any[String], any) } test("Logging#debug enabled with parameters") { - val f = fixture(_.isDebugEnabled, isEnabled = true) + val f = Fixture(_.isDebugEnabled, isEnabled = true) f.logger.debugWith(f.message, f.arg1) - f.underlying.debug(f.message, Seq(f.arg1): _*) was called + verify(f.underlying).debug(f.message, Seq(f.arg1): _*) f.logger.debugWith(f.message, f.arg1, f.arg2) - f.underlying.debug(f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).debug(f.message, Seq(f.arg1, f.arg2): _*) f.logger.debugWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.debug(f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).debug(f.message, f.arg1, f.arg2, f.arg3) } test("Logging#debug not enabled with parameters") { - val f = fixture(_.isDebugEnabled, isEnabled = false) + val f = Fixture(_.isDebugEnabled, isEnabled = false) f.logger.debugWith(f.message, f.arg1) - f.underlying.debug(f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).debug(f.message, Seq(f.arg1): _*) f.logger.debugWith(f.message, f.arg1, f.arg2) - f.underlying.debug(f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).debug(f.message, Seq(f.arg1, f.arg2): _*) f.logger.debugWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.debug(f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).debug(f.message, f.arg1, f.arg2, f.arg3) } /* Info */ test("Logging#info enabled") { - val f = fixture(_.isInfoEnabled, isEnabled = true) + val f = Fixture(_.isInfoEnabled, isEnabled = true) f.logger.info(f.message) - f.underlying.info(f.message) was called + verify(f.underlying).info(f.message) } test("Logging#info not enabled") { - val f = fixture(_.isInfoEnabled, isEnabled = false) + val f = Fixture(_.isInfoEnabled, isEnabled = false) f.logger.info(f.message) - f.underlying.info(any[String]) wasNever called + verify(f.underlying, never()).info(any[String]) } test("Logging#info enabled with message and cause") { - val f = fixture(_.isInfoEnabled, isEnabled = true) + val f = Fixture(_.isInfoEnabled, isEnabled = true) f.logger.info(f.message, f.cause) - f.underlying.info(f.message, f.cause) was called + verify(f.underlying).info(f.message, f.cause) } test("Logging#info not enabled with message and cause") { - val f = fixture(_.isInfoEnabled, isEnabled = false) + val f = Fixture(_.isInfoEnabled, isEnabled = false) f.logger.info(f.message, f.cause) - f.underlying.info(any[String], any) wasNever called + verify(f.underlying, never()).info(any[String], any) } test("Logging#info enabled with parameters") { - val f = fixture(_.isInfoEnabled, isEnabled = true) + val f = Fixture(_.isInfoEnabled, isEnabled = true) f.logger.infoWith(f.message, f.arg1) - f.underlying.info(f.message, Seq(f.arg1): _*) was called + verify(f.underlying).info(f.message, Seq(f.arg1): _*) f.logger.infoWith(f.message, f.arg1, f.arg2) - f.underlying.info(f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).info(f.message, Seq(f.arg1, f.arg2): _*) f.logger.infoWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.info(f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).info(f.message, f.arg1, f.arg2, f.arg3) } test("Logging#info not enabled with parameters") { - val f = fixture(_.isInfoEnabled, isEnabled = false) + val f = Fixture(_.isInfoEnabled, isEnabled = false) f.logger.infoWith(f.message, f.arg1) - f.underlying.info(f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).info(f.message, Seq(f.arg1): _*) f.logger.infoWith(f.message, f.arg1, f.arg2) - f.underlying.info(f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).info(f.message, Seq(f.arg1, f.arg2): _*) f.logger.infoWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.info(f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).info(f.message, f.arg1, f.arg2, f.arg3) } /* Warn */ test("Logging#warn enabled") { - val f = fixture(_.isWarnEnabled, isEnabled = true) + val f = Fixture(_.isWarnEnabled, isEnabled = true) f.logger.warn(f.message) - f.underlying.warn(f.message) was called + verify(f.underlying).warn(f.message) } test("Logging#warn not enabled") { - val f = fixture(_.isWarnEnabled, isEnabled = false) + val f = Fixture(_.isWarnEnabled, isEnabled = false) f.logger.warn(f.message) - f.underlying.warn(any[String]) wasNever called + verify(f.underlying, never()).warn(any[String]) } test("Logging#warn enabled with message and cause") { - val f = fixture(_.isWarnEnabled, isEnabled = true) + val f = Fixture(_.isWarnEnabled, isEnabled = true) f.logger.warn(f.message, f.cause) - f.underlying.warn(f.message, f.cause) was called + verify(f.underlying).warn(f.message, f.cause) } test("Logging#warn not enabled with message and cause") { - val f = fixture(_.isWarnEnabled, isEnabled = false) + val f = Fixture(_.isWarnEnabled, isEnabled = false) f.logger.warn(f.message, f.cause) - f.underlying.warn(any[String], any) wasNever called + verify(f.underlying, never()).warn(any[String], any) } test("Logging#warn enabled with parameters") { - val f = fixture(_.isWarnEnabled, isEnabled = true) + val f = Fixture(_.isWarnEnabled, isEnabled = true) f.logger.warnWith(f.message, f.arg1) - f.underlying.warn(f.message, Seq(f.arg1): _*) was called + verify(f.underlying).warn(f.message, Seq(f.arg1): _*) f.logger.warnWith(f.message, f.arg1, f.arg2) - f.underlying.warn(f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).warn(f.message, Seq(f.arg1, f.arg2): _*) f.logger.warnWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.warn(f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).warn(f.message, f.arg1, f.arg2, f.arg3) } test("Logging#warn not enabled with parameters") { - val f = fixture(_.isWarnEnabled, isEnabled = false) + val f = Fixture(_.isWarnEnabled, isEnabled = false) f.logger.warnWith(f.message, f.arg1) - f.underlying.warn(f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).warn(f.message, Seq(f.arg1): _*) f.logger.warnWith(f.message, f.arg1, f.arg2) - f.underlying.warn(f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).warn(f.message, Seq(f.arg1, f.arg2): _*) f.logger.warnWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.warn(f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).warn(f.message, f.arg1, f.arg2, f.arg3) } /* Error */ test("Logging#error enabled") { - val f = fixture(_.isErrorEnabled, isEnabled = true) + val f = Fixture(_.isErrorEnabled, isEnabled = true) f.logger.error(f.message) - f.underlying.error(f.message) was called + verify(f.underlying).error(f.message) } test("Logging#error not enabled") { - val f = fixture(_.isErrorEnabled, isEnabled = false) + val f = Fixture(_.isErrorEnabled, isEnabled = false) f.logger.error(f.message) - f.underlying.error(any[String]) wasNever called + verify(f.underlying, never()).error(any[String]) } test("Logging#error enabled with message and cause") { - val f = fixture(_.isErrorEnabled, isEnabled = true) + val f = Fixture(_.isErrorEnabled, isEnabled = true) f.logger.error(f.message, f.cause) - f.underlying.error(f.message, f.cause) was called + verify(f.underlying).error(f.message, f.cause) } test("Logging#error not enabled with message and cause") { - val f = fixture(_.isErrorEnabled, isEnabled = false) + val f = Fixture(_.isErrorEnabled, isEnabled = false) f.logger.error(f.message, f.cause) - f.underlying.error(any[String], any) wasNever called + verify(f.underlying, never()).error(any[String], any) } test("Logging#error enabled with parameters") { - val f = fixture(_.isErrorEnabled, isEnabled = true) + val f = Fixture(_.isErrorEnabled, isEnabled = true) f.logger.errorWith(f.message, f.arg1) - f.underlying.error(f.message, Seq(f.arg1): _*) was called + verify(f.underlying).error(f.message, Seq(f.arg1): _*) f.logger.errorWith(f.message, f.arg1, f.arg2) - f.underlying.error(f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).error(f.message, Seq(f.arg1, f.arg2): _*) f.logger.errorWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.error(f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).error(f.message, f.arg1, f.arg2, f.arg3) } test("Logging#error not enabled with parameters") { - val f = fixture(_.isErrorEnabled, isEnabled = false) + val f = Fixture(_.isErrorEnabled, isEnabled = false) f.logger.errorWith(f.message, f.arg1) - f.underlying.error(f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).error(f.message, Seq(f.arg1): _*) f.logger.errorWith(f.message, f.arg1, f.arg2) - f.underlying.error(f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).error(f.message, Seq(f.arg1, f.arg2): _*) f.logger.errorWith(f.message, f.arg1, f.arg2, f.arg3) - f.underlying.error(f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).error(f.message, f.arg1, f.arg2, f.arg3) } /* Instance Logging */ @@ -356,17 +357,16 @@ class LoggingTest extends AnyFunSuite with Matchers with Mockito { } /* Private */ - - private def fixture(isEnabledFn: org.slf4j.Logger => Boolean, isEnabled: Boolean) = { - new { - val message = "msg" - val cause = new RuntimeException("TEST EXCEPTION") - val arg1 = "arg1" - val arg2 = Integer.valueOf(1) - val arg3 = "arg3" - val underlying: org.slf4j.Logger = mock[org.slf4j.Logger] - when(isEnabledFn(underlying)).thenReturn(isEnabled) - val logger: Logger = Logger(underlying) - } + private case class Fixture( + isEnabledFn: org.slf4j.Logger => Boolean, + isEnabled: Boolean) { + val message = "msg" + val cause = new RuntimeException("TEST EXCEPTION") + val arg1 = "arg1" + val arg2 = Integer.valueOf(1) + val arg3 = "arg3" + val underlying: org.slf4j.Logger = mock[org.slf4j.Logger] + when(isEnabledFn(underlying)).thenReturn(isEnabled) + val logger: Logger = Logger(underlying) } } diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala index 91689745e9..5215673199 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala @@ -1,275 +1,276 @@ package com.twitter.util.logging -import com.twitter.util.mock.Mockito +import org.mockito.Mockito._ import org.mockito.Mockito.when +import org.mockito.ArgumentMatchers.any import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import org.scalatestplus.mockito.MockitoSugar import org.slf4j.Marker -import scala.language.reflectiveCalls -class MarkerLoggingTest extends AnyFunSuite with Matchers with Mockito { +class MarkerLoggingTest extends AnyFunSuite with Matchers with MockitoSugar { test("Marker Logging#trace enabled") { - val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isTraceEnabled(any[Marker]), isEnabled = true) f.logger.trace(f.marker, f.message) - f.underlying.trace(f.marker, f.message) was called + verify(f.underlying).trace(f.marker, f.message) } test("Marker Logging#trace not enabled") { - val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isTraceEnabled(any[Marker]), isEnabled = false) f.logger.trace(f.marker, f.message) - f.underlying.trace(any[Marker], any[String]) wasNever called + verify(f.underlying, never()).trace(any[Marker], any[String]) } test("Marker Logging#trace enabled with message and cause") { - val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isTraceEnabled(any[Marker]), isEnabled = true) f.logger.trace(f.marker, f.message, f.cause) - f.underlying.trace(f.marker, f.message, f.cause) was called + verify(f.underlying).trace(f.marker, f.message, f.cause) } test("Marker Logging#trace not enabled with message and cause") { - val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isTraceEnabled(any[Marker]), isEnabled = false) f.logger.trace(f.marker, f.message, f.cause) - f.underlying.trace(any[Marker], any[String], any) wasNever called + verify(f.underlying, never()).trace(any[Marker], any[String], any) } test("Marker Logging#trace enabled with parameters") { - val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isTraceEnabled(any[Marker]), isEnabled = true) f.logger.traceWith(f.marker, f.message, f.arg1) - f.underlying.trace(f.marker, f.message, Seq(f.arg1): _*) was called + verify(f.underlying).trace(f.marker, f.message, Seq(f.arg1): _*) f.logger.traceWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.trace(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).trace(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.traceWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.trace(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).trace(f.marker, f.message, f.arg1, f.arg2, f.arg3) } test("Marker Logging#trace not enabled with parameters") { - val f = fixture(_.isTraceEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isTraceEnabled(any[Marker]), isEnabled = false) f.logger.traceWith(f.marker, f.message, f.arg1) - f.underlying.trace(f.marker, f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).trace(f.marker, f.message, Seq(f.arg1): _*) f.logger.traceWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.trace(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).trace(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.traceWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.trace(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).trace(f.marker, f.message, f.arg1, f.arg2, f.arg3) } /* Debug */ test("Marker Logging#debug enabled") { - val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isDebugEnabled(any[Marker]), isEnabled = true) f.logger.debug(f.marker, f.message) - f.underlying.debug(f.marker, f.message) was called + verify(f.underlying).debug(f.marker, f.message) } test("Marker Logging#debug not enabled") { - val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isDebugEnabled(any[Marker]), isEnabled = false) f.logger.debug(f.marker, f.message) - f.underlying.debug(any[Marker], any[String]) wasNever called + verify(f.underlying, never()).debug(any[Marker], any[String]) } test("Marker Logging#debug enabled with message and cause") { - val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isDebugEnabled(any[Marker]), isEnabled = true) f.logger.debug(f.marker, f.message, f.cause) - f.underlying.debug(f.marker, f.message, f.cause) was called + verify(f.underlying).debug(f.marker, f.message, f.cause) } test("Marker Logging#debug not enabled with message and cause") { - val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isDebugEnabled(any[Marker]), isEnabled = false) f.logger.debug(f.marker, f.message, f.cause) - f.underlying.debug(any[Marker], any[String], any) wasNever called + verify(f.underlying, never()).debug(any[Marker], any[String], any) } test("Marker Logging#debug enabled with parameters") { - val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isDebugEnabled(any[Marker]), isEnabled = true) f.logger.debugWith(f.marker, f.message, f.arg1) - f.underlying.debug(f.marker, f.message, Seq(f.arg1): _*) was called + verify(f.underlying).debug(f.marker, f.message, Seq(f.arg1): _*) f.logger.debugWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.debug(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).debug(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.debugWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.debug(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).debug(f.marker, f.message, f.arg1, f.arg2, f.arg3) } test("Marker Logging#debug not enabled with parameters") { - val f = fixture(_.isDebugEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isDebugEnabled(any[Marker]), isEnabled = false) f.logger.debugWith(f.marker, f.message, f.arg1) - f.underlying.debug(f.marker, f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).debug(f.marker, f.message, Seq(f.arg1): _*) f.logger.debugWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.debug(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).debug(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.debugWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.debug(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).debug(f.marker, f.message, f.arg1, f.arg2, f.arg3) } /* Info */ test("Marker Logging#info enabled") { - val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isInfoEnabled(any[Marker]), isEnabled = true) f.logger.info(f.marker, f.message) - f.underlying.info(f.marker, f.message) was called + verify(f.underlying).info(f.marker, f.message) } test("Marker Logging#info not enabled") { - val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isInfoEnabled(any[Marker]), isEnabled = false) f.logger.info(f.marker, f.message) - f.underlying.info(any[Marker], any[String]) wasNever called + verify(f.underlying, never()).info(any[Marker], any[String]) } test("Marker Logging#info enabled with message and cause") { - val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isInfoEnabled(any[Marker]), isEnabled = true) f.logger.info(f.marker, f.message, f.cause) - f.underlying.info(f.marker, f.message, f.cause) was called + verify(f.underlying).info(f.marker, f.message, f.cause) } test("Marker Logging#info not enabled with message and cause") { - val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isInfoEnabled(any[Marker]), isEnabled = false) f.logger.info(f.marker, f.message, f.cause) - f.underlying.info(any[Marker], any[String], any) wasNever called + verify(f.underlying, never()).info(any[Marker], any[String], any) } test("Marker Logging#info enabled with parameters") { - val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isInfoEnabled(any[Marker]), isEnabled = true) f.logger.infoWith(f.marker, f.message, f.arg1) - f.underlying.info(f.marker, f.message, Seq(f.arg1): _*) was called + verify(f.underlying).info(f.marker, f.message, Seq(f.arg1): _*) f.logger.infoWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.info(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).info(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.infoWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.info(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).info(f.marker, f.message, f.arg1, f.arg2, f.arg3) } test("Marker Logging#info not enabled with parameters") { - val f = fixture(_.isInfoEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isInfoEnabled(any[Marker]), isEnabled = false) f.logger.infoWith(f.marker, f.message, f.arg1) - f.underlying.info(f.marker, f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).info(f.marker, f.message, Seq(f.arg1): _*) f.logger.infoWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.info(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).info(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.infoWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.info(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).info(f.marker, f.message, f.arg1, f.arg2, f.arg3) } /* Warn */ test("Marker Logging#warn enabled") { - val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isWarnEnabled(any[Marker]), isEnabled = true) f.logger.warn(f.marker, f.message) - f.underlying.warn(f.marker, f.message) was called + verify(f.underlying).warn(f.marker, f.message) } test("Marker Logging#warn not enabled") { - val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isWarnEnabled(any[Marker]), isEnabled = false) f.logger.warn(f.marker, f.message) - f.underlying.warn(any[Marker], any[String]) wasNever called + verify(f.underlying, never()).warn(any[Marker], any[String]) } test("Marker Logging#warn enabled with message and cause") { - val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isWarnEnabled(any[Marker]), isEnabled = true) f.logger.warn(f.marker, f.message, f.cause) - f.underlying.warn(f.marker, f.message, f.cause) was called + verify(f.underlying).warn(f.marker, f.message, f.cause) } test("Marker Logging#warn not enabled with message and cause") { - val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isWarnEnabled(any[Marker]), isEnabled = false) f.logger.warn(f.marker, f.message, f.cause) - f.underlying.warn(any[Marker], any[String], any) wasNever called + verify(f.underlying, never()).warn(any[Marker], any[String], any) } test("Marker Logging#warn enabled with parameters") { - val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isWarnEnabled(any[Marker]), isEnabled = true) f.logger.warnWith(f.marker, f.message, f.arg1) - f.underlying.warn(f.marker, f.message, Seq(f.arg1): _*) was called + verify(f.underlying).warn(f.marker, f.message, Seq(f.arg1): _*) f.logger.warnWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.warn(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).warn(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.warnWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.warn(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).warn(f.marker, f.message, f.arg1, f.arg2, f.arg3) } test("Marker Logging#warn not enabled with parameters") { - val f = fixture(_.isWarnEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isWarnEnabled(any[Marker]), isEnabled = false) f.logger.warnWith(f.marker, f.message, f.arg1) - f.underlying.warn(f.marker, f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).warn(f.marker, f.message, Seq(f.arg1): _*) f.logger.warnWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.warn(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).warn(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.warnWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.warn(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).warn(f.marker, f.message, f.arg1, f.arg2, f.arg3) } /* Error */ test("Marker Logging#error enabled") { - val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isErrorEnabled(any[Marker]), isEnabled = true) f.logger.error(f.marker, f.message) - f.underlying.error(f.marker, f.message) was called + verify(f.underlying).error(f.marker, f.message) } test("Marker Logging#error not enabled") { - val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isErrorEnabled(any[Marker]), isEnabled = false) f.logger.error(f.marker, f.message) - f.underlying.error(any[Marker], any[String]) wasNever called + verify(f.underlying, never()).error(any[Marker], any[String]) } test("Marker Logging#error enabled with message and cause") { - val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isErrorEnabled(any[Marker]), isEnabled = true) f.logger.error(f.marker, f.message, f.cause) - f.underlying.error(f.marker, f.message, f.cause) was called + verify(f.underlying).error(f.marker, f.message, f.cause) } test("Marker Logging#error not enabled with message and cause") { - val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isErrorEnabled(any[Marker]), isEnabled = false) f.logger.error(f.marker, f.message, f.cause) - f.underlying.error(any[Marker], any[String], any) wasNever called + verify(f.underlying, never()).error(any[Marker], any[String], any) } test("Marker Logging#error enabled with parameters") { - val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = true) + val f = Fixture(_.isErrorEnabled(any[Marker]), isEnabled = true) f.logger.errorWith(f.marker, f.message, f.arg1) - f.underlying.error(f.marker, f.message, Seq(f.arg1): _*) was called + verify(f.underlying).error(f.marker, f.message, Seq(f.arg1): _*) f.logger.errorWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.error(f.marker, f.message, Seq(f.arg1, f.arg2): _*) was called + verify(f.underlying).error(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.errorWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.error(f.marker, f.message, f.arg1, f.arg2, f.arg3) was called + verify(f.underlying).error(f.marker, f.message, f.arg1, f.arg2, f.arg3) } test("Marker Logging#error not enabled with parameters") { - val f = fixture(_.isErrorEnabled(any[Marker]), isEnabled = false) + val f = Fixture(_.isErrorEnabled(any[Marker]), isEnabled = false) f.logger.errorWith(f.marker, f.message, f.arg1) - f.underlying.error(f.marker, f.message, Seq(f.arg1): _*) wasNever called + verify(f.underlying, never()).error(f.marker, f.message, Seq(f.arg1): _*) f.logger.errorWith(f.marker, f.message, f.arg1, f.arg2) - f.underlying.error(f.marker, f.message, Seq(f.arg1, f.arg2): _*) wasNever called + verify(f.underlying, never()).error(f.marker, f.message, Seq(f.arg1, f.arg2): _*) f.logger.errorWith(f.marker, f.message, f.arg1, f.arg2, f.arg3) - f.underlying.error(f.marker, f.message, f.arg1, f.arg2, f.arg3) wasNever called + verify(f.underlying, never()).error(f.marker, f.message, f.arg1, f.arg2, f.arg3) } /* Private */ - private def fixture(p: org.slf4j.Logger => Boolean, isEnabled: Boolean) = new { + private case class Fixture(p: org.slf4j.Logger => Boolean, isEnabled: Boolean) { val marker: Marker = TestMarker val message = "msg" val cause = new RuntimeException("TEST EXCEPTION") diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/Stock.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/Stock.scala index e399716c52..8288e401e5 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/Stock.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/Stock.scala @@ -1,6 +1,6 @@ package com.twitter.util.logging -class Stock(symbol: String, price: BigDecimal) extends Logging { +class Stock(symbol: String, price: Double) extends Logging { info(f"New stock with symbol = $symbol and price = $price%1.2f") def quote: String = f"$symbol = $price%1.2f" } From 9615a9f14ced61de3fb30846127c7094dda72a8f Mon Sep 17 00:00:00 2001 From: Natalia Margarita Gonzalez Herrera Date: Wed, 18 Aug 2021 16:44:01 +0000 Subject: [PATCH 552/796] util/util-routing: scala3 migration **Problem** We're migrating util-routing to scala3. Dotty has new syntax rules that need to be addressed. **Solution** Use migration flags to solve syntax warnings. Add parentheses to make sure method call and definition match. JIRA Issues: CSL-11113 Differential Revision: https://phabricator.twitter.biz/D727120 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- .../util/routing/RouterBuilderTest.scala | 25 +++++++++++-------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa3963a3ff..5c58de0461 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,8 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* util-routing: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D727120`` + * util-sl4j-api: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D725491`` * util-cache-guava: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D716101``. diff --git a/build.sbt b/build.sbt index b1e77f4f9c..e8e4906ee1 100644 --- a/build.sbt +++ b/build.sbt @@ -574,7 +574,7 @@ lazy val utilRouting = Project( id = "util-routing", base = file("util-routing") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-routing" ).dependsOn(utilCore, utilSlf4jApi) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala index ba5d07d10b..2cfadb5a53 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala +++ b/util-routing/src/test/scala/com/twitter/util/routing/RouterBuilderTest.scala @@ -15,14 +15,15 @@ class RouterBuilderTest extends AnyFunSuite { private object ValidatingTestRouterBuilder { def newBuilder(): RouterBuilder[String, SimpleRoute, SimpleRouter] = - TestRouter.newBuilder.withValidator( - new Validator[SimpleRoute] { - def apply(routes: Iterable[SimpleRoute]): Iterable[ValidationError] = - routes.collect { - case r if r.in == "invalid" => ValidationError(s"INVALID @ $r") - } - } - ) + TestRouter + .newBuilder().withValidator( + new Validator[SimpleRoute] { + def apply(routes: Iterable[SimpleRoute]): Iterable[ValidationError] = + routes.collect { + case r if r.in == "invalid" => ValidationError(s"INVALID @ $r") + } + } + ) } test("can build routes") { @@ -40,7 +41,8 @@ class RouterBuilderTest extends AnyFunSuite { } test("throws when invalid route is passed") { - val router = ValidatingTestRouterBuilder.newBuilder + val router = ValidatingTestRouterBuilder + .newBuilder() .withRoute(SimpleRoute("x", true)) .withRoute(SimpleRoute("y", false)) .withRoute(SimpleRoute("z", true)) @@ -55,7 +57,8 @@ class RouterBuilderTest extends AnyFunSuite { } test("throws when multiple invalid routes are passed") { - val router = ValidatingTestRouterBuilder.newBuilder + val router = ValidatingTestRouterBuilder + .newBuilder() .withRoute(SimpleRoute("x", true)) .withRoute(SimpleRoute("y", false)) .withRoute(SimpleRoute("z", true)) @@ -75,7 +78,7 @@ class RouterBuilderTest extends AnyFunSuite { test("support contravariant builder") { assertCompiles { """ - | val typed: RouterBuilder[String, SimpleRoute, SimpleRouter] = ValidatingTestRouterBuilder.newBuilder + | val typed: RouterBuilder[String, SimpleRoute, SimpleRouter] = ValidatingTestRouterBuilder.newBuilder() | val generic: RouterBuilder[String, SimpleRoute, Router[String, SimpleRoute]] = typed | val simpleRouter: SimpleRouter = typed.newRouter() | val simpleRouterB: Router[String, SimpleRoute] = generic.newRouter() From 16a4a945d1107a3030944595babb250e010954c8 Mon Sep 17 00:00:00 2001 From: Natalia Margarita Gonzalez Herrera Date: Wed, 18 Aug 2021 20:41:10 +0000 Subject: [PATCH 553/796] util/util-slf4j-jul-bridge: scala3 migration **Problem** We're migrating util-slf4j-jul-bridge to scala3. Dotty has new syntax rules that need to be addressed. **Solution** Use migration flags to solve syntax warnings. In this case there was no need for any changes other that the sharedScala3EnabledSettings JIRA Issues: CSL-11111 Differential Revision: https://phabricator.twitter.biz/D726468 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5c58de0461..4a9ea3baac 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,8 @@ Unreleased ---------- * util-routing: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D727120`` +* util-sl4j-jul-bridge: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D726468`` + * util-sl4j-api: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D725491`` * util-cache-guava: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D716101``. diff --git a/build.sbt b/build.sbt index e8e4906ee1..a522c62b94 100644 --- a/build.sbt +++ b/build.sbt @@ -597,7 +597,7 @@ lazy val utilSlf4jJulBridge = Project( id = "util-slf4j-jul-bridge", base = file("util-slf4j-jul-bridge") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-slf4j-jul-bridge", libraryDependencies ++= Seq(slf4jApi, "org.slf4j" % "jul-to-slf4j" % slf4jVersion) From ea54f59c0d31b2cd37065af59dcd1d37a39e6a09 Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Thu, 19 Aug 2021 04:09:31 +0000 Subject: [PATCH 554/796] util/util-stats: scala 3 migration **Problem** Build util-stats with Scala 3 **Solution** # Change util-stats project settings. # Compile project + test with Scala 3 migration flags # Remove flags # Fix incompatibilities # Run cross-building tests **Result** //Auto-rewrite fixes:// Scala 3 requires parentheses around the parameter of a lambda: - DelegatingStatsReceiverTest.scala //Other fixes:// Method `filterKeys` in trait MapOps is deprecated, should use .view.filterKeys: - CachedRegex.scala - CategorizingExceptionStatsHandlerTest.scala - MultiCategorizingExceptionStatsHandlerTest.scala `schemaKey` method must be called with `()`: - InMemoryStatsReceiver.scala Implicit conversion `long2float` in object Long is deprecated, need to be explicit: - Stat.scala - InMemoryStatsReceiverTest.scala - ExpressionSchemaTest.scala Cannot access apply method of case classes without specifying package: - Expression.scala Types in implicit definitions need to be given explicitly: - DelegatingStatsReceiverTest.scala Note: `print` changed to `println` in InMemoryStatsReceiver.scala to account for a Scala 3 bug where the split method does not work properly in string interpolation with special symbols. See https://jira.twitter.biz/browse/CSL-11259 JIRA Issues: CSL-11092 Differential Revision: https://phabricator.twitter.biz/D720514 --- CHANGELOG.rst | 16 ++++++---- build.sbt | 30 +++++++++++++------ .../twitter/finagle/stats/CachedRegex.scala | 3 +- .../finagle/stats/InMemoryStatsReceiver.scala | 4 +-- .../com/twitter/finagle/stats/Stat.scala | 8 +++-- .../finagle/stats/exp/Expression.scala | 9 +++--- ...ategorizingExceptionStatsHandlerTest.scala | 17 ++++++----- .../stats/DelegatingStatsReceiverTest.scala | 4 +-- .../stats/InMemoryStatsReceiverTest.scala | 2 +- ...ategorizingExceptionStatsHandlerTest.scala | 5 ++-- .../stats/exp/ExpressionSchemaTest.scala | 2 +- 11 files changed, 61 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4a9ea3baac..de58e08329 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,18 +6,22 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -* util-routing: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D727120`` -* util-sl4j-jul-bridge: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D726468`` +New Features +~~~~~~~~~~~~ -* util-sl4j-api: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D725491`` +* util-cache: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D714304``. * util-cache-guava: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D716101``. -* util-cache: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D714304``. +* util-routing: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D727120`` + +* util-sl4j-api: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D725491`` + +* util-sl4j-jul-bridge: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D726468`` + +* util-stats: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D720514`` -New Features -~~~~~~~~~~~~ * util-zk-test: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D720603`` 21.8.0 (No 21.7.0 Release) diff --git a/build.sbt b/build.sbt index a522c62b94..f28ad7f585 100644 --- a/build.sbt +++ b/build.sbt @@ -229,6 +229,18 @@ def scalatestMockitoVersionedDep(scalaVersion: String) = { } } +def scalatestScalacheckVersionedDep(scalaVersion: String) = { + if (scalaVersion.startsWith("2")) { + Seq( + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" + ) + } else { + Seq( + "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test" + ) + } +} + lazy val util = Project( id = "util", base = file(".") @@ -629,7 +641,7 @@ lazy val utilStats = Project( id = "util-stats", base = file("util-stats") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-stats", libraryDependencies ++= Seq( @@ -638,16 +650,16 @@ lazy val utilStats = Project( scalacheckLib, "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, - "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava"), - "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test", - "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" - ) ++ { + ("com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava")).cross(CrossVersion.for3Use2_13), + "org.mockito" % "mockito-core" % mockitoVersion % "test" + ) ++ scalatestMockitoVersionedDep(scalaVersion.value) + ++ scalatestScalacheckVersionedDep(scalaVersion.value) + ++ { CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, major)) if major >= 13 => - Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "0.2.0" % "test") - case _ => + case Some((2, major)) if major <= 12 => Seq() + case _ => + Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.3" % "test") } } ).dependsOn(utilCore, utilLint) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala index 142a3ee997..269b22f3e6 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/CachedRegex.scala @@ -2,6 +2,7 @@ package com.twitter.finagle.stats import java.util.concurrent.ConcurrentHashMap import java.util.function.Consumer +import scala.collection.compat._ import scala.util.matching.Regex /** @@ -50,6 +51,6 @@ private[stats] class CachedRegex(regex: Regex) } } }) - samples.filterKeys(filterFn).toMap + samples.view.filterKeys(filterFn).toMap } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 49a2963be2..3aba0df9d6 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -199,7 +199,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { p.println("---------") } for ((k, v) <- sortedCounters) - p.print(f"$k%s $v%d\n") + p.println(f"$k%s $v%d") if (includeHeaders && sortedGauges.nonEmpty) { p.println("\nGauges:") p.println("-------") @@ -264,7 +264,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { * Designed to match the behavior of Metrics::registerExpression(). */ override protected[finagle] def registerExpression(schema: ExpressionSchema): Unit = { - expressions.put(schema.schemaKey, schema) + expressions.put(schema.schemaKey(), schema) } /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala index 15a712115c..a46289f8be 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/Stat.scala @@ -37,7 +37,7 @@ object Stat { try { f } finally { - stat.add(elapsed().inUnit(unit)) + stat.add(elapsed().inUnit(unit).toFloat) } } @@ -52,10 +52,12 @@ object Stat { def timeFuture[A](stat: Stat, unit: TimeUnit)(f: => Future[A]): Future[A] = { val start = Stopwatch.timeNanos() try { - f.respond { _ => stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS)) } + f.respond { _ => + stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS).toFloat) + } } catch { case NonFatal(e) => - stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS)) + stat.add(unit.convert(Stopwatch.timeNanos() - start, TimeUnit.NANOSECONDS).toFloat) Future.exception(e) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index f5b7958a58..b19f9f8307 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -86,25 +86,26 @@ private[twitter] sealed trait Expression { /** * Represents a constant double number */ -case class ConstantExpression private (repr: String) extends Expression +case class ConstantExpression private[exp] (repr: String) extends Expression /** * Represents compound metrics and their arithmetical(or others) calculations */ -case class FunctionExpression private (fnName: String, exprs: Seq[Expression]) extends Expression { +case class FunctionExpression private[exp] (fnName: String, exprs: Seq[Expression]) + extends Expression { require(exprs.size != 0, "Functions must have at least 1 argument") } /** * Represents the leaf metrics */ -case class MetricExpression private (metricBuilder: MetricBuilder) extends Expression +case class MetricExpression private[exp] (metricBuilder: MetricBuilder) extends Expression /** * Represent a histogram expression with specified component, for example the average, or a percentile * @param component either a [[HistogramComponent]] or a percentile in Double */ -case class HistogramExpression private ( +case class HistogramExpression private[exp] ( metricBuilder: MetricBuilder, component: Either[HistogramComponent, Double]) extends Expression diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala index bde55f1085..406450a3bc 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/CategorizingExceptionStatsHandlerTest.scala @@ -1,6 +1,7 @@ package com.twitter.finagle.stats import org.scalatest.funsuite.AnyFunSuite +import scala.collection.compat._ class CategorizingExceptionStatsHandlerTest extends AnyFunSuite { val categorizer = (t: Throwable) => { "clienterrors" } @@ -19,9 +20,9 @@ class CategorizingExceptionStatsHandlerTest extends AnyFunSuite { val keys = receiver.counters.keys.map(_.mkString("/")).toSeq.sorted - assert(receiver.counters.filterKeys(_.contains("failures")).size == 0) + assert(receiver.counters.view.filterKeys(_.contains("failures")).size == 0) - assert(receiver.counters.filterKeys(_.contains("clienterrors")).size == 3) + assert(receiver.counters.view.filterKeys(_.contains("clienterrors")).size == 3) assert(receiver.counters(Seq("clienterrors")) == 1) assert(receiver.counters(Seq("clienterrors", classOf[RuntimeException].getName)) == 1) assert( @@ -30,7 +31,7 @@ class CategorizingExceptionStatsHandlerTest extends AnyFunSuite { ) == 1 ) - assert(receiver.counters.filterKeys(_.contains("sourcedfailures")).size == 3) + assert(receiver.counters.view.filterKeys(_.contains("sourcedfailures")).size == 3) assert(receiver.counters(Seq("sourcedfailures", "service")) == 1) assert( receiver.counters(Seq("sourcedfailures", "service", classOf[RuntimeException].getName)) == 1 @@ -67,8 +68,8 @@ class CategorizingExceptionStatsHandlerTest extends AnyFunSuite { val keys = receiver.counters.keys.map(_.mkString("/")).toSeq.sorted - assert(receiver.counters.filterKeys(_.contains("failures")).size == 3) - assert(receiver.counters.filterKeys(_.contains("sourcedfailures")).size == 0) + assert(receiver.counters.view.filterKeys(_.contains("failures")).size == 3) + assert(receiver.counters.view.filterKeys(_.contains("sourcedfailures")).size == 0) assert( keys == Seq( @@ -90,9 +91,9 @@ class CategorizingExceptionStatsHandlerTest extends AnyFunSuite { val keys = receiver.counters.keys.map(_.mkString("/")).toSeq.sorted - assert(receiver.counters.filterKeys(_.contains("failures")).size == 0) + assert(receiver.counters.view.filterKeys(_.contains("failures")).size == 0) - assert(receiver.counters.filterKeys(_.contains("clienterrors")).size == 2) + assert(receiver.counters.view.filterKeys(_.contains("clienterrors")).size == 2) assert(receiver.counters(Seq("clienterrors")) == 1) assert( receiver.counters( @@ -100,7 +101,7 @@ class CategorizingExceptionStatsHandlerTest extends AnyFunSuite { ) == 1 ) - assert(receiver.counters.filterKeys(_.contains("sourcedfailures")).size == 2) + assert(receiver.counters.view.filterKeys(_.contains("sourcedfailures")).size == 2) assert(receiver.counters(Seq("sourcedfailures", "service")) == 1) assert( receiver.counters( diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala index bc7aa742c5..530a37c2b8 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DelegatingStatsReceiverTest.scala @@ -64,14 +64,14 @@ class DelegatingStatsReceiverTest extends AnyFunSuite with ScalaCheckDrivenPrope Gen.oneOf(seq).flatMap(identity) } - implicit val impl = Arbitrary(Gen.delay(statsReceiverTopology(0))) + implicit val impl: Arbitrary[StatsReceiver] = Arbitrary(Gen.delay(statsReceiverTopology(0))) } test("DelegatingStatsReceiver.all collects effectively across many StatsReceivers") { val ctx = new Ctx import ctx._ - forAll { statsReceiver: StatsReceiver => + forAll { (statsReceiver: StatsReceiver) => assert(DelegatingStatsReceiver.all(statsReceiver).size == leafCounter) leafCounter = 0 } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index c8ad0b0a77..07e3472ee0 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -58,7 +58,7 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra test("ReadableGauge.toString") { var n = 0 val stats = new InMemoryStatsReceiver() - val g = stats.addGauge("a", "b") { n } + val g = stats.addGauge("a", "b") { n.toFloat } assert("Gauge(a/b=0.0)" == g.toString) n = 11 diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala index 4a4cf7dccf..578ca6f73e 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala @@ -1,6 +1,7 @@ package com.twitter.finagle.stats import org.scalatest.funsuite.AnyFunSuite +import scala.collection.compat._ class MultiCategorizingExceptionStatsHandlerTest extends AnyFunSuite { test("uses label, flags, source, exception chain and rolls up") { @@ -141,8 +142,8 @@ class MultiCategorizingExceptionStatsHandlerTest extends AnyFunSuite { val keys = receiver.counters.keys.map(_.mkString("/")).toSeq.sorted - assert(receiver.counters.filterKeys(_.contains("failures")).size == 3) - assert(receiver.counters.filterKeys(_.contains("sourcedfailures")).size == 0) + assert(receiver.counters.view.filterKeys(_.contains("failures")).size == 3) + assert(receiver.counters.view.filterKeys(_.contains("sourcedfailures")).size == 0) assert( keys == Seq( diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 5f4a037662..aff73b96f6 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -56,7 +56,7 @@ class ExpressionSchemaTest extends AnyFunSuite { case NonFatal(exn) => failuresCounter.incr() } finally { - latencyStat.add(elapsed().inMilliseconds) + latencyStat.add(elapsed().inMilliseconds.toFloat) } } } From 72b9ba1e6464af1dcab33d6d336b207342034e99 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Fri, 20 Aug 2021 15:15:19 +0000 Subject: [PATCH 555/796] 3rdparty: bump jackson version to 2.11.4 Problem We're on an old version of jackson Solution Upgrade!! Differential Revision: https://phabricator.twitter.biz/D727879 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index de58e08329..331315d516 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,11 @@ New Features * util-zk-test: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D720603`` +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Bump version of Jackson to 2.11.4. ``PHAB_ID=D727879`` + 21.8.0 (No 21.7.0 Release) -------------------------- diff --git a/build.sbt b/build.sbt index f28ad7f585..af278833d8 100644 --- a/build.sbt +++ b/build.sbt @@ -7,7 +7,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "21.9.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.11.2" +val jacksonVersion = "2.11.4" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" From b44e820c7992746a24fc98e509c749035ded288f Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Fri, 20 Aug 2021 23:03:04 +0000 Subject: [PATCH 556/796] util-app: make the App#flags field and Flags class final Problem `App` is a trait that exposes non-final fields as `val`, which can result in access ordering issues when fields are overridden and then accessed by the base trait. While attempting to solve this problem, `Flags` overrides introduce other complications, as there are multiple ways to hook into flag parsing (via both `App#parseArgs` and `Flags#parseArgs`. Solution We move our `val` fields to `def` and expose the necessary properties that can be overridden to achieve the existing behavior, just more consistently. We make the `App#flags` def final and `Flags` class becomes final. The `App#name` val is similarly made a `def`. JIRA Issues: CSL-11231 Differential Revision: https://phabricator.twitter.biz/D723956 --- CHANGELOG.rst | 12 ++++++++++++ util-app/src/main/scala/com/twitter/app/App.scala | 12 +++++++++--- util-app/src/main/scala/com/twitter/app/Flags.scala | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 331315d516..65af59d274 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -29,6 +29,18 @@ Runtime Behavior Changes * util: Bump version of Jackson to 2.11.4. ``PHAB_ID=D727879`` +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-app: the `c.t.app.App#flags` field is now a `final def` instead of a `val` to address + `override val` scenarios where the `c.t.app.App#flags` are accessed as part of construction, + resulting in a NullPointerException due to access ordering issues. + The `c.t.app.Flags` class is now made final. The `c.t.app.App#name` field has changed from + a `val` and is now a `def`. A new `c.t.app.App#includeGlobalFlags` def has been exposed, which + defaults to `true`. The `c.t.app#includeGlobalFlags` def can be overridden to `false` + (ex: `override protected def includeGlobalFlags: Boolean = false`) in order to skip discovery + of `GlobalFlag`s during flag parsing. ```PHAB_ID=D723956`` + 21.8.0 (No 21.7.0 Release) -------------------------- diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index 32cfb2b38d..d69d7314f1 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -42,12 +42,15 @@ import scala.util.control.NonFatal trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { /** The name of the application, based on the classname */ - val name: String = getClass.getName.stripSuffix("$") + def name: String = getClass.getName.stripSuffix("$") - /** The [[com.twitter.app.Flags]] instance associated with this application */ //failfastOnFlagsNotParsed is called in the ctor of App.scala here which is a bad idea //as things like this can happen https://stackoverflow.com/questions/18138397/calling-method-from-constructor - val flag: Flags = new Flags(name, includeGlobal = true, failfastOnFlagsNotParsed) + private[this] val _flag: Flags = + new Flags(name, includeGlobal = includeGlobalFlags, failfastOnFlagsNotParsed) + + /** The [[com.twitter.app.Flags]] instance associated with this application */ + final def flag: Flags = _flag private var _args = Array[String]() @@ -57,6 +60,9 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { /** Whether or not to accept undefined flags */ protected def allowUndefinedFlags: Boolean = false + /** Whether or not to include [[GlobalFlag GlobalFlag's]] when [[Flags]] are parsed */ + protected def includeGlobalFlags: Boolean = true + /** * Users of this code should override this to `true` so that * you fail-fast instead of being surprised at runtime by code that diff --git a/util-app/src/main/scala/com/twitter/app/Flags.scala b/util-app/src/main/scala/com/twitter/app/Flags.scala index 7fc4f87667..ac6c028f21 100644 --- a/util-app/src/main/scala/com/twitter/app/Flags.scala +++ b/util-app/src/main/scala/com/twitter/app/Flags.scala @@ -81,7 +81,7 @@ object Flags { * be included during flag parsing. If false, only flags defined in the * application itself will be consulted. */ -class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) { +final class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Boolean) { import com.twitter.app.Flags._ def this(argv0: String, includeGlobal: Boolean) = this(argv0, includeGlobal, false) From 4e68dd096bf241a7a62b26963578b2cedea8b5b5 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 20 Aug 2021 23:16:36 +0000 Subject: [PATCH 557/796] util-stats: Add APIs for providing descriptions Problem/Solution Add metric description APIs in StatsReceiver and hydrate some key indicators. JIRA Issues: CSL-9962 Differential Revision: https://phabricator.twitter.biz/D615481 --- CHANGELOG.rst | 4 ++ .../twitter/finagle/stats/StatsReceiver.scala | 53 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 65af59d274..838944d4ea 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,10 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-stats: Counter, Gauge, and Stat can be instrumented with descriptions. ``PHAB_ID = D615481`` New Features ~~~~~~~~~~~~ diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index c917dbf01a..39c33f32d9 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -125,6 +125,13 @@ trait StatsReceiver { @varargs def counter(name: String*): Counter = counter(Verbosity.Default, name: _*) + /** + * Get a [[Counter counter]] with the given `description` and `name`. + */ + @varargs + def counter(description: Some[String], name: String*): Counter = + counter(description.get, Verbosity.Default, name: _*) + /** * Get a [[Counter counter]] with the given `name`. */ @@ -132,6 +139,18 @@ trait StatsReceiver { def counter(verbosity: Verbosity, name: String*): Counter = counter(this.metricBuilder(CounterType).withVerbosity(verbosity).withName(name: _*)) + /** + * Get a [[Counter counter]] with the given `description` and `name`. + */ + @varargs + def counter(description: String, verbosity: Verbosity, name: String*): Counter = + counter( + this + .metricBuilder(CounterType) + .withVerbosity(verbosity) + .withName(name: _*) + .withDescription(description)) + /** * Get a [[Counter counter]] with the given schema. */ @@ -143,6 +162,13 @@ trait StatsReceiver { @varargs def stat(name: String*): Stat = stat(Verbosity.Default, name: _*) + /** + * Get a [[Stat stat]] with the given `description` and `name`. + */ + @varargs + def stat(description: Some[String], name: String*): Stat = + stat(description.get, Verbosity.Default, name: _*) + /** * Get a [[Stat stat]] with the given name. */ @@ -150,6 +176,18 @@ trait StatsReceiver { def stat(verbosity: Verbosity, name: String*): Stat = stat(this.metricBuilder(HistogramType).withVerbosity(verbosity).withName(name: _*)) + /** + * Get a [[Stat stat]] with the given `description` and `name`. + */ + @varargs + def stat(description: String, verbosity: Verbosity, name: String*): Stat = + stat( + this + .metricBuilder(HistogramType) + .withVerbosity(verbosity) + .withName(name: _*) + .withDescription(description)) + /** * Get a [[Stat stat]] with the given schema. */ @@ -203,6 +241,12 @@ trait StatsReceiver { */ def addGauge(name: String*)(f: => Float): Gauge = addGauge(Verbosity.Default, name: _*)(f) + /** + * Add the function `f` as a [[Gauge gauge]] with the given name and description. + */ + def addGauge(description: Some[String], name: String*)(f: => Float): Gauge = + addGauge(description.get, Verbosity.Default, name: _*)(f) + /** * Add the function `f` as a [[Gauge gauge]] with the given name. * @@ -225,6 +269,15 @@ trait StatsReceiver { def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = addGauge(this.metricBuilder(GaugeType).withVerbosity(verbosity).withName(name: _*))(f) + /** + * Add the function `f` as a [[Gauge gauge]] with the given name and description. + */ + def addGauge(description: String, verbosity: Verbosity, name: String*)(f: => Float): Gauge = + addGauge( + this + .metricBuilder(GaugeType).withVerbosity(verbosity).withName(name: _*).withDescription( + description))(f) + /** * Just like $AddGaugeScaladocLink but optimized for better Java experience. */ From a9afb222f1418995eebea38750df460ff134cea6 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Wed, 25 Aug 2021 15:31:42 +0000 Subject: [PATCH 558/796] json4s: bump to 3.6.11 json4s: bump to 3.6.11 Differential Revision: https://phabricator.twitter.biz/D729764 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 838944d4ea..e345a5d413 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -33,6 +33,8 @@ Runtime Behavior Changes * util: Bump version of Jackson to 2.11.4. ``PHAB_ID=D727879`` +* util: Bump version of json4s to 3.6.11. ``PHAB_ID=D729764`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index af278833d8..13a5d4cdc8 100644 --- a/build.sbt +++ b/build.sbt @@ -501,7 +501,7 @@ lazy val utilJackson = Project( "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava"), "jakarta.validation" % "jakarta.validation-api" % "3.0.0", - "org.json4s" %% "json4s-core" % "3.6.7", + "org.json4s" %% "json4s-core" % "3.6.11", "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion % "test", scalacheckLib, "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", From 0def519daf1cfd5dbed03faba750fbf331672a4e Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Wed, 25 Aug 2021 21:16:08 +0000 Subject: [PATCH 559/796] util-app: roll up flag parsing errors Problem When parsing flags, we escape on the first parsing failure. This behavior results in poor user experience and odd side effects in certain scenarios. For the user, if they have multiple flag errors, they are forced to iterate and fix the errors 1-by-1. Without knowing that there are other parse failures to fix - this is a slow and time consuming feedback loop. In instances where an app references flag values as part of the lifecycle (i.e. as part of cleaning up/closing the app), a parse failure can result in remaining flags not being parsed, which cascades when we fail when flags are referenced, but not yet parsed, resulting in unclean shutdown of the app. Solution We will ensure that all flags get parsed consistently and that we roll-up errors as a single error result. We will also print a help message as part of a parse error, so that users are given the reason their flags were failed to be parsed, along with the expected usage for defined/available flags. Result A more streamlined experience for users attempting to remedy mistaken flag definitions and better behavior when flags are referenced during various phases of the application lifecycle. JIRA Issues: CSL-11231 Differential Revision: https://phabricator.twitter.biz/D729700 --- CHANGELOG.rst | 10 +++++++--- .../main/scala/com/twitter/app/Flags.scala | 19 ++++++++++++------- .../scala/com/twitter/app/FlagsTest.scala | 12 ++++++++++++ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e345a5d413..d4fc961382 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,14 +6,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- + New Features ~~~~~~~~~~~~ * util-stats: Counter, Gauge, and Stat can be instrumented with descriptions. ``PHAB_ID = D615481`` -New Features -~~~~~~~~~~~~ - * util-cache: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D714304``. * util-cache-guava: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D716101``. @@ -28,6 +26,12 @@ New Features * util-zk-test: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D720603`` +* util-app: Flags parsing will now roll-up multiple flag parsing errors into a single + error message. When an error is encountered, flag parsing will continue to collect parse error + information instead of escaping on the first flag failure. After parsing all flags, if any errors + are present, a message containing all of the failed flags and their error reason, + along with the `help` usage message will be emitted. ```PHAB_ID=D729700`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-app/src/main/scala/com/twitter/app/Flags.scala b/util-app/src/main/scala/com/twitter/app/Flags.scala index ac6c028f21..b9a992a7b6 100644 --- a/util-app/src/main/scala/com/twitter/app/Flags.scala +++ b/util-app/src/main/scala/com/twitter/app/Flags.scala @@ -49,7 +49,9 @@ object Flags { * * @param reason A string explaining the error that occurred. */ - case class Error(reason: String) extends FlagParseResult + case class Error(reason: String) extends FlagParseResult { + override def toString: String = reason + } } /** @@ -128,6 +130,7 @@ final class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Bo synchronized { reset() val remaining = new ArrayBuffer[String] + val errors = new ArrayBuffer[Error] var i = 0 while (i < args.length) { val a = args(i) @@ -144,7 +147,7 @@ final class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Bo if (allowUndefinedFlags) remaining += a else - return Error( + errors += Error( "Error parsing flag \"%s\": %s".format(k, FlagUndefinedMessage) ) @@ -153,7 +156,7 @@ final class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Bo if (allowUndefinedFlags) remaining += a else - return Error( + errors += Error( "Error parsing flag \"%s\": %s".format(k, FlagUndefinedMessage) ) @@ -163,7 +166,7 @@ final class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Bo // Mandatory argument without a value and with no more arguments. case Array(k) if i == args.length => - return Error( + errors += Error( "Error parsing flag \"%s\": %s".format(k, FlagValueRequiredMessage) ) @@ -173,7 +176,7 @@ final class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Bo try flag(k).parse(args(i - 1)) catch { case NonFatal(e) => - return Error( + errors += Error( "Error parsing flag \"%s\": %s".format(k, e.getMessage) ) } @@ -182,8 +185,8 @@ final class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Bo case Array(k, v) => try flag(k).parse(v) catch { - case e: Throwable => - return Error( + case NonFatal(e) => + errors += Error( "Error parsing flag \"%s\": %s".format(k, e.getMessage) ) } @@ -196,6 +199,8 @@ final class Flags(argv0: String, includeGlobal: Boolean, failFastUntilParsed: Bo if (helpFlag()) Help(usage) + else if (errors.nonEmpty) + Error(s"Error parsing flags: ${errors.mkString("[\n ", ",\n ", "\n]")}\n\n$usage") else Ok(remaining.toSeq) } diff --git a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala index 2d48d45302..59f41aa26a 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala @@ -165,6 +165,18 @@ class FlagsTest extends AnyFunSuite { ) } + test("Flag: multiple parse errors roll-up") { + val ctx = new Ctx + import ctx._ + flag.parseArgs(Array("-undefined", "-foo", "blah")) match { + case Flags.Error(reason) => + assert(reason.contains("Error parsing flag \"undefined\": flag undefined")) + assert(reason.contains("Error parsing flag \"foo\": For input string: \"blah\"")) + assert(reason.contains("usage:")) + case other => fail(s"expected a flag error, but received $other") + } + } + test("formatFlagValues") { val flagWithGlobal = new Flags("my", includeGlobal = true) From bfdde0713ea93f29f2fe72800313c0b66f3d9e0b Mon Sep 17 00:00:00 2001 From: Michael Sutton Date: Fri, 27 Aug 2021 19:46:42 +0000 Subject: [PATCH 560/796] util/util-jvm: scala 3 migration **Problem** Build util-jvm with Scala 3 **Solution** # Change util-jvm project settings. # Compile project + test with Scala 3 migration flags # Remove flags # Fix incompatibilities # Run cross-building tests **Result** //Auto-rewrite fixes:// Scala 3 requires parentheses around the parameter of a lambda: - Jvm.scala - JvmStatsTest.scala //Other fixes:// DurationOps expects a Long for as an input for operations. The conversion to long used to happen implicitly, but now needs to be explicit in order to work. - CpuProfile.scala - CpuProfileTest.scala Implicit conversion `long2float` in object Long is deprecated, need to be explicit: - JvmStats.scala getter `Stream` is deprecated, use `compat.LazyList` instead: - CpuProfileTest.scala Widening conversion from Long to Double is deprecated: - EstimatorApp.scala Scala 3 infers that executor is of type `ScheduledExecutorService` when it should be of type `MockScheduledExecutorService`. Explicitly give type to avoid error: - JvmTest.scala JIRA Issues: CSL-11094 Differential Revision: https://phabricator.twitter.biz/D728602 --- CHANGELOG.rst | 2 + build.sbt | 5 +- .../scala/com/twitter/jvm/CpuProfile.scala | 2 +- .../src/main/scala/com/twitter/jvm/Jvm.scala | 2 +- .../main/scala/com/twitter/jvm/JvmStats.scala | 82 +++++++++++-------- .../com/twitter/jvm/CpuProfileTest.scala | 5 +- .../scala/com/twitter/jvm/EstimatorApp.scala | 2 +- .../scala/com/twitter/jvm/JvmStatsTest.scala | 2 +- .../test/scala/com/twitter/jvm/JvmTest.scala | 2 +- 9 files changed, 58 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d4fc961382..89b29dafcf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,8 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-jvm: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D728602`` + * util-stats: Counter, Gauge, and Stat can be instrumented with descriptions. ``PHAB_ID = D615481`` * util-cache: Experimentally crossbuilds with Scala 3. ``PHAB_ID=D714304``. diff --git a/build.sbt b/build.sbt index 13a5d4cdc8..6371746a1a 100644 --- a/build.sbt +++ b/build.sbt @@ -519,13 +519,12 @@ lazy val utilJvm = Project( id = "util-jvm", base = file("util-jvm") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-jvm", libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % "test", - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" - ) + ) ++ scalatestMockitoVersionedDep(scalaVersion.value) ).dependsOn(utilApp, utilCore, utilStats) lazy val utilLint = Project( diff --git a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala index f06d944f7c..fff58c38ac 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/CpuProfile.scala @@ -116,7 +116,7 @@ object CpuProfile { val bean = ManagementFactory.getThreadMXBean() val elapsed = Stopwatch.start() val end = howlong.fromNow - val period = (1000000 / frequency).microseconds + val period = ((1000000L / frequency)).microseconds val myId = Thread.currentThread().getId() var next = Time.now diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala index 2b6f45be5a..5d9a157c76 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Jvm.scala @@ -131,7 +131,7 @@ trait Jvm { buffer = (gc :: buffer).takeWhile(_.timestamp > floor) } - since: Time => buffer.takeWhile(_.timestamp > since) + (since: Time) => buffer.takeWhile(_.timestamp > since) } def forceGc(): Unit diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 7bb3bca842..5cb6cb3ac8 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -22,33 +22,33 @@ object JvmStats { def heap = mem.getHeapMemoryUsage() val heapStats = stats.scope("heap") - gauges.add(heapStats.addGauge("committed") { heap.getCommitted() }) - gauges.add(heapStats.addGauge("max") { heap.getMax() }) - gauges.add(heapStats.addGauge("used") { heap.getUsed() }) + gauges.add(heapStats.addGauge("committed") { heap.getCommitted().toFloat }) + gauges.add(heapStats.addGauge("max") { heap.getMax().toFloat }) + gauges.add(heapStats.addGauge("used") { heap.getUsed().toFloat }) def nonHeap = mem.getNonHeapMemoryUsage() val nonHeapStats = stats.scope("nonheap") - gauges.add(nonHeapStats.addGauge("committed") { nonHeap.getCommitted() }) - gauges.add(nonHeapStats.addGauge("max") { nonHeap.getMax() }) - gauges.add(nonHeapStats.addGauge("used") { nonHeap.getUsed() }) + gauges.add(nonHeapStats.addGauge("committed") { nonHeap.getCommitted().toFloat }) + gauges.add(nonHeapStats.addGauge("max") { nonHeap.getMax().toFloat }) + gauges.add(nonHeapStats.addGauge("used") { nonHeap.getUsed().toFloat }) val threads = ManagementFactory.getThreadMXBean() val threadStats = stats.scope("thread") - gauges.add(threadStats.addGauge("daemon_count") { threads.getDaemonThreadCount().toLong }) - gauges.add(threadStats.addGauge("count") { threads.getThreadCount().toLong }) - gauges.add(threadStats.addGauge("peak_count") { threads.getPeakThreadCount().toLong }) + gauges.add(threadStats.addGauge("daemon_count") { threads.getDaemonThreadCount().toFloat }) + gauges.add(threadStats.addGauge("count") { threads.getThreadCount().toFloat }) + gauges.add(threadStats.addGauge("peak_count") { threads.getPeakThreadCount().toFloat }) val runtime = ManagementFactory.getRuntimeMXBean() - val uptime = stats.addGauge("uptime") { runtime.getUptime() } + val uptime = stats.addGauge("uptime") { runtime.getUptime().toFloat } gauges.add(uptime) - gauges.add(stats.addGauge("start_time") { runtime.getStartTime() }) + gauges.add(stats.addGauge("start_time") { runtime.getStartTime().toFloat }) val os = ManagementFactory.getOperatingSystemMXBean() - gauges.add(stats.addGauge("num_cpus") { os.getAvailableProcessors().toLong }) + gauges.add(stats.addGauge("num_cpus") { os.getAvailableProcessors().toFloat }) os match { case unix: com.sun.management.UnixOperatingSystemMXBean => - gauges.add(stats.addGauge("fd_count") { unix.getOpenFileDescriptorCount }) - gauges.add(stats.addGauge("fd_limit") { unix.getMaxFileDescriptorCount }) + gauges.add(stats.addGauge("fd_count") { unix.getOpenFileDescriptorCount.toFloat }) + gauges.add(stats.addGauge("fd_limit") { unix.getMaxFileDescriptorCount.toFloat }) case _ => } @@ -64,15 +64,21 @@ object JvmStats { case null => case compilation => val compilationStats = stats.scope("compilation") - gauges.add(compilationStats.addGauge("time_msec") { compilation.getTotalCompilationTime() }) + gauges.add(compilationStats.addGauge("time_msec") { + compilation.getTotalCompilationTime().toFloat + }) } val classes = ManagementFactory.getClassLoadingMXBean() val classLoadingStats = stats.scope("classes") - gauges.add(classLoadingStats.addGauge("total_loaded") { classes.getTotalLoadedClassCount() }) - gauges.add(classLoadingStats.addGauge("total_unloaded") { classes.getUnloadedClassCount() }) + gauges.add(classLoadingStats.addGauge("total_loaded") { + classes.getTotalLoadedClassCount().toFloat + }) + gauges.add(classLoadingStats.addGauge("total_unloaded") { + classes.getUnloadedClassCount().toFloat + }) gauges.add( - classLoadingStats.addGauge("current_loaded") { classes.getLoadedClassCount().toLong } + classLoadingStats.addGauge("current_loaded") { classes.getLoadedClassCount().toFloat } ) val memPool = ManagementFactory.getMemoryPoolMXBeans.asScala @@ -83,19 +89,19 @@ object JvmStats { val name = pool.getName.regexSub("""[^\w]""".r) { m => "_" } if (pool.getCollectionUsage != null) { def usage = pool.getCollectionUsage // this is a snapshot, we can't reuse the value - gauges.add(postGCStats.addGauge(name, "used") { usage.getUsed }) + gauges.add(postGCStats.addGauge(name, "used") { usage.getUsed.toFloat }) } if (pool.getUsage != null) { def usage = pool.getUsage // this is a snapshot, we can't reuse the value - gauges.add(currentMem.addGauge(name, "used") { usage.getUsed }) - gauges.add(currentMem.addGauge(name, "max") { usage.getMax }) + gauges.add(currentMem.addGauge(name, "used") { usage.getUsed.toFloat }) + gauges.add(currentMem.addGauge(name, "max") { usage.getMax.toFloat }) } } gauges.add(postGCStats.addGauge("used") { - memPool.flatMap(p => Option(p.getCollectionUsage)).map(_.getUsed).sum + memPool.flatMap(p => Option(p.getCollectionUsage)).map(_.getUsed).sum.toFloat }) gauges.add(currentMem.addGauge("used") { - memPool.flatMap(p => Option(p.getUsage)).map(_.getUsed).sum + memPool.flatMap(p => Option(p.getUsage)).map(_.getUsed).sum.toFloat }) // the Hotspot JVM exposes the full size that the metaspace can grow to @@ -103,14 +109,14 @@ object JvmStats { val jvm = Jvm() jvm.metaspaceUsage.foreach { usage => gauges.add(memStats.scope("metaspace").addGauge("max_capacity") { - usage.maxCapacity.inBytes + usage.maxCapacity.inBytes.toFloat }) } val spStats = stats.scope("safepoint") - gauges.add(spStats.addGauge("sync_time_millis") { jvm.safepoint.syncTimeMillis }) - gauges.add(spStats.addGauge("total_time_millis") { jvm.safepoint.totalTimeMillis }) - gauges.add(spStats.addGauge("count") { jvm.safepoint.count }) + gauges.add(spStats.addGauge("sync_time_millis") { jvm.safepoint.syncTimeMillis.toFloat }) + gauges.add(spStats.addGauge("total_time_millis") { jvm.safepoint.totalTimeMillis.toFloat }) + gauges.add(spStats.addGauge("count") { jvm.safepoint.count.toFloat }) ManagementFactory.getPlatformMXBeans(classOf[BufferPoolMXBean]) match { case null => @@ -118,9 +124,9 @@ object JvmStats { val bufferPoolStats = memStats.scope("buffer") jBufferPool.asScala.foreach { bp => val name = bp.getName - gauges.add(bufferPoolStats.addGauge(name, "count") { bp.getCount }) - gauges.add(bufferPoolStats.addGauge(name, "used") { bp.getMemoryUsed }) - gauges.add(bufferPoolStats.addGauge(name, "max") { bp.getTotalCapacity }) + gauges.add(bufferPoolStats.addGauge(name, "count") { bp.getCount.toFloat }) + gauges.add(bufferPoolStats.addGauge(name, "used") { bp.getMemoryUsed.toFloat }) + gauges.add(bufferPoolStats.addGauge(name, "max") { bp.getTotalCapacity.toFloat }) } } @@ -130,10 +136,10 @@ object JvmStats { val name = gc.getName.regexSub("""[^\w]""".r) { m => "_" } val poolCycles = gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "cycles") { - gc.getCollectionCount + gc.getCollectionCount.toFloat } val poolMsec = gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "msec") { - gc.getCollectionTime + gc.getCollectionTime.toFloat } ExpressionSchema(s"gc_cycles", Expression(poolCycles.metadata)) @@ -154,8 +160,12 @@ object JvmStats { } // note, these could be -1 if the collector doesn't have support for it. - val cycles = gcStats.addGauge("cycles") { gcPool.map(_.getCollectionCount).filter(_ > 0).sum } - val msec = gcStats.addGauge("msec") { gcPool.map(_.getCollectionTime).filter(_ > 0).sum } + val cycles = gcStats.addGauge("cycles") { + gcPool.map(_.getCollectionCount).filter(_ > 0).sum.toFloat + } + val msec = gcStats.addGauge("msec") { + gcPool.map(_.getCollectionTime).filter(_ > 0).sum.toFloat + } ExpressionSchema("jvm_uptime", Expression(uptime.metadata)) .withLabel(ExpressionSchema.Role, "jvm") @@ -179,12 +189,12 @@ object JvmStats { if (allocations.trackingEden) { val allocationStats = memStats.scope("allocations") val eden = allocationStats.scope("eden") - gauges.add(eden.addGauge("bytes") { allocations.eden }) + gauges.add(eden.addGauge("bytes") { allocations.eden.toFloat }) } // return ms from ns while retaining precision gauges.add(stats.addGauge("application_time_millis") { jvm.applicationTime.toFloat / 1000000 }) - gauges.add(stats.addGauge("tenuring_threshold") { jvm.tenuringThreshold }) + gauges.add(stats.addGauge("tenuring_threshold") { jvm.tenuringThreshold.toFloat }) } } diff --git a/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala index a87a1f150e..c806175927 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/CpuProfileTest.scala @@ -4,15 +4,16 @@ import com.twitter.conversions.DurationOps._ import com.twitter.util.Time import java.io.ByteArrayOutputStream import org.scalatest.funsuite.AnyFunSuite +import scala.collection.compat.immutable.LazyList class CpuProfileTest extends AnyFunSuite { test("record") { // record() calls Time.now 3 times initially, and then 3 times on every loop iteration. - val times: Stream[Int] = (0 #:: Stream.from(0)).flatMap(x => List(x, x, x)) + val times: LazyList[Int] = (0 #:: LazyList.from(0)).flatMap(x => List(x, x, x)) val iter = times.iterator val start = Time.now - def nextTime: Time = start + iter.next().milliseconds * 10 + def nextTime: Time = start + iter.next().toLong.milliseconds * 10 val t = new Thread("CpuProfileTest") { override def run(): Unit = { diff --git a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala index d600b054b8..8ac1c40084 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/EstimatorApp.scala @@ -42,7 +42,7 @@ object EstimatorApp extends App { var elapsed = 1 for (List(begin, end) <- states.toList.sliding(2)) { val allocated = (end - begin).used - estimator.measure(allocated.inBytes) + estimator.measure(allocated.inBytes.toDouble) val r = end.capacity - end.used val i = (r.inBytes / estimator.estimate.toLong) + elapsed val j = states.indexWhere(_.numCollections > end.numCollections) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala index 0e4aaaf9e8..5719b1a76e 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala @@ -18,7 +18,7 @@ class JvmStatsTest extends AnyFunSuite { assert(expressions.size == 7) - expressions.foreach { x: (ExpressionSchemaKey, ExpressionSchema) => + expressions.foreach { (x: (ExpressionSchemaKey, ExpressionSchema)) => assertMetric(x._2) } } diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala index c0eb0da0d8..59e9b27cc8 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala @@ -28,7 +28,7 @@ class JvmTest extends AnyWordSpec with MockitoSugar { currentSnap = snap } - override val executor = new MockScheduledExecutorService + override val executor: MockScheduledExecutorService = new MockScheduledExecutorService def snap = currentSnap From 8574468572abc1986451d1afa5fe98686b98e294 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Thu, 2 Sep 2021 03:37:59 +0000 Subject: [PATCH 561/796] util: Update OS docs link Problem/Solution/Result The link to Util's open source documentation was incorrect. Update the link. JIRA Issues: CSL-11285 Differential Revision: https://phabricator.twitter.biz/D732859 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9bf746a747..0bcef31a65 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ A bunch of idiomatic, small, general purpose tools. -See the Scaladoc [here](https://twitter.github.com/util). +See the Scaladoc [here](https://twitter.github.io/util/docs/#com.twitter.util.package) +or check out the [user guide](https://twitter.github.io/util). ## Status From 569bae6c9f95304b9bfa9d4175c68998627cea55 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 14 Sep 2021 05:27:43 +0000 Subject: [PATCH 562/796] source: Migrate from Service[Future] and Service.FutureIface to Service.MethodPerEndpoint Problem Both `Service[Future]` and `Service.FutureIface` have been deprecated in Scrooge for a few years. Solution Migrate to a modern-day alternative - `Servive.MethodPerEndpoint`. Authors: Moses Nakamura, Yufan Gong, Joy Bestourous, Vladimir Kostyukov, Ian Bennett NO_USER_HOOK=1 Differential Revision: https://phabricator.twitter.biz/D741246 --- util-core/src/main/scala/com/twitter/util/Future.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index f6a620a968..6604b9d9da 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -28,7 +28,7 @@ import scala.util.control.NoStackTrace * [[[[com.twitter.util.Future.collectToTry[A](fs:Seq[com\.twitter\.util\.Future[A]]):com\.twitter\.util\.Future[Seq[com\.twitter\.util\.Try[A]]]* collectToTry(Seq[Future[A]])]]]] * * @define FutureSelectScaladocLink - * [[[com.twitter.util.Future.select[A](fs:Seq[com\.twitter\.util\.Future[A]]):com\.twitter\.util\.Future[(com\.twitter\.util\.Try[A],Seq[com\.twitter\.util\.Future[A]])]* select(Seq[Future])]]] + * [[[com.twitter.util.Future.select[A](fs:Seq[com\.twitter\.util\.Future[A]]):com\.twitter\.util\.Future[(com\.twitter\.util\.Try[A],Seq[com\.twitter\.util\.Future[A]])]* select(Seq.MethodPerEndpoint)]]] */ object Future { val DEFAULT_TIMEOUT: Duration = Duration.Top From bcc421b2e7a1e573286481dcdd1043e9cbccc083 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Wed, 15 Sep 2021 22:49:53 +0000 Subject: [PATCH 563/796] util-core: Fix bug in Reader.fromFuture Problem Reader.fromFuture has bug where duplicate calls to `read` can race against the internal state. This was causing code like this to fail: ``` test("Reader.fromFuture - unsatisfied") { val p = Promise[Int]() val r1 = Reader.fromFuture(p) val f1 = r1.read() val f2 = f1.flatMap(_ => r1.read()) p.setValue(1) assert(await(f1) == Some(1)) // we already consumed the single item via the operation // in `f1`, so `f2` should be empty since it's strictly // sequenced after `f1`. assert(await(f2) == None) } ``` Solution Ensure the internal state is set before returning a derivative Future inside of `read`. JIRA Issues: CSL-11326 Differential Revision: https://phabricator.twitter.biz/D744629 --- .../scala/com/twitter/io/FutureReader.scala | 11 +++++--- .../scala/com/twitter/io/ReaderTest.scala | 26 ++++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/FutureReader.scala b/util-core/src/main/scala/com/twitter/io/FutureReader.scala index 806ba77288..f7206c5a93 100644 --- a/util-core/src/main/scala/com/twitter/io/FutureReader.scala +++ b/util-core/src/main/scala/com/twitter/io/FutureReader.scala @@ -1,7 +1,10 @@ package com.twitter.io import com.twitter.io.FutureReader._ -import com.twitter.util.{Future, Promise, Return, Throw} +import com.twitter.util.Future +import com.twitter.util.Promise +import com.twitter.util.Return +import com.twitter.util.Throw /** * We want to ensure that this reader always satisfies these invariants: @@ -21,11 +24,13 @@ private[io] final class FutureReader[A](fa: Future[A]) extends Reader[A] { val (updatedState, result) = synchronized { val result = state match { case State.Idle => - fa.map(Some.apply).respond { + fa.transform { case Throw(t) => state = State.Failed(t) - case Return(_) => + Future.exception(t) + case Return(v) => state = State.Read + Future.value(Some(v)) } case State.Read => state = State.FullyRead diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 376e73843a..403ac99255 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -3,13 +3,18 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.conversions.DurationOps._ import com.twitter.conversions.StorageUnitOps._ -import com.twitter.util.{Await, Awaitable, Future, Promise} +import com.twitter.util.Await +import com.twitter.util.Awaitable +import com.twitter.util.Future +import com.twitter.util.Promise import java.io.ByteArrayInputStream import java.nio.charset.{StandardCharsets => JChar} import java.util.concurrent.atomic.AtomicBoolean import org.mockito.Mockito._ -import org.scalacheck.{Arbitrary, Gen} -import org.scalatest.concurrent.{Eventually, IntegrationPatience} +import org.scalacheck.Arbitrary +import org.scalacheck.Gen +import org.scalatest.concurrent.Eventually +import org.scalatest.concurrent.IntegrationPatience import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -467,6 +472,21 @@ class ReaderTest } } + test("Reader.fromFuture - unsatisfied") { + val p = Promise[Int]() + val r1 = Reader.fromFuture(p) + + val f1 = r1.read() + val f2 = f1.flatMap(_ => r1.read()) + + p.setValue(1) + assert(await(f1) == Some(1)) + // we already consumed the single item via the operation + // in `f1`, so `f2` should be empty since it's strictly + // sequenced after `f1`. + assert(await(f2) == None) + } + test("Reader.map") { forAll { (l: List[Int]) => val pipe = new Pipe[Int] From bc0c32af2dfccfde9af3f437cabebbd19208a3c5 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Fri, 17 Sep 2021 21:28:38 +0000 Subject: [PATCH 564/796] util-stats: add default labels for histograms Problem Histogram expressions can represent a bunch of metrics: max/min/p99 etc. which means we have to either name them differently or put a label in expression. And using labels would be a better solution in terms of rendering expressions in dashboards. Solution We hold the Histogram bucket information upon creation, fill the label by defualt. JIRA Issues: CSL-11239 Differential Revision: https://phabricator.twitter.biz/D743299 --- .../finagle/stats/HistogramFormatter.scala | 27 ++++++++++++ .../finagle/stats/exp/ExpressionSchema.scala | 33 +++++++++++++-- .../stats/exp/ExpressionSchemaTest.scala | 41 +++++++++++++++++-- 3 files changed, 94 insertions(+), 7 deletions(-) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/HistogramFormatter.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramFormatter.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramFormatter.scala new file mode 100644 index 0000000000..93a7c8a04a --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramFormatter.scala @@ -0,0 +1,27 @@ +package com.twitter.finagle.stats + +/** + * The default behavior for formatting, align with Commons Metrics. + */ +private[twitter] object HistogramFormatter { + def labelPercentile(p: Double): String = { + // this has a strange quirk that p999 gets formatted as p9990 + // Round for precision issues; e.g. 0.9998999... converts to "p9998" with a direct int cast. + val gname: String = "p" + (p * 10000).round + if (3 < gname.length && ("00" == gname.substring(3))) { + gname.substring(0, 3) + } else { + gname + } + } + + val labelMin: String = "min" + + val labelMax: String = "max" + + val labelAverage: String = "avg" + + val labelCount: String = "count" + + val labelSum: String = "sum" +} diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index 8a7ce39e38..698036259b 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -1,6 +1,15 @@ package com.twitter.finagle.stats.exp -import com.twitter.finagle.stats.{MetricUnit, SourceRole, StatsReceiver, Unspecified} +import com.twitter.finagle.stats.HistogramFormatter +import com.twitter.finagle.stats.MetricUnit +import com.twitter.finagle.stats.SourceRole +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.stats.Unspecified +import com.twitter.finagle.stats.exp.Expression.Avg +import com.twitter.finagle.stats.exp.Expression.Count +import com.twitter.finagle.stats.exp.Expression.Max +import com.twitter.finagle.stats.exp.Expression.Min +import com.twitter.finagle.stats.exp.Expression.Sum /** * ExpressionSchema is builder class that construct an expression with its metadata. @@ -79,14 +88,32 @@ private[twitter] object ExpressionSchema { val ServiceName: String = "service_name" val ProcessPath: String = "process_path" - def apply(name: String, expr: Expression): ExpressionSchema = + def apply(name: String, expr: Expression): ExpressionSchema = { + val preLabels = expr match { + case histoExpr: HistogramExpression => histogramLabel(histoExpr) + case _ => Map.empty[String, String] + } + ExpressionSchema( name = name, - labels = Map.empty, + labels = preLabels, namespace = Seq.empty, expr = expr, bounds = Unbounded.get, description = "Unspecified", unit = Unspecified, exprQuery = "") + } + + private def histogramLabel(histoExpr: HistogramExpression): Map[String, String] = { + val value = histoExpr.component match { + case Right(percentile) => HistogramFormatter.labelPercentile(percentile) + case Left(Min) => HistogramFormatter.labelMin + case Left(Max) => HistogramFormatter.labelMax + case Left(Avg) => HistogramFormatter.labelAverage + case Left(Sum) => HistogramFormatter.labelSum + case Left(Count) => HistogramFormatter.labelCount + } + Map("bucket" -> value) + } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index aff73b96f6..7b92022ef6 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -1,9 +1,12 @@ package com.twitter.finagle.stats.exp import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.MetricBuilder.{CounterType, HistogramType} +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats._ -import com.twitter.util.{Stopwatch, Time, TimeControl} +import com.twitter.util.Stopwatch +import com.twitter.util.Time +import com.twitter.util.TimeControl import scala.util.control.NonFatal import org.scalatest.funsuite.AnyFunSuite @@ -72,9 +75,17 @@ class ExpressionSchemaTest extends AnyFunSuite { sr.expressions( ExpressionSchemaKey("success_rate", downstreamLabel, Nil)).expr == successRate.expr) assert( - sr.expressions(ExpressionSchemaKey("latency", downstreamLabel, Nil)).name == latency.name) + sr.expressions( + ExpressionSchemaKey( + "latency", + downstreamLabel + ("bucket" -> "p99"), + Nil)).name == latency.name) assert( - sr.expressions(ExpressionSchemaKey("latency", downstreamLabel, Nil)).expr == latency.expr) + sr.expressions( + ExpressionSchemaKey( + "latency", + downstreamLabel + ("bucket" -> "p99"), + Nil)).expr == latency.expr) assert(sr.counters(Seq("success")) == 1) assert(sr.counters(Seq("failures")) == 1) @@ -96,6 +107,28 @@ class ExpressionSchemaTest extends AnyFunSuite { } } + test("histogram expressions come with default labels") { + new Ctx { + val latencyP90 = ExpressionSchema("latency", Expression(latencyMb, Right(0.9))).register() + val latencyP99 = ExpressionSchema("latency", Expression(latencyMb, Right(0.99))).register() + val latencyAvg = + ExpressionSchema("latency", Expression(latencyMb, Left(Expression.Avg))).register() + + val downstreamLabel = + Map(ExpressionSchema.Role -> Client.toString, ExpressionSchema.ServiceName -> "downstream") + + assert( + sr.expressions.contains( + ExpressionSchemaKey("latency", downstreamLabel + ("bucket" -> "p90"), Nil))) + assert( + sr.expressions.contains( + ExpressionSchemaKey("latency", downstreamLabel + ("bucket" -> "p99"), Nil))) + assert( + sr.expressions.contains( + ExpressionSchemaKey("latency", downstreamLabel + ("bucket" -> "avg"), Nil))) + } + } + test("expressions with different namespaces are all stored") { new Ctx { val successRate1 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) From 081d47cac1eeccb93902665605c2dddb86dfeaf3 Mon Sep 17 00:00:00 2001 From: Steve Niemitz Date: Fri, 17 Sep 2021 21:39:31 +0000 Subject: [PATCH 565/796] util-core: Allow propagating interrupts in Reader.readAllItems Problem: If the future returned from Reader.readAllItems is interrupted, the interrupt is ignored. It can be desirable to instead propagate the interrupt back to the reader itself. Solution: If propagateInterrupts is set, discard the reader if the returned future is interrupted. Differential Revision: https://phabricator.twitter.biz/D745511 --- .../main/scala/com/twitter/io/Reader.scala | 42 +++++++++++++++---- .../scala/com/twitter/io/ReaderTest.scala | 39 +++++++++++++++++ 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Reader.scala b/util-core/src/main/scala/com/twitter/io/Reader.scala index 0433a2dda6..c07bf8a0a7 100644 --- a/util-core/src/main/scala/com/twitter/io/Reader.scala +++ b/util-core/src/main/scala/com/twitter/io/Reader.scala @@ -3,8 +3,11 @@ package com.twitter.io import com.twitter.concurrent.AsyncStream import com.twitter.util.Promise.Detachable import com.twitter.util._ -import java.io.{File, FileInputStream, InputStream} +import java.io.File +import java.io.FileInputStream +import java.io.InputStream import java.util.{Collection => JCollection} +import scala.collection.mutable import scala.jdk.CollectionConverters._ /** @@ -192,19 +195,42 @@ object Reader { */ def exception[A](e: Throwable): Reader[A] = fromFuture[A](Future.exception(e)) + private def failOrDiscard(r: Reader[_], t: Throwable): Unit = r match { + case p: Pipe[_] => p.fail(t) + case r => r.discard() + } + + private def consume[A](r: Reader[A], builder: mutable.Builder[A, IndexedSeq[A]]): Future[Seq[A]] = + r.read().flatMap { + case Some(v) => + builder += v + consume(r, builder) + case None => + Future.value(builder.result()) + } + /** * Read all items from the Reader r. + * @param r The reader to read from * @return A Sequence of items. */ def readAllItems[A](r: Reader[A]): Future[Seq[A]] = { - val acc = List.newBuilder[A] - def loop(): Future[List[A]] = r.read().flatMap { - case None => Future.value(acc.result()) - case Some(t) => - acc += t - loop() + // This could be implemented as `readAllItemsInterruptable(r).masked`, but doing so would cause multiple extra + // promise allocations. + consume(r, IndexedSeq.newBuilder) + } + + /** + * Read all items from the Reader r, interrupting the returned future will discard the reader. + * @param r The reader to read from + * @return A Sequence of items. + */ + def readAllItemsInterruptible[A](r: Reader[A]): Future[Seq[A]] = { + val p = new Promise[Seq[A]] with Promise.InterruptHandler { + override protected def onInterrupt(t: Throwable): Unit = failOrDiscard(r, t) } - loop() + consume(r, IndexedSeq.newBuilder).proxyTo(p) + p } /** diff --git a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala index 403ac99255..98b94de420 100644 --- a/util-core/src/test/scala/com/twitter/io/ReaderTest.scala +++ b/util-core/src/test/scala/com/twitter/io/ReaderTest.scala @@ -7,6 +7,7 @@ import com.twitter.util.Await import com.twitter.util.Awaitable import com.twitter.util.Future import com.twitter.util.Promise +import com.twitter.util.TimeoutException import java.io.ByteArrayInputStream import java.nio.charset.{StandardCharsets => JChar} import java.util.concurrent.atomic.AtomicBoolean @@ -533,4 +534,42 @@ class ReaderTest assert(await(Reader.readAllItems(r)) == l) } } + + test("Reader.readAllItems - propagates interrupts if enabled") { + val p = new Promise[Int]() + val reader = Reader.fromFuture(p) + + val fut = Reader.readAllItemsInterruptible(reader) + fut.raise(new TimeoutException("")) + + assert(Await.result(reader.onClose) == StreamTermination.Discarded) + } + + test("Reader.readAllItems - ignores interrupts if disabled") { + val p = new Promise[Int]() + val reader = Reader.fromFuture(p) + + val fut = Reader.readAllItems(reader) + fut.raise(new TimeoutException("")) + + p.setValue(1) + assert(await(fut) == Seq(1)) + assert(await(reader.onClose) == StreamTermination.FullyRead) + } + + test("Reader - fails pipe with exception") { + val p = new Pipe[Int]() + val fut = Reader.readAllItemsInterruptible(p) + + fut.raise(new TimeoutException("")) + assertThrows[TimeoutException](await(p.read())) + } + + test("Reader - fails non-pipe with discard") { + val r = Reader.fromFuture(new Promise[Int]()) + val fut = Reader.readAllItemsInterruptible(r) + + fut.raise(new TimeoutException("")) + assertThrows[ReaderDiscardedException](await(r.read())) + } } From 2d5817f65a77198c592e273506796ee08c137796 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Wed, 22 Sep 2021 23:58:10 +0000 Subject: [PATCH 566/796] util-stats: change the expression register method signature Problem ExpressionSchema#register() is mutable and can hide the errored operation. Solution Rename it to `build`. Change the return type to Try. Result Finagle doesn't fail when clients/servers are created using duplicated labels, hence we shouldn't fail with the same expression schema key. Return a Try to delegate error handling to the service owners who instrument their expressions. JIRA Issues: CSL-11284 Differential Revision: https://phabricator.twitter.biz/D747728 --- .../test/scala/com/twitter/util/TryTest.scala | 4 +- .../main/scala/com/twitter/jvm/JvmStats.scala | 19 ++-- .../stats/BroadcastStatsReceiver.scala | 25 ++++- .../finagle/stats/InMemoryStatsReceiver.scala | 22 ++++- .../stats/RoleConfiguredStatsReceiver.scala | 3 +- .../twitter/finagle/stats/StatsReceiver.scala | 19 ++-- .../finagle/stats/StatsReceiverProxy.scala | 6 +- .../finagle/stats/exp/Expression.scala | 4 +- .../finagle/stats/exp/ExpressionSchema.scala | 17 +++- .../stats/InMemoryStatsReceiverTest.scala | 20 ++-- .../stats/exp/ExpressionSchemaTest.scala | 93 +++++++++++-------- 11 files changed, 148 insertions(+), 84 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/util/TryTest.scala b/util-core/src/test/scala/com/twitter/util/TryTest.scala index fa0ed84882..8c54d5fa58 100644 --- a/util-core/src/test/scala/com/twitter/util/TryTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TryTest.scala @@ -211,7 +211,9 @@ class TryTest extends AnyFunSuite { } test("Try to scala.util.Try works") { - import scala.util.{Try => STry, Failure, Success} + import scala.util.{Try => STry} + import scala.util.Failure + import scala.util.Success assert(STry(1) == Try(1).asScala) assert(Try(sys.error("boom!")).asScala match { diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 5cb6cb3ac8..26b8cff02e 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -2,9 +2,12 @@ package com.twitter.jvm import com.twitter.conversions.StringOps._ import com.twitter.finagle.stats.MetricBuilder.GaugeType -import com.twitter.finagle.stats.{Milliseconds, StatsReceiver} -import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema} -import java.lang.management.{BufferPoolMXBean, ManagementFactory} +import com.twitter.finagle.stats.Milliseconds +import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.stats.exp.Expression +import com.twitter.finagle.stats.exp.ExpressionSchema +import java.lang.management.BufferPoolMXBean +import java.lang.management.ManagementFactory import scala.collection.mutable import scala.jdk.CollectionConverters._ @@ -147,13 +150,13 @@ object JvmStats { .withLabel("gc_pool", name) .withDescription( s"The total number of collections that have occurred for the $name gc pool") - .register() + .build() ExpressionSchema(s"gc_latency", Expression(poolMsec.metadata)) .withLabel(ExpressionSchema.Role, "jvm") .withLabel("gc_pool", name) .withUnit(Milliseconds) .withDescription(s"The total elapsed time spent doing collections for the $name gc pool") - .register() + .build() gauges.add(poolCycles) gauges.add(poolMsec) @@ -171,16 +174,16 @@ object JvmStats { .withLabel(ExpressionSchema.Role, "jvm") .withUnit(Milliseconds) .withDescription("The uptime of the JVM in MS") - .register() + .build() ExpressionSchema("gc_cycles", Expression(cycles.metadata)) .withLabel(ExpressionSchema.Role, "jvm") .withDescription("The total number of collections that have occurred") - .register() + .build() ExpressionSchema("gc_latency", Expression(msec.metadata)) .withLabel(ExpressionSchema.Role, "jvm") .withUnit(Milliseconds) .withDescription("The total elapsed time spent doing collections") - .register() + .build() gauges.add(cycles) gauges.add(msec) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala index 729114ea56..e48881647a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala @@ -1,6 +1,9 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.util.Return +import com.twitter.util.Throw +import com.twitter.util.Try /** * BroadcastStatsReceiver is a helper object that create a StatsReceiver wrapper around multiple @@ -39,9 +42,16 @@ object BroadcastStatsReceiver { def metadata: Metadata = MultiMetadata(Seq(firstGauge.metadata, secondGauge.metadata)) } - override protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = { - first.registerExpression(expressionSchema) - second.registerExpression(expressionSchema) + override protected[finagle] def registerExpression( + expressionSchema: ExpressionSchema + ): Try[Unit] = { + Try.collect( + Seq( + first.registerExpression(expressionSchema), + second.registerExpression(expressionSchema))) match { + case Return(_) => Return.Unit + case Throw(throwable) => Throw(throwable) + } } def underlying: Seq[StatsReceiver] = Seq(first, second) @@ -65,8 +75,13 @@ object BroadcastStatsReceiver { def metadata: Metadata = MultiMetadata(gauges.map(_.metadata)) } - override protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = - srs.map(_.registerExpression(expressionSchema)) + override protected[finagle] def registerExpression( + expressionSchema: ExpressionSchema + ): Try[Unit] = + Try.collect(srs.map(_.registerExpression(expressionSchema))) match { + case Return(_) => Return.Unit + case Throw(throwable) => Throw(throwable) + } def underlying: Seq[StatsReceiver] = srs diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 3aba0df9d6..775e50fc5b 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -1,12 +1,20 @@ package com.twitter.finagle.stats -import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} -import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.exp.ExpressionSchema.ExpressionCollisionException +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey +import com.twitter.util.Throw +import com.twitter.util.Try import java.io.PrintStream import java.util.Locale import java.util.concurrent.ConcurrentHashMap -import scala.collection.{SortedMap, mutable, Map => scalaMap} import scala.collection.compat._ +import scala.collection.SortedMap +import scala.collection.mutable +import scala.collection.{Map => scalaMap} import scala.jdk.CollectionConverters._ object InMemoryStatsReceiver { @@ -263,8 +271,12 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { /** * Designed to match the behavior of Metrics::registerExpression(). */ - override protected[finagle] def registerExpression(schema: ExpressionSchema): Unit = { - expressions.put(schema.schemaKey(), schema) + override protected[finagle] def registerExpression(schema: ExpressionSchema): Try[Unit] = { + if (expressions.contains(schema.schemaKey())) { + Throw( + ExpressionCollisionException( + s"An expression with the key ${schema.schemaKey()} had already been defined.")) + } else Try { expressions.put(schema.schemaKey(), schema) } } /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala index 612a47e6de..1c9e7bb960 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiver.scala @@ -1,6 +1,7 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.util.Try /** * A StatsReceiver proxy that configures all counter, stat, and gauge @@ -27,7 +28,7 @@ case class RoleConfiguredStatsReceiver( self.addGauge(metricBuilder.withRole(role))(f) } - override def registerExpression(expressionSchema: ExpressionSchema): Unit = { + override def registerExpression(expressionSchema: ExpressionSchema): Try[Unit] = { val configuredExprSchema = name match { case Some(serviceName) => expressionSchema.withRole(role).withServiceName(serviceName) case None => expressionSchema.withRole(role) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 39c33f32d9..2acdfffe09 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -1,14 +1,14 @@ package com.twitter.finagle.stats -import com.twitter.finagle.stats.MetricBuilder.{ - CounterType, - CounterishGaugeType, - GaugeType, - HistogramType, - MetricType, - UnlatchedCounter -} +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType +import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.MetricBuilder.MetricType +import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.util.Return +import com.twitter.util.Try import java.lang.{Float => JFloat} import java.util.concurrent.Callable import java.util.function.Supplier @@ -375,7 +375,8 @@ trait StatsReceiver { final def scope(namespaces: String*): StatsReceiver = namespaces.foldLeft(this)((statsReceiver, name) => statsReceiver.scope(name)) - protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = {} + protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Try[Unit] = + Return.Unit /** * Prepend a suffix value to the next scope. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala index c058020512..9dd6a6af62 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala @@ -1,6 +1,7 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.util.Try /** * A proxy [[StatsReceiver]] that delegates all calls to the `self` stats receiver. @@ -15,8 +16,9 @@ trait StatsReceiverProxy extends StatsReceiver with DelegatingStatsReceiver { def stat(metricBuilder: MetricBuilder): Stat = self.stat(metricBuilder) def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = self.addGauge(metricBuilder)(f) - override protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Unit = - self.registerExpression(expressionSchema) + override protected[finagle] def registerExpression( + expressionSchema: ExpressionSchema + ): Try[Unit] = self.registerExpression(expressionSchema) def underlying: Seq[StatsReceiver] = Seq(self) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index b19f9f8307..d7b125995a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -2,7 +2,9 @@ package com.twitter.finagle.stats.exp import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.exp.Expression.HistogramComponent -import com.twitter.finagle.stats.{Metadata, MetricBuilder, StatsReceiver} +import com.twitter.finagle.stats.Metadata +import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.StatsReceiver import scala.annotation.varargs private[twitter] object Expression { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index 698036259b..f13ceb3561 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -3,13 +3,13 @@ package com.twitter.finagle.stats.exp import com.twitter.finagle.stats.HistogramFormatter import com.twitter.finagle.stats.MetricUnit import com.twitter.finagle.stats.SourceRole -import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.Unspecified import com.twitter.finagle.stats.exp.Expression.Avg import com.twitter.finagle.stats.exp.Expression.Count import com.twitter.finagle.stats.exp.Expression.Max import com.twitter.finagle.stats.exp.Expression.Min import com.twitter.finagle.stats.exp.Expression.Sum +import com.twitter.util.Try /** * ExpressionSchema is builder class that construct an expression with its metadata. @@ -57,11 +57,17 @@ case class ExpressionSchema private ( private[finagle] def withServiceName(name: String): ExpressionSchema = withLabel(ExpressionSchema.ServiceName, name) - def register(): Unit = { + /** + * Register this ExpressionSchema in StatsReceiver. + * + * Returns: Unit when the schema is successfully published to StatsReceiver + * Throws: [[ExpressionCollisionException]] if the schemaKey has mapped value + */ + def build(): Try[Unit] = { Expression.getStatsReceivers(expr).toSeq match { case Seq(sr) => sr.registerExpression(this) - case srs: Seq[StatsReceiver] if srs.nonEmpty => srs.map(_.registerExpression(this)) - case _ => // should not happen + case first +: _ => first.registerExpression(this) + case _ => Try.Unit // should not happen } } @@ -84,6 +90,9 @@ case class ExpressionSchemaKey( // expose for testing in twitter-server private[twitter] object ExpressionSchema { + + case class ExpressionCollisionException(msg: String) extends IllegalArgumentException(msg) + val Role: String = "role" val ServiceName: String = "service_name" val ProcessPath: String = "process_path" diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 07e3472ee0..312f386622 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -1,10 +1,16 @@ package com.twitter.finagle.stats -import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType} -import com.twitter.finagle.stats.exp.{Expression, ExpressionSchema, ExpressionSchemaKey} -import java.io.{ByteArrayOutputStream, PrintStream} +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.exp.Expression +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey +import java.io.ByteArrayOutputStream +import java.io.PrintStream import java.nio.charset.StandardCharsets -import org.scalatest.concurrent.{Eventually, IntegrationPatience} +import org.scalatest.concurrent.Eventually +import org.scalatest.concurrent.IntegrationPatience import org.scalatest.funsuite.AnyFunSuite import scala.collection.parallel.immutable.ParRange @@ -378,7 +384,7 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra "test_expression", Expression(aCounter.metadata).plus(Expression(bHisto.metadata, Left(Expression.Min)) .plus(Expression(cGauge.metadata))) - ).register() + ).build() // what we expected as hydrated metric builders val aaSchema = @@ -405,12 +411,12 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra ExpressionSchema( "test_expression_0", Expression(aCounter.metadata) - ).withLabel(ExpressionSchema.Role, "test").register() + ).withLabel(ExpressionSchema.Role, "test").build() ExpressionSchema( "test_expression_1", Expression(aCounter.metadata) - ).withLabel(ExpressionSchema.Role, "dont appear").register() + ).withLabel(ExpressionSchema.Role, "dont appear").build() val withLabel = sr.getAllExpressionsWithLabel(ExpressionSchema.Role, "test") assert(withLabel.size == 1) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 7b92022ef6..4f41cad293 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -29,6 +29,16 @@ class ExpressionSchemaTest extends AnyFunSuite { throw new Exception("boom!") } } + + val downstreamLabel = + Map(ExpressionSchema.Role -> Client.toString, ExpressionSchema.ServiceName -> "downstream") + + protected[this] def nameToKey( + name: String, + labels: Map[String, String] = Map(), + namespaces: Seq[String] = Seq() + ): ExpressionSchemaKey = + ExpressionSchemaKey(name, downstreamLabel ++ labels, namespaces) } test("Demonstrate an end to end example of using metric expressions") { @@ -46,8 +56,8 @@ class ExpressionSchemaTest extends AnyFunSuite { .withUnit(Milliseconds) .withDescription("The latency of the slow query") - successRate.register() - latency.register() + successRate.build() + latency.build() def runTheQuery(succeed: Boolean): Unit = { Time.withCurrentTimeFrozen { ctl => @@ -66,34 +76,17 @@ class ExpressionSchemaTest extends AnyFunSuite { runTheQuery(true) runTheQuery(false) - val downstreamLabel = - Map(ExpressionSchema.Role -> Client.toString, ExpressionSchema.ServiceName -> "downstream") - assert( - sr.expressions( - ExpressionSchemaKey("success_rate", downstreamLabel, Nil)).name == successRate.name) - assert( - sr.expressions( - ExpressionSchemaKey("success_rate", downstreamLabel, Nil)).expr == successRate.expr) - assert( - sr.expressions( - ExpressionSchemaKey( - "latency", - downstreamLabel + ("bucket" -> "p99"), - Nil)).name == latency.name) - assert( - sr.expressions( - ExpressionSchemaKey( - "latency", - downstreamLabel + ("bucket" -> "p99"), - Nil)).expr == latency.expr) + assert(sr.expressions(nameToKey("success_rate")).name == successRate.name) + assert(sr.expressions(nameToKey("success_rate")).expr == successRate.expr) + assert(sr.expressions(nameToKey("latency", Map("bucket" -> "p99"))).name == latency.name) + assert(sr.expressions(nameToKey("latency", Map("bucket" -> "p99"))).expr == latency.expr) assert(sr.counters(Seq("success")) == 1) assert(sr.counters(Seq("failures")) == 1) assert(sr.stats(Seq("latency")) == Seq(50, 50)) assert( - sr.expressions(ExpressionSchemaKey("success_rate", downstreamLabel, Nil)).labels( - ExpressionSchema.Role) == Client.toString) + sr.expressions(nameToKey("success_rate")).labels(ExpressionSchema.Role) == Client.toString) } } @@ -109,23 +102,14 @@ class ExpressionSchemaTest extends AnyFunSuite { test("histogram expressions come with default labels") { new Ctx { - val latencyP90 = ExpressionSchema("latency", Expression(latencyMb, Right(0.9))).register() - val latencyP99 = ExpressionSchema("latency", Expression(latencyMb, Right(0.99))).register() + val latencyP90 = ExpressionSchema("latency", Expression(latencyMb, Right(0.9))).build() + val latencyP99 = ExpressionSchema("latency", Expression(latencyMb, Right(0.99))).build() val latencyAvg = - ExpressionSchema("latency", Expression(latencyMb, Left(Expression.Avg))).register() - - val downstreamLabel = - Map(ExpressionSchema.Role -> Client.toString, ExpressionSchema.ServiceName -> "downstream") + ExpressionSchema("latency", Expression(latencyMb, Left(Expression.Avg))).build() - assert( - sr.expressions.contains( - ExpressionSchemaKey("latency", downstreamLabel + ("bucket" -> "p90"), Nil))) - assert( - sr.expressions.contains( - ExpressionSchemaKey("latency", downstreamLabel + ("bucket" -> "p99"), Nil))) - assert( - sr.expressions.contains( - ExpressionSchemaKey("latency", downstreamLabel + ("bucket" -> "avg"), Nil))) + assert(sr.expressions.contains(nameToKey("latency", Map("bucket" -> "p90")))) + assert(sr.expressions.contains(nameToKey("latency", Map("bucket" -> "p99")))) + assert(sr.expressions.contains(nameToKey("latency", Map("bucket" -> "avg")))) } } @@ -136,14 +120,14 @@ class ExpressionSchemaTest extends AnyFunSuite { .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) .withUnit(Percentage) .withDescription("The success rate of the slow query") - .register() + .build() val successRate2 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) .withNamespace("section2", "a") .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) .withUnit(Percentage) .withDescription("The success rate of the slow query") - .register() + .build() assert(sr.expressions.values.size == 2) val namespaces = sr.expressions.values.toSeq.map { exprSchema => @@ -153,4 +137,31 @@ class ExpressionSchemaTest extends AnyFunSuite { assert(namespaces.contains(Seq("section2", "a"))) } } + + test("expressions with the same key - name, labels and namespaces") { + new Ctx { + val successRate1 = ExpressionSchema("success_rate", Expression(successMb)) + .withNamespace("section1") + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query") + .build() + + val successRate2 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) + .withNamespace("section1") + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query") + .build() + + // only store the first + // the second attempt will be a Throw + assert(sr.expressions.values.size == 1) + assert( + sr.expressions(nameToKey("success_rate", Map(), Seq("section1"))).expr == Expression( + successMb)) + assert(successRate1.isReturn) + assert(successRate2.isThrow) + } + } } From 0bfe12d3a6cf7c9540153e2ceef4098c187ea2a7 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Thu, 23 Sep 2021 19:44:58 +0000 Subject: [PATCH 567/796] util-stats: propagate CounterishGaugeType Problem: Regardess of having set `.withCounterishGauge()`, when building a gauge via the `MetricBuilder.gauge()` api, we always return a gauge schema. This makes the `.withCounterishGauge()`, in effect, non-functional. Solution: Rather than copying a `MetricBuilder` and resetting its `metricType`, simply verify that we're only calling the `.gauge()` `.counter()` or `.stat()` api on a `MetricBuilder` with the appropriate `metricType` JIRA Issues: CSL-11280, CSL-11336 Differential Revision: https://phabricator.twitter.biz/D749274 --- .../main/scala/com/twitter/jvm/JvmStats.scala | 4 +-- .../scala/com/twitter/jvm/JvmStatsTest.scala | 4 ++- .../twitter/finagle/stats/MetricBuilder.scala | 30 +++++++++---------- .../twitter/finagle/stats/StatsReceiver.scala | 5 +++- .../stats/MetricBuilderCompilationTest.java | 11 ++++--- .../finagle/stats/MetricBuilderTest.scala | 16 ++++++++++ 6 files changed, 46 insertions(+), 24 deletions(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 26b8cff02e..d06477c068 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -163,10 +163,10 @@ object JvmStats { } // note, these could be -1 if the collector doesn't have support for it. - val cycles = gcStats.addGauge("cycles") { + val cycles = gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge("cycles") { gcPool.map(_.getCollectionCount).filter(_ > 0).sum.toFloat } - val msec = gcStats.addGauge("msec") { + val msec = gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge("msec") { gcPool.map(_.getCollectionTime).filter(_ > 0).sum.toFloat } diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala index 5719b1a76e..441d8ff4f3 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala @@ -1,7 +1,9 @@ package com.twitter.jvm import com.twitter.finagle.stats.InMemoryStatsReceiver -import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey, MetricExpression} +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey +import com.twitter.finagle.stats.exp.MetricExpression import org.scalatest.funsuite.AnyFunSuite class JvmStatsTest extends AnyFunSuite { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 764823b6e7..9623225ba8 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -1,13 +1,11 @@ package com.twitter.finagle.stats -import com.twitter.finagle.stats.MetricBuilder.{ - CounterType, - CounterishGaugeType, - GaugeType, - HistogramType, - MetricType, - UnlatchedCounter -} +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType +import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.MetricBuilder.MetricType +import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter import java.util.function.Supplier import scala.annotation.varargs @@ -268,8 +266,8 @@ class MetricBuilder private ( */ @varargs final def counter(name: String*): Counter = { - val schema = withName(name: _*).counterSchema - this.statsReceiver.counter(schema) + this.statsReceiver.validateMetricType(this, CounterType) + this.statsReceiver.counter(this.withName(name: _*)) } /** @@ -277,8 +275,8 @@ class MetricBuilder private ( * @return the gauge created. */ final def gauge(name: String*)(f: => Float): Gauge = { - val schema = withName(name: _*).gaugeSchema - this.statsReceiver.addGauge(schema)(f) + this.statsReceiver.validateMetricType(this, GaugeType) + this.statsReceiver.addGauge(this.withName(name: _*))(f) } /** @@ -288,8 +286,8 @@ class MetricBuilder private ( */ @varargs final def gauge(f: Supplier[Float], name: String*): Gauge = { - val schema = withName(name: _*).gaugeSchema - this.statsReceiver.addGauge(schema)(f.get()) + this.statsReceiver.validateMetricType(this, GaugeType) + this.statsReceiver.addGauge(this.withName(name: _*))(f.get()) } /** @@ -298,8 +296,8 @@ class MetricBuilder private ( */ @varargs final def histogram(name: String*): Stat = { - val schema = withName(name: _*).histogramSchema - this.statsReceiver.stat(schema) + this.statsReceiver.validateMetricType(this, HistogramType) + this.statsReceiver.stat(this.withName(name: _*)) } def canEqual(other: Any): Boolean = other.isInstanceOf[MetricBuilder] diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 2acdfffe09..fb2f5e6556 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -404,7 +404,10 @@ trait StatsReceiver { private[stats] def counter0(name: String): Counter = counter(name) private[stats] def stat0(name: String): Stat = stat(name) - protected def validateMetricType(metricBuilder: MetricBuilder, expectedType: MetricType): Unit = { + private[stats] def validateMetricType( + metricBuilder: MetricBuilder, + expectedType: MetricType + ): Unit = { val typeMatch = expectedType match { case CounterType => metricBuilder.metricType == CounterType || metricBuilder.metricType == UnlatchedCounter diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java b/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java index 7beb7e8db4..fe53eb8508 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java @@ -33,9 +33,12 @@ public void testWithMethods() { @Test public void testConstructingMetrics() { StatsReceiver sr = new InMemoryStatsReceiver(); - MetricBuilder mb = sr.metricBuilder(MetricBuilder.CounterType$.MODULE$); - Gauge g = mb.gauge(() -> 3.0f, "my", "cool", "gauge"); - Stat s = mb.histogram("my", "cool", "histogram"); - Counter c = mb.counter("my", "cool", "counter"); + MetricBuilder gmb = sr.metricBuilder(MetricBuilder.GaugeType$.MODULE$); + MetricBuilder smb = sr.metricBuilder(MetricBuilder.HistogramType$.MODULE$); + MetricBuilder cmb = sr.metricBuilder(MetricBuilder.CounterType$.MODULE$); + + Gauge g = gmb.gauge(() -> 3.0f, "my", "cool", "gauge"); + Stat s = smb.histogram("my", "cool", "histogram"); + Counter c = cmb.counter("my", "cool", "counter"); } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala index 4c29c73e87..7d433e8957 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala @@ -1,6 +1,8 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType +import com.twitter.finagle.stats.MetricBuilder.GaugeType import org.scalatest.funsuite.AnyFunSuite class MetricBuilderTest extends AnyFunSuite { @@ -16,4 +18,18 @@ class MetricBuilderTest extends AnyFunSuite { metadataScopeSeparator.setSeparator("_aTerriblyLongStringForSomeReason_") assert(mb.toString.contains("foo_aTerriblyLongStringForSomeReason_bar")) } + + test("the .gauge() api builds CounterishGauge vs Gauge where appropriate") { + val mb = MetricBuilder( + name = Seq("c_gauge"), + metricType = GaugeType, + statsReceiver = new InMemoryStatsReceiver + ).withCounterishGauge + + assert(mb.metricType == CounterishGaugeType) + + val g = mb.gauge("c_gauge") { 1.0.toFloat } + + assert(g.metadata.toMetricBuilder.get.metricType == CounterishGaugeType) + } } From 10e0e8edc4d7bf297483aaf108b2416905f11a51 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 23 Sep 2021 21:25:21 +0000 Subject: [PATCH 568/796] util: add missing doc on latched counter Problem The faq section was deleted accidentally when we were refactoring the doc bird links in an earlier branch. Solution Add back the missing faq. Differential Revision: https://phabricator.twitter.biz/D750596 --- doc/src/sphinx/util-stats/faq.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/src/sphinx/util-stats/faq.rst b/doc/src/sphinx/util-stats/faq.rst index d44b57ba19..caa336d258 100644 --- a/doc/src/sphinx/util-stats/faq.rst +++ b/doc/src/sphinx/util-stats/faq.rst @@ -52,3 +52,29 @@ illustrates the advantages of `finagle-stats` over the older libraries. histograms. This is available on the `/admin/histogram `_ admin page. + +Are metric values reset or latched after an observability collection? +--------------------------------------------------------------------- +``Stats`` are latched by all three implementations. +This means that for each observability collection window, +you will see that window’s distribution of values. + +There is no notion of latching/resetting a ``Gauge``, as it is +always a discrete point in time value. This is the case across +all three implementations, except for CounterishGauges, which +creates a gauge that models an unlatched counter. + +``Counters`` have different behavior depending on which implementation +is being used. This in turn has an impact in how you write your +queries for your dashboards and alerts. Both Commons Metrics :sup:`[1]` and Commons Stats +are **not** “latched”, which means that their values are monotonically +increasing (assuming you never call ``incr()`` with a negative value). +This implies that you use `CQLs rate function `_ +if you want to see per minute counts. Ostrich’s ``Counters`` are latched +when they are collected (think of this as being a “reset”) +which means that you should not need to use ``rate`` in order +to see per minute values. + +:sup:`[1]` Note that by using ``finagle-stats`` and setting the flag +``com.twitter.finagle.stats.useCounterDeltas=true`` you can +get per-minute deltas in Commons Metrics. From 2035db3a058c703e47a2fabb8bad8b69ce5fdb17 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 30 Sep 2021 22:31:17 +0000 Subject: [PATCH 569/796] Twitter-oss: Prepare OSS libraries for release 21.9.0 Problem We want to release the next versions of our Twitter OSS libraries 21.9.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D754337 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 89b29dafcf..fed4688270 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.9.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 0bcef31a65..147304fe6c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -36,7 +36,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.8.0" +val utilCore = "com.twitter" %% "util-core" % "21.9.0" ``` # Units diff --git a/build.sbt b/build.sbt index 6371746a1a..4276d377a1 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.9.0-SNAPSHOT" +val releaseVersion = "21.9.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.4" From bf493f918154f173927fd5da4dd2ec807229302f Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Fri, 1 Oct 2021 01:09:58 +0000 Subject: [PATCH 570/796] Twitter-oss: update OSS libraries post-release to 21.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D754461 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 147304fe6c..2525fb59c1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 4276d377a1..41da52c1cb 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.9.0" +val releaseVersion = "21.10.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.4" From c2750767b44b3ab329028b4e3617b78b9124696e Mon Sep 17 00:00:00 2001 From: Kostas Pagratis Date: Sat, 9 Oct 2021 00:06:25 +0000 Subject: [PATCH 571/796] util-core: add convenience functions for java.time Instant, ZonedDateTime, and OffsetDateTime Problem: When working with both com.twitter.util.Time and java.time, switching to the latter is verbose. Solution: Add convenience functions to convert to java.time objects. Differential Revision: https://phabricator.twitter.biz/D757636 --- CHANGELOG.rst | 8 +++ .../main/scala/com/twitter/util/Time.scala | 29 ++++++++++- .../scala/com/twitter/util/TimeTest.scala | 50 ++++++++++++++++--- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fed4688270..d50d1474e9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,14 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-core: Add convenience methods to convert to java.time.ZonedDateTime and + java.time.OffsetDateTime. `toInstant`, `toZonedDateTime`, and `toOffsetDateTime` also preserve + nanosecond resolution. ``PHAB_ID=D757636`` + + 21.9.0 ------ diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index dcf80ad0d0..e94edb88ca 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -17,9 +17,15 @@ package com.twitter.util import java.io.Serializable -import java.time.{Clock, Instant} +import java.time.ZonedDateTime +import java.time.Clock +import java.time.Instant +import java.time.OffsetDateTime +import java.time.ZoneOffset import java.util.concurrent.TimeUnit -import java.util.{Date, Locale, TimeZone} +import java.util.Date +import java.util.Locale +import java.util.TimeZone /** * @define now @@ -688,6 +694,25 @@ sealed class Time private[util] (protected val nanos: Long) */ def toDate: Date = new Date(inMillis) + /** + * Converts this Time object to a java.time.Instant + */ + def toInstant: Instant = { + val millis = inMilliseconds + val nanos = inNanoseconds - (millis * Duration.NanosPerMillisecond) + Instant.ofEpochMilli(millis).plusNanos(nanos) + } + + /** + * Converts this Time object to a java.time.ZonedDateTime with ZoneId of UTC + */ + def toZonedDateTime: ZonedDateTime = ZonedDateTime.ofInstant(toInstant, ZoneOffset.UTC) + + /** + * Converts this Time object to a java.time.OffsetDateTime with ZoneId of UTC + */ + def toOffsetDateTime: OffsetDateTime = OffsetDateTime.ofInstant(toInstant, ZoneOffset.UTC) + private def writeReplace(): Object = TimeBox.Finite(inNanoseconds) /** diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 0bd395311b..6fb25bf350 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -1,13 +1,20 @@ package com.twitter.util -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, ObjectOutputStream} -import java.util.{Locale, TimeZone} +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.io.ObjectInputStream +import java.io.ObjectOutputStream +import java.util.Locale +import java.util.TimeZone import java.util.concurrent.TimeUnit - -import org.scalatest.concurrent.{Eventually, IntegrationPatience} +import org.scalatest.concurrent.Eventually +import org.scalatest.concurrent.IntegrationPatience import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks - import com.twitter.conversions.DurationOps._ +import java.time.Instant +import java.time.OffsetDateTime +import java.time.ZoneOffset +import java.time.ZonedDateTime import org.scalatest.wordspec.AnyWordSpec trait TimeLikeSpec[T <: TimeLike[T]] extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { @@ -509,7 +516,9 @@ class TimeTest val t = new Thread(r) t.start() assert(t.isAlive == true) - eventually { assert(t.getState == Thread.State.TIMED_WAITING) } + eventually { + assert(t.getState == Thread.State.TIMED_WAITING) + } ctl.advance(5.seconds) t.join() assert(t.isAlive == false) @@ -712,5 +721,34 @@ class TimeTest assert(t0.untilNow == 100.hours) } } + + "toInstant" in { + Time.withCurrentTimeFrozen { _ => + val instant = Instant.ofEpochMilli(Time.now.inMilliseconds) + assert(instant.toEpochMilli == Time.now.inMillis) + // java.time.Instant:getNano returns the nanoseconds of the second + assert(instant.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) + } + } + + "toZonedDateTime" in { + Time.withCurrentTimeFrozen { _ => + val zonedDateTime = + ZonedDateTime.ofInstant(Instant.ofEpochMilli(Time.now.inMilliseconds), ZoneOffset.UTC) + assert(Time.now.toZonedDateTime == zonedDateTime) + // java.time.Instant:getNano returns the nanoseconds of the second + assert(zonedDateTime.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) + } + } + + "toOffsetDateTime" in { + Time.withCurrentTimeFrozen { _ => + val offsetDateTime = + OffsetDateTime.ofInstant(Instant.ofEpochMilli(Time.now.inMilliseconds), ZoneOffset.UTC) + assert(Time.now.toOffsetDateTime == offsetDateTime) + // java.time.Instant:getNano returns the nanoseconds of the second + assert(offsetDateTime.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) + } + } } } From e550817b4da6a0f9d6c3f0efc3ff95fc8068f9f1 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 11 Oct 2021 17:11:35 +0000 Subject: [PATCH 572/796] oss: update code-of-conduct links Problem/Solution Update code of conduct links. Differential Revision: https://phabricator.twitter.biz/D759918 --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2525fb59c1..9948ead043 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,10 @@ public API breaks and changes in runtime behavior. ## Contributing +We feel that a welcoming community is important and we ask that you follow Twitter's +[Open Source Code of Conduct](https://github.com/twitter/.github/blob/main/code-of-conduct.md) +in all interactions with the community. + The `release` branch of this repository contains the latest stable release of Util, and weekly snapshots are published to the `develop` branch. In general pull requests should be submitted against `develop`. See From ad8b22b9adbe1958f33b08cf133a9baac2c0aa9a Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Wed, 13 Oct 2021 21:25:53 +0000 Subject: [PATCH 573/796] util/util-core: Fix conversions to units from Int in Scala3 This fixes lost source-compatibility for unit conversions from Int values in Scala3. As discussed in #295 I extracted/adapted this from #290 (previous discussion). Previously something like 1.toInt.second did compile in Scala2 (with default settings) but didn't in Scala3. I added forwarders for all units which use Long as base, as that seemed reasonable and safe. I didn't add an implicit conversion from Float to Double because floating-point types are dumb and I don't trust the conversion to be lossless in all cases. Signed-off-by: Yufan Gong Differential Revision: https://phabricator.twitter.biz/D759923 --- .../scala/com/twitter/conversions/DurationOps.scala | 12 ++++++++++++ .../com/twitter/conversions/StorageUnitOps.scala | 13 +++++++++++++ .../main/scala/com/twitter/conversions/U64Ops.scala | 13 +++++++++++++ .../com/twitter/conversions/DurationOpsTest.scala | 9 ++++++++- .../twitter/conversions/StorageUnitOpsTest.scala | 9 ++++++++- .../scala/com/twitter/conversions/U64OpsTest.scala | 6 ++++++ 6 files changed, 60 insertions(+), 2 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala b/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala index 8ca387897b..ea104ce058 100644 --- a/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/DurationOps.scala @@ -2,6 +2,7 @@ package com.twitter.conversions import com.twitter.util.Duration import java.util.concurrent.TimeUnit +import scala.language.implicitConversions /** * Implicits for writing readable [[com.twitter.util.Duration]]s. @@ -37,4 +38,15 @@ object DurationOps { def day: Duration = days } + /** + * Forwarder for Int, as Scala 3.0 doesn't seem to do the implicit conversion to Long anymore. + * This is not a bug, as Scala 2.13 already had a flag ("-Ywarn-implicit-widening") to turn on warnings/errors + * for that. + * + * The implicit conversion from Int to Long here doesn't lose precision and keeps backwards source compatibliity + * with previous releases. + */ + implicit def richDurationFromInt(numNanos: Int): RichDuration = + new RichDuration(numNanos.toLong) + } diff --git a/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala b/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala index 28ee0ef9a5..c7eac3846f 100644 --- a/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala +++ b/util-core/src/main/scala/com/twitter/conversions/StorageUnitOps.scala @@ -2,6 +2,8 @@ package com.twitter.conversions import com.twitter.util.StorageUnit +import scala.language.implicitConversions + /** * Implicits for writing readable [[com.twitter.util.StorageUnit]]s. * @@ -34,4 +36,15 @@ object StorageUnitOps { def million: Long = numBytes * 1000 * 1000 def billion: Long = numBytes * 1000 * 1000 * 1000 } + + /** + * Forwarder for Int, as Scala 3.0 doesn't seem to do the implicit conversion to Long anymore. + * This is not a bug, as Scala 2.13 already had a flag ("-Ywarn-implicit-widening") to turn on warnings/errors + * for that. + * + * The implicit conversion from Int to Long here doesn't lose precision and keeps backwards source compatibliity + * with previous releases. + */ + implicit def richStorageUnitFromInt(numBytes: Int): RichStorageUnit = + new RichStorageUnit(numBytes.toLong) } diff --git a/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala index c4cee43411..75599ffe57 100644 --- a/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala +++ b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala @@ -1,5 +1,7 @@ package com.twitter.conversions +import scala.language.implicitConversions + object U64Ops { /** @@ -19,4 +21,15 @@ object U64Ops { def toU64HexString: String = "%016x".format(self) } + /** + * Forwarder for Int, as Scala 3.0 doesn't seem to do the implicit conversion to Long anymore. + * This is not a bug, as Scala 2.13 already had a flag ("-Ywarn-implicit-widening") to turn on warnings/errors + * for that. + * + * The implicit conversion from Int to Long here doesn't lose precision and keeps backwards source compatibliity + * with previous releases. + */ + implicit def richU64LongFromInt(self: Int): RichU64Long = + new RichU64Long(self.toLong) + } diff --git a/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala index bc2143bc76..9e9538173d 100644 --- a/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/DurationOpsTest.scala @@ -3,8 +3,9 @@ package com.twitter.conversions import com.twitter.util.Duration import com.twitter.conversions.DurationOps._ import org.scalatest.funsuite.AnyFunSuite +import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -class DurationOpsTest extends AnyFunSuite { +class DurationOpsTest extends AnyFunSuite with ScalaCheckPropertyChecks { test("converts Duration.Zero") { assert(0.seconds eq Duration.Zero) assert(0.milliseconds eq Duration.Zero) @@ -17,4 +18,10 @@ class DurationOpsTest extends AnyFunSuite { assert(100L.nanoseconds == Duration.fromNanoseconds(100L)) } + test("conversion works for Int values") { + forAll { (intValue: Int) => + assert(intValue.seconds == intValue.toLong.seconds) + } + } + } diff --git a/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala index c46bfc7093..587b25cf73 100644 --- a/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/StorageUnitOpsTest.scala @@ -3,8 +3,9 @@ package com.twitter.conversions import com.twitter.util.StorageUnit import com.twitter.conversions.StorageUnitOps._ import org.scalatest.funsuite.AnyFunSuite +import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -class StorageUnitOpsTest extends AnyFunSuite { +class StorageUnitOpsTest extends AnyFunSuite with ScalaCheckPropertyChecks { test("converts") { assert(StorageUnit.fromBytes(1) == 1.byte) @@ -26,4 +27,10 @@ class StorageUnitOpsTest extends AnyFunSuite { assert(StorageUnit.fromPetabytes(7) == 7.petabytes) } + test("conversion works for Int values") { + forAll { (intValue: Int) => + assert(intValue.bytes == intValue.toLong.bytes) + } + } + } diff --git a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala index 942c41bb8c..f6c20fc168 100644 --- a/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala +++ b/util-core/src/test/scala/com/twitter/conversions/U64OpsTest.scala @@ -16,4 +16,10 @@ class U64OpsTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { assert(s.toU64Long == java.lang.Long.parseUnsignedLong(s, 16)) } } + + test("conversion works for Int values") { + forAll { (intValue: Int) => + assert(intValue.toU64HexString == intValue.toLong.toU64HexString) + } + } } From 91551dc95e07e0ed487eea6035b948185fa9dd80 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Thu, 14 Oct 2021 15:10:18 +0000 Subject: [PATCH 574/796] [util] set strict_deps = True Problem/Solution Enable strict deps for Pants for util:: targets Differential Revision: https://phabricator.twitter.biz/D757554 --- .../src/main/scala/com/twitter/app/lifecycle/BUILD | 1 + .../src/test/scala/com/twitter/app/lifecycle/BUILD | 1 + util-app/src/main/java/com/twitter/app/BUILD | 1 + util-app/src/main/scala/BUILD | 2 ++ util-app/src/test/java/BUILD | 1 + util-app/src/test/scala/BUILD | 2 ++ util-benchmark/src/main/scala/BUILD | 1 + util-benchmark/src/test/scala/BUILD | 1 + .../src/main/scala/com/twitter/cache/guava/BUILD | 1 + .../src/test/java/com/twitter/cache/guava/BUILD | 1 + .../src/test/scala/com/twitter/cache/guava/BUILD | 1 + util-cache/src/main/scala/BUILD | 1 + util-cache/src/main/scala/com/twitter/cache/BUILD | 1 + .../src/main/scala/com/twitter/cache/caffeine/BUILD | 1 + util-cache/src/test/java/BUILD | 1 + util-cache/src/test/scala/com/twitter/cache/BUILD | 2 ++ .../src/test/scala/com/twitter/cache/caffeine/BUILD | 1 + util-codec/src/main/scala/com/twitter/util/BUILD | 1 + util-codec/src/test/scala/com/twitter/util/BUILD | 1 + util-core/BUILD | 2 ++ util-core/src/main/java/com/twitter/concurrent/BUILD | 1 + util-core/src/main/java/com/twitter/io/BUILD | 1 + util-core/src/main/java/com/twitter/service/BUILD | 1 + util-core/src/main/java/com/twitter/util/BUILD | 1 + .../src/main/java/com/twitter/util/javainterop/BUILD | 1 + util-core/src/main/scala/com/twitter/concurrent/BUILD | 1 + util-core/src/main/scala/com/twitter/conversions/BUILD | 1 + util-core/src/main/scala/com/twitter/io/BUILD | 1 + util-core/src/test/java/com/twitter/concurrent/BUILD | 1 + util-core/src/test/java/com/twitter/io/BUILD | 1 + util-core/src/test/java/com/twitter/service/BUILD | 1 + util-core/src/test/java/com/twitter/util/BUILD | 2 ++ .../src/test/java/com/twitter/util/javainterop/BUILD | 1 + util-core/src/test/scala/com/twitter/concurrent/BUILD | 1 + util-core/src/test/scala/com/twitter/conversions/BUILD | 1 + util-core/src/test/scala/com/twitter/io/BUILD | 1 + util-core/src/test/scala/com/twitter/util/BUILD | 1 + util-function/src/main/java/com/twitter/function/BUILD | 1 + util-hashing/src/main/scala/com/twitter/hashing/BUILD | 1 + util-hashing/src/test/scala/com/twitter/hashing/BUILD | 1 + util-inject/src/test/java/com/twitter/util/inject/BUILD | 1 + util-inject/src/test/scala/com/twitter/util/inject/BUILD | 1 + .../src/test/java/com/twitter/util/jackson/BUILD | 2 +- .../src/test/java/com/twitter/util/jackson/tests/BUILD | 1 + .../src/test/scala/com/twitter/util/jackson/BUILD | 1 + util-jvm/src/main/scala/com/twitter/jvm/BUILD | 1 + util-jvm/src/test/java/BUILD | 1 + util-jvm/src/test/scala/BUILD | 9 +++++++-- util-lint/src/main/scala/com/twitter/util/lint/BUILD | 1 + util-lint/src/test/scala/com/twitter/util/lint/BUILD | 1 + util-logging/src/main/scala/com/twitter/logging/BUILD | 1 + util-logging/src/test/java/BUILD | 1 + util-logging/src/test/scala/BUILD | 1 + util-mock/src/test/java/com/twitter/util/mock/BUILD | 2 +- util-mock/src/test/scala/com/twitter/util/mock/BUILD | 1 + .../src/main/scala/com/twitter/util/reflect/BUILD | 1 + .../src/test/java/com/twitter/util/reflect/BUILD | 1 + .../src/test/scala/com/twitter/util/reflect/BUILD | 1 + .../src/main/scala/com/twitter/util/registry/BUILD | 1 + util-registry/src/test/java/BUILD | 1 + util-registry/src/test/scala/BUILD | 1 + .../src/main/scala/com/twitter/util/routing/BUILD | 1 + .../src/test/java/com/twitter/util/routing/BUILD | 1 + .../src/test/scala/com/twitter/util/routing/BUILD | 1 + .../test/scala/com/twitter/util/routing/dynamic/BUILD | 1 + .../src/test/scala/com/twitter/util/routing/simple/BUILD | 1 + .../src/main/scala/com/twitter/util/security/BUILD | 1 + .../src/test/java/com/twitter/util/security/BUILD | 1 + .../src/test/scala/com/twitter/util/security/BUILD | 1 + .../src/main/scala/com/twitter/util/logging/BUILD | 1 + .../src/test/java/com/twitter/util/logging/BUILD | 1 + .../src/test/scala/com/twitter/util/logging/BUILD | 3 +++ .../src/main/scala/com/twitter/util/logging/BUILD | 1 + .../src/main/scala/com/twitter/finagle/stats/BUILD | 1 + util-stats/src/test/java/BUILD | 1 + util-stats/src/test/scala/BUILD | 1 + util-test/src/main/scala/BUILD | 1 + util-test/src/main/scala/com/twitter/logging/BUILD | 1 + util-test/src/main/scala/com/twitter/util/testing/BUILD | 1 + util-test/src/test/scala/com/twitter/util/testing/BUILD | 1 + util-thrift/src/main/scala/com/twitter/util/BUILD | 1 + util-thrift/src/test/java/com/twitter/util/BUILD | 1 + util-thrift/src/test/scala/com/twitter/util/BUILD | 1 + .../src/main/scala/com/twitter/util/tunable/BUILD | 1 + .../src/main/scala/com/twitter/util/tunable/linter/BUILD | 1 + .../src/test/java/com/twitter/util/tunable/BUILD | 1 + .../src/test/scala/com/twitter/util/tunable/BUILD | 1 + .../test/java/com/twitter/util/validation/tests/BUILD | 1 + .../src/test/scala/com/twitter/util/validation/BUILD | 1 + util-zk-test/src/main/scala/com/twitter/zk/BUILD | 1 + util-zk-test/src/test/java/BUILD | 1 + util-zk-test/src/test/scala/BUILD | 1 + util-zk/src/main/scala/BUILD | 1 + util-zk/src/main/scala/com/twitter/zk/BUILD | 1 + util-zk/src/main/scala/com/twitter/zk/coordination/BUILD | 1 + util-zk/src/test/scala/com/twitter/zk/BUILD | 1 + util-zk/src/test/scala/com/twitter/zk/coordination/BUILD | 1 + 97 files changed, 110 insertions(+), 4 deletions(-) diff --git a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD index 2e1962913b..a723d1b67e 100644 --- a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD +++ b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-app-lifecycle", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD index 2bcfb504e7..ee6ac23c93 100644 --- a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD +++ b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-app/src/main/java/com/twitter/app/BUILD b/util-app/src/main/java/com/twitter/app/BUILD index 47798edc03..3267814a89 100644 --- a/util-app/src/main/java/com/twitter/app/BUILD +++ b/util-app/src/main/java/com/twitter/app/BUILD @@ -7,5 +7,6 @@ java_library( ), # Not detected as required by zinc. scope = "forced", + strict_deps = True, tags = ["bazel-compatible"], ) diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index 259e20a441..831041edb1 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -6,11 +6,13 @@ scala_library( name = "util-app", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle", "util/util-app/src/main/java/com/twitter/app", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-core/src/main/scala/com/twitter/io", "util/util-registry/src/main/scala/com/twitter/util/registry", diff --git a/util-app/src/test/java/BUILD b/util-app/src/test/java/BUILD index bc7b8f6f54..5783c51559 100644 --- a/util-app/src/test/java/BUILD +++ b/util-app/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index 3fe4b661a6..7f352a7b01 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", @@ -14,6 +15,7 @@ junit_tests( "util/util-app/src/main/scala", "util/util-app/src/test/resources", "util/util-core:util-core-util", + "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index 78af6dda35..6570923a1e 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -4,6 +4,7 @@ scala_library( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", diff --git a/util-benchmark/src/test/scala/BUILD b/util-benchmark/src/test/scala/BUILD index 9f3bc2c1cb..97c15bc825 100644 --- a/util-benchmark/src/test/scala/BUILD +++ b/util-benchmark/src/test/scala/BUILD @@ -3,6 +3,7 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scalacheck", diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD index e174575ae9..bd4f3a57ad 100644 --- a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-cache-guava", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/guava", diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD index 99fb6f4bcb..b13589394c 100644 --- a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD index 0fa75f778e..1245078c44 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-cache/src/main/scala/BUILD b/util-cache/src/main/scala/BUILD index e280d8024a..03257e94d6 100644 --- a/util-cache/src/main/scala/BUILD +++ b/util-cache/src/main/scala/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-cache", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-cache/src/main/scala/com/twitter/cache", diff --git a/util-cache/src/main/scala/com/twitter/cache/BUILD b/util-cache/src/main/scala/com/twitter/cache/BUILD index bf995eff07..476e56da0f 100644 --- a/util-cache/src/main/scala/com/twitter/cache/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-cache-cache", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD index e7cb5b2e03..43d840823a 100644 --- a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-cache-caffeine", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", diff --git a/util-cache/src/test/java/BUILD b/util-cache/src/test/java/BUILD index 530e0dc9f9..705f570301 100644 --- a/util-cache/src/test/java/BUILD +++ b/util-cache/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index ff9989b804..64b862b654 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -5,6 +5,7 @@ junit_tests( "*.scala", ], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", @@ -31,6 +32,7 @@ scala_library( "AbstractLoadingFutureCacheTest.scala", ], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/junit", diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index d134aa8d72..b0931bfe58 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-codec/src/main/scala/com/twitter/util/BUILD b/util-codec/src/main/scala/com/twitter/util/BUILD index 7121b15b7e..631158a2b6 100644 --- a/util-codec/src/main/scala/com/twitter/util/BUILD +++ b/util-codec/src/main/scala/com/twitter/util/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-codec", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core/src/main/scala/com/twitter/io", diff --git a/util-codec/src/test/scala/com/twitter/util/BUILD b/util-codec/src/test/scala/com/twitter/util/BUILD index 413e57007b..996e06795b 100644 --- a/util-codec/src/test/scala/com/twitter/util/BUILD +++ b/util-codec/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-core/BUILD b/util-core/BUILD index f1f1666ecf..d4877d3f6d 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -32,6 +32,7 @@ scala_library( name = "util-core-util", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-reflect", @@ -55,6 +56,7 @@ scala_library( name = "util-core", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ ":util-core-util", diff --git a/util-core/src/main/java/com/twitter/concurrent/BUILD b/util-core/src/main/java/com/twitter/concurrent/BUILD index 29917ba462..cbe1b1c4b9 100644 --- a/util-core/src/main/java/com/twitter/concurrent/BUILD +++ b/util-core/src/main/java/com/twitter/concurrent/BUILD @@ -5,6 +5,7 @@ java_library( name = "util-core-java-concurrent", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/main/java/com/twitter/io/BUILD b/util-core/src/main/java/com/twitter/io/BUILD index 83f7ca73fc..902ed1fdbd 100644 --- a/util-core/src/main/java/com/twitter/io/BUILD +++ b/util-core/src/main/java/com/twitter/io/BUILD @@ -5,6 +5,7 @@ java_library( name = "util-core-java-io", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/main/java/com/twitter/service/BUILD b/util-core/src/main/java/com/twitter/service/BUILD index fd6484b981..6fd0561ef6 100644 --- a/util-core/src/main/java/com/twitter/service/BUILD +++ b/util-core/src/main/java/com/twitter/service/BUILD @@ -6,5 +6,6 @@ java_library( name = "util-core-java-service", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], ) diff --git a/util-core/src/main/java/com/twitter/util/BUILD b/util-core/src/main/java/com/twitter/util/BUILD index d0e3d94d13..47cf7a5d8a 100644 --- a/util-core/src/main/java/com/twitter/util/BUILD +++ b/util-core/src/main/java/com/twitter/util/BUILD @@ -5,6 +5,7 @@ java_library( name = "util-core-java", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/main/java/com/twitter/util/javainterop/BUILD b/util-core/src/main/java/com/twitter/util/javainterop/BUILD index cd785e5fc0..b3ef14a3a4 100644 --- a/util-core/src/main/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/main/java/com/twitter/util/javainterop/BUILD @@ -5,6 +5,7 @@ java_library( name = "util-core-java-javainterop", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scala-lang:scala-library", diff --git a/util-core/src/main/scala/com/twitter/concurrent/BUILD b/util-core/src/main/scala/com/twitter/concurrent/BUILD index 19b636b184..c3e2fb8676 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/main/scala/com/twitter/concurrent/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-core-concurrent", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-core/src/main/scala/com/twitter/conversions/BUILD b/util-core/src/main/scala/com/twitter/conversions/BUILD index 61cd4788da..ce2c90c609 100644 --- a/util-core/src/main/scala/com/twitter/conversions/BUILD +++ b/util-core/src/main/scala/com/twitter/conversions/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-core-conversions", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-core/src/main/scala/com/twitter/io/BUILD b/util-core/src/main/scala/com/twitter/io/BUILD index 0e936c2914..ce279b5be3 100644 --- a/util-core/src/main/scala/com/twitter/io/BUILD +++ b/util-core/src/main/scala/com/twitter/io/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-core-io", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-core/src/test/java/com/twitter/concurrent/BUILD b/util-core/src/test/java/com/twitter/concurrent/BUILD index e4c62645a0..9444394b34 100644 --- a/util-core/src/test/java/com/twitter/concurrent/BUILD +++ b/util-core/src/test/java/com/twitter/concurrent/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD index d7c025c52e..917232f1af 100644 --- a/util-core/src/test/java/com/twitter/io/BUILD +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-core/src/test/java/com/twitter/service/BUILD b/util-core/src/test/java/com/twitter/service/BUILD index 4ad37e5f2c..045267f323 100644 --- a/util-core/src/test/java/com/twitter/service/BUILD +++ b/util-core/src/test/java/com/twitter/service/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index bcdbfdad4b..70d15c860c 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -2,6 +2,7 @@ java_library( name = "object_size_calculator", sources = ["ObjectSizeCalculator.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", @@ -15,6 +16,7 @@ junit_tests( "*.java", ], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-core/src/test/java/com/twitter/util/javainterop/BUILD b/util-core/src/test/java/com/twitter/util/javainterop/BUILD index d1c67a6836..157a877319 100644 --- a/util-core/src/test/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/test/java/com/twitter/util/javainterop/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index c89563ca72..0ea4d86f1a 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD index 0ff8820550..58d6c35e82 100644 --- a/util-core/src/test/scala/com/twitter/conversions/BUILD +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 2a8f0b9245..1d21d9cb2d 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index 2cde85376f..5983bc72c2 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-function/src/main/java/com/twitter/function/BUILD b/util-function/src/main/java/com/twitter/function/BUILD index 3e368d9187..7be7675e1b 100644 --- a/util-function/src/main/java/com/twitter/function/BUILD +++ b/util-function/src/main/java/com/twitter/function/BUILD @@ -5,5 +5,6 @@ java_library( name = "util-function-java", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], ) diff --git a/util-hashing/src/main/scala/com/twitter/hashing/BUILD b/util-hashing/src/main/scala/com/twitter/hashing/BUILD index 3986ec5c74..aaf6592722 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/main/scala/com/twitter/hashing/BUILD @@ -5,5 +5,6 @@ scala_library( name = "util-hashing", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], ) diff --git a/util-hashing/src/test/scala/com/twitter/hashing/BUILD b/util-hashing/src/test/scala/com/twitter/hashing/BUILD index 1e2a259c9f..4d0be301d8 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/test/scala/com/twitter/hashing/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-inject/src/test/java/com/twitter/util/inject/BUILD b/util-inject/src/test/java/com/twitter/util/inject/BUILD index 3fda5202cc..eb4a1ebe48 100644 --- a/util-inject/src/test/java/com/twitter/util/inject/BUILD +++ b/util-inject/src/test/java/com/twitter/util/inject/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = [], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/google/inject:guice", diff --git a/util-inject/src/test/scala/com/twitter/util/inject/BUILD b/util-inject/src/test/scala/com/twitter/util/inject/BUILD index ef10ad10fe..5ee73e7571 100644 --- a/util-inject/src/test/scala/com/twitter/util/inject/BUILD +++ b/util-inject/src/test/scala/com/twitter/util/inject/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/BUILD index 68c104eb52..ba1cb182ca 100644 --- a/util-jackson/src/test/java/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/java/com/twitter/util/jackson/BUILD @@ -1,7 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - strict_deps = "True", + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD index ce8b05468f..1909557de1 100644 --- a/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD +++ b/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD @@ -1,5 +1,6 @@ junit_tests( sources = ["**/*.java"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/dataformat:jackson-dataformat-yaml", diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD index 8ba58ec7ea..8bc8aedc45 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD @@ -5,6 +5,7 @@ junit_tests( "caseclass/*.scala", "serde/*.scala", ], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", diff --git a/util-jvm/src/main/scala/com/twitter/jvm/BUILD b/util-jvm/src/main/scala/com/twitter/jvm/BUILD index 665c3259a2..759853a0e7 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/main/scala/com/twitter/jvm/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-jvm", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-app/src/main/scala", diff --git a/util-jvm/src/test/java/BUILD b/util-jvm/src/test/java/BUILD index 3ef7054c13..0f9aa0b2c7 100644 --- a/util-jvm/src/test/java/BUILD +++ b/util-jvm/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index b2295dac08..a98d11ebc0 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -15,11 +15,15 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", ], - dependencies = COMMON_DEPS + ["3rdparty/jvm/org/scalatestplus:junit"], + dependencies = COMMON_DEPS + [ + "3rdparty/jvm/org/scalatestplus:junit", + "util/util-stats/src/main/scala/com/twitter/finagle/stats:stats", + ], ) scala_library( @@ -32,8 +36,9 @@ scala_library( # does not strictly enforce it. sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], - dependencies = COMMON_DEPS, + dependencies = COMMON_DEPS + ["util/util-stats/src/main/scala/com/twitter/finagle/stats:stats"], ) jvm_binary( diff --git a/util-lint/src/main/scala/com/twitter/util/lint/BUILD b/util-lint/src/main/scala/com/twitter/util/lint/BUILD index b2b903bfc7..bd8676cd38 100644 --- a/util-lint/src/main/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/main/scala/com/twitter/util/lint/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-lint", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index 5856cf7ac8..b6834e255b 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-logging/src/main/scala/com/twitter/logging/BUILD b/util-logging/src/main/scala/com/twitter/logging/BUILD index e59dd7b84b..73efcef057 100644 --- a/util-logging/src/main/scala/com/twitter/logging/BUILD +++ b/util-logging/src/main/scala/com/twitter/logging/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-logging", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-app/src/main/scala", diff --git a/util-logging/src/test/java/BUILD b/util-logging/src/test/java/BUILD index af4c2a831c..41a7884036 100644 --- a/util-logging/src/test/java/BUILD +++ b/util-logging/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 3f81913fd5..6126473b03 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-mock/src/test/java/com/twitter/util/mock/BUILD b/util-mock/src/test/java/com/twitter/util/mock/BUILD index 3c58bcbf7d..639cf3823f 100644 --- a/util-mock/src/test/java/com/twitter/util/mock/BUILD +++ b/util-mock/src/test/java/com/twitter/util/mock/BUILD @@ -1,7 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], - strict_deps = "True", + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ ], diff --git a/util-mock/src/test/scala/com/twitter/util/mock/BUILD b/util-mock/src/test/scala/com/twitter/util/mock/BUILD index fab2e74fe3..5eef907b39 100644 --- a/util-mock/src/test/scala/com/twitter/util/mock/BUILD +++ b/util-mock/src/test/scala/com/twitter/util/mock/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD index 74ea3ef074..974f7e9415 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-reflect", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/BUILD b/util-reflect/src/test/java/com/twitter/util/reflect/BUILD index 773deadb6b..4acac5497f 100644 --- a/util-reflect/src/test/java/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/java/com/twitter/util/reflect/BUILD @@ -1,5 +1,6 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], ) diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD index 7addfa0ec1..2f49348915 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/junit", diff --git a/util-registry/src/main/scala/com/twitter/util/registry/BUILD b/util-registry/src/main/scala/com/twitter/util/registry/BUILD index 62cfff778a..d8c88c2b2c 100644 --- a/util-registry/src/main/scala/com/twitter/util/registry/BUILD +++ b/util-registry/src/main/scala/com/twitter/util/registry/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-registry", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-registry/src/test/java/BUILD b/util-registry/src/test/java/BUILD index f433781d70..ea04e8fe8b 100644 --- a/util-registry/src/test/java/BUILD +++ b/util-registry/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index a245b56740..825a2e1355 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-routing/src/main/scala/com/twitter/util/routing/BUILD b/util-routing/src/main/scala/com/twitter/util/routing/BUILD index 58ed27f316..c93d9fb289 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/main/scala/com/twitter/util/routing/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-routing", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-routing/src/test/java/com/twitter/util/routing/BUILD b/util-routing/src/test/java/com/twitter/util/routing/BUILD index b5a2cd8386..654b453ac1 100644 --- a/util-routing/src/test/java/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/java/com/twitter/util/routing/BUILD @@ -3,6 +3,7 @@ junit_tests( "*.java", ], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index 80c5daa72b..cfe39b7d5b 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD index bdc9144cd6..3b46f0d23f 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD index 49ab57e12e..8d095944d5 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-security/src/main/scala/com/twitter/util/security/BUILD b/util-security/src/main/scala/com/twitter/util/security/BUILD index ce3549f163..75f995aca1 100644 --- a/util-security/src/main/scala/com/twitter/util/security/BUILD +++ b/util-security/src/main/scala/com/twitter/util/security/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-security", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD index f5d30abd13..2997d293b2 100644 --- a/util-security/src/test/java/com/twitter/util/security/BUILD +++ b/util-security/src/test/java/com/twitter/util/security/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index fcb291b7d7..006ba93dff 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD index 344c06a330..ba59507a01 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-slf4j", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/slf4j:slf4j-api", diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD index 698c646f0b..4b0fca6eb2 100644 --- a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index f09e317844..78fc535f3e 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -5,6 +5,7 @@ scala_library( "*.java", "*.scala", ], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/mockito:mockito-core", @@ -27,6 +28,7 @@ scala_library( junit_tests( compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", @@ -36,6 +38,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-scala", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", + "3rdparty/src/jvm/org/scalatestplus/mockito", scoped( "3rdparty/jvm/org/slf4j:slf4j-simple", scope = "runtime", diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD index 9d76fc524f..2fad866749 100644 --- a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-slf4j-jul-bridge", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/slf4j:jul-to-slf4j", diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD index 188e68f068..e3896b3b00 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-stats", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson:jackson-module-scala", diff --git a/util-stats/src/test/java/BUILD b/util-stats/src/test/java/BUILD index b309d5dbda..ecf328608b 100644 --- a/util-stats/src/test/java/BUILD +++ b/util-stats/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-stats/src/test/scala/BUILD b/util-stats/src/test/scala/BUILD index 8db981842a..7296b62097 100644 --- a/util-stats/src/test/scala/BUILD +++ b/util-stats/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-test/src/main/scala/BUILD b/util-test/src/main/scala/BUILD index fa0246117b..096495e79a 100644 --- a/util-test/src/main/scala/BUILD +++ b/util-test/src/main/scala/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-test", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-test/src/main/scala/com/twitter/logging", diff --git a/util-test/src/main/scala/com/twitter/logging/BUILD b/util-test/src/main/scala/com/twitter/logging/BUILD index 6214ce299b..1b130090cb 100644 --- a/util-test/src/main/scala/com/twitter/logging/BUILD +++ b/util-test/src/main/scala/com/twitter/logging/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-test-logging", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/scalatest", diff --git a/util-test/src/main/scala/com/twitter/util/testing/BUILD b/util-test/src/main/scala/com/twitter/util/testing/BUILD index 1fb374f8da..1c151e7679 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/main/scala/com/twitter/util/testing/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-test-testing", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/mockito:mockito-all", diff --git a/util-test/src/test/scala/com/twitter/util/testing/BUILD b/util-test/src/test/scala/com/twitter/util/testing/BUILD index b007195ef9..192e06ef14 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/test/scala/com/twitter/util/testing/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-thrift/src/main/scala/com/twitter/util/BUILD b/util-thrift/src/main/scala/com/twitter/util/BUILD index af3920f510..6809678a85 100644 --- a/util-thrift/src/main/scala/com/twitter/util/BUILD +++ b/util-thrift/src/main/scala/com/twitter/util/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-thrift", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-databind", diff --git a/util-thrift/src/test/java/com/twitter/util/BUILD b/util-thrift/src/test/java/com/twitter/util/BUILD index e2b3c0c2c8..53d6e32382 100644 --- a/util-thrift/src/test/java/com/twitter/util/BUILD +++ b/util-thrift/src/test/java/com/twitter/util/BUILD @@ -1,5 +1,6 @@ java_library( compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/apache/thrift:libthrift", diff --git a/util-thrift/src/test/scala/com/twitter/util/BUILD b/util-thrift/src/test/scala/com/twitter/util/BUILD index ca918342ff..faef2628f8 100644 --- a/util-thrift/src/test/scala/com/twitter/util/BUILD +++ b/util-thrift/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD index 512cc0b812..3e2f06da1f 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-tunable", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD index 562f0c25dc..dd8f13c04c 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD @@ -1,5 +1,6 @@ scala_library( name = "scala", + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-core", diff --git a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD index 6a08d5c20d..faf49c68c9 100644 --- a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD index 5ae4bc2cbf..c5d87fc0fb 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD b/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD index 0308bb0f58..fc358ab7b7 100644 --- a/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD +++ b/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD @@ -1,5 +1,6 @@ junit_tests( sources = ["**/*.java"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", diff --git a/util-validator/src/test/scala/com/twitter/util/validation/BUILD b/util-validator/src/test/scala/com/twitter/util/validation/BUILD index 4eef2dd742..b0671326b5 100644 --- a/util-validator/src/test/scala/com/twitter/util/validation/BUILD +++ b/util-validator/src/test/scala/com/twitter/util/validation/BUILD @@ -4,6 +4,7 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/fasterxml/jackson/core:jackson-annotations", diff --git a/util-zk-test/src/main/scala/com/twitter/zk/BUILD b/util-zk-test/src/main/scala/com/twitter/zk/BUILD index 9b856ddfbb..8b51a37b97 100644 --- a/util-zk-test/src/main/scala/com/twitter/zk/BUILD +++ b/util-zk-test/src/main/scala/com/twitter/zk/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-zk-test", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-server", diff --git a/util-zk-test/src/test/java/BUILD b/util-zk-test/src/test/java/BUILD index 471b7c9d56..475c98385b 100644 --- a/util-zk-test/src/test/java/BUILD +++ b/util-zk-test/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-zk-test/src/test/scala/BUILD b/util-zk-test/src/test/scala/BUILD index fb9a61bba3..286b63f686 100644 --- a/util-zk-test/src/test/scala/BUILD +++ b/util-zk-test/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-zk/src/main/scala/BUILD b/util-zk/src/main/scala/BUILD index 19e04b8761..81f73e9c24 100644 --- a/util-zk/src/main/scala/BUILD +++ b/util-zk/src/main/scala/BUILD @@ -6,6 +6,7 @@ scala_library( name = "util-zk", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "util/util-zk/src/main/scala/com/twitter/zk", diff --git a/util-zk/src/main/scala/com/twitter/zk/BUILD b/util-zk/src/main/scala/com/twitter/zk/BUILD index da966400d6..855cff33d2 100644 --- a/util-zk/src/main/scala/com/twitter/zk/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-zk-zk", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD index 8c82b86b7f..fadff9ef1c 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD @@ -5,6 +5,7 @@ scala_library( name = "util-zk-coordination", repo = artifactory, ), + strict_deps = True, tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/org/apache/zookeeper:zookeeper-client", diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index 5f80696f0a..ec09793534 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index ebaf0e721d..9e3c40a7c7 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + strict_deps = True, tags = [ "bazel-compatible", "non-exclusive", From b8a30a72928c4192e3fc0fcfd8b77df6a901f2c9 Mon Sep 17 00:00:00 2001 From: Felix Bruckmeier Date: Mon, 18 Oct 2021 19:19:30 +0000 Subject: [PATCH 575/796] util/util-logging: Add Scala3 cross-build Cross build util-logging with Scala 3. Util PR #295 Signed-off-by: Hamdi Allam JIRA Issues: CSL-11409 Differential Revision: https://phabricator.twitter.biz/D764508 --- build.sbt | 2 +- .../com/twitter/logging/FileHandler.scala | 23 +++++++++++---- .../scala/com/twitter/logging/Handler.scala | 2 +- .../scala/com/twitter/logging/Logger.scala | 4 +-- .../com/twitter/logging/FileHandlerTest.scala | 11 ++++++-- .../com/twitter/logging/LogRecordTest.scala | 4 +-- .../com/twitter/logging/LoggerTest.scala | 28 +++++++++++-------- .../twitter/logging/SyslogHandlerTest.scala | 9 +++--- 8 files changed, 53 insertions(+), 30 deletions(-) diff --git a/build.sbt b/build.sbt index 41da52c1cb..2c2183b66f 100644 --- a/build.sbt +++ b/build.sbt @@ -540,7 +540,7 @@ lazy val utilLogging = Project( id = "util-logging", base = file("util-logging") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-logging" ).dependsOn(utilCore, utilApp, utilStats) diff --git a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala index b36bfa2f5d..75d1e8fa31 100644 --- a/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/FileHandler.scala @@ -16,11 +16,21 @@ package com.twitter.logging -import java.io.{File, FileOutputStream, FilenameFilter, OutputStream} +import java.io.File +import java.io.FileOutputStream +import java.io.FilenameFilter +import java.io.OutputStream import java.nio.charset.Charset -import java.util.{Calendar, Date, logging => javalog} +import java.util.Calendar +import java.util.Date +import java.util.{logging => javalog} -import com.twitter.util.{TwitterDateFormat, HandleSignal, Return, StorageUnit, Time, Try} +import com.twitter.util.TwitterDateFormat +import com.twitter.util.HandleSignal +import com.twitter.util.Return +import com.twitter.util.StorageUnit +import com.twitter.util.Time +import com.twitter.util.Try sealed abstract class Policy object Policy { @@ -224,9 +234,12 @@ class FileHandler( } case Policy.Weekly(weekday) => { next.set(Calendar.HOUR_OF_DAY, 0) - do { + + next.add(Calendar.DAY_OF_MONTH, 1) + while (next.get(Calendar.DAY_OF_WEEK) != weekday) { next.add(Calendar.DAY_OF_MONTH, 1) - } while (next.get(Calendar.DAY_OF_WEEK) != weekday) + } + Some(next) } } diff --git a/util-logging/src/main/scala/com/twitter/logging/Handler.scala b/util-logging/src/main/scala/com/twitter/logging/Handler.scala index 5a7f6935ee..855672ea3a 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Handler.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Handler.scala @@ -150,5 +150,5 @@ class ConsoleHandler(formatter: Formatter = new Formatter(), level: Option[Level def close(): Unit = {} - def flush(): Unit = Console.flush + def flush(): Unit = Console.flush() } diff --git a/util-logging/src/main/scala/com/twitter/logging/Logger.scala b/util-logging/src/main/scala/com/twitter/logging/Logger.scala index 349501c2bb..64340ebfe4 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Logger.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Logger.scala @@ -66,7 +66,7 @@ class Logger protected (val name: String, private val wrapped: javalog.Logger) { */ @varargs final def log(level: Level, thrown: Throwable, message: String, items: Any*): Unit = { - val myLevel = getLevel + val myLevel = getLevel() if ((myLevel eq null) || (level.intValue >= myLevel.intValue)) { val record = @@ -136,7 +136,7 @@ class Logger protected (val name: String, private val wrapped: javalog.Logger) { * and attach an exception and stack trace. */ def logLazy(level: Level, thrown: Throwable, message: => AnyRef): Unit = { - val myLevel = getLevel + val myLevel = getLevel() if ((myLevel eq null) || (level.intValue >= myLevel.intValue)) { val record = new LazyLogRecord(level, message) record.setLoggerName(wrapped.getName) diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index 74bbe2af00..dac13d0161 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -17,13 +17,18 @@ package com.twitter.logging import java.io._ -import java.util.{Calendar, Date, UUID, logging => javalog} +import java.util.Calendar +import java.util.Date +import java.util.UUID +import java.util.{logging => javalog} import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import com.twitter.util.Time import java.nio.file.attribute.BasicFileAttributes -import java.nio.file.{Files, Path, Paths} +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths import java.util.function.BiPredicate import org.scalatest.wordspec.AnyWordSpec @@ -353,7 +358,7 @@ class FileHandlerTest extends AnyWordSpec with TempFolder { } val files = listLogFiles(folderName + "/LogFileDir") - files.foreach { f: File => + files.foreach { (f: File) => val len = f.length().bytes if (len > fileSizeInMegaBytes.megabytes) { fail("Failed to roll over the log file") diff --git a/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala b/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala index 3d16a8e372..a9ac801d9f 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LogRecordTest.scala @@ -6,7 +6,7 @@ import org.scalatest.funsuite.AnyFunSuite class LogRecordTest extends AnyFunSuite { test("LogRecord should getMethod properly") { Logger.withLoggers(Nil) { - new LogRecordTestHelper({ r: JRecord => r.getSourceMethodName() }) { + new LogRecordTestHelper({ (r: JRecord) => r.getSourceMethodName() }) { def makingLogRecord() = { logger.log(Level.INFO, "OK") assert(handler.get == "makingLogRecord") @@ -34,7 +34,7 @@ abstract class LogRecordTestHelper(formats: JRecord => String) { logger.addHandler(handler) } -class Foo extends LogRecordTestHelper({ r: JRecord => r.getSourceClassName() }) { +class Foo extends LogRecordTestHelper({ (r: JRecord) => r.getSourceClassName }) { def makingLogRecord(): Unit = { logger.log(Level.INFO, "OK") } diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index 975c3f9db4..17ee59e186 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -19,7 +19,11 @@ package com.twitter.logging import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import java.net.InetSocketAddress -import java.util.concurrent.{Callable, CountDownLatch, Executors, Future, TimeUnit} +import java.util.concurrent.Callable +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit import java.util.{logging => javalog} import org.scalatest.BeforeAndAfter import scala.collection.mutable @@ -68,7 +72,7 @@ class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { logger.addHandler(traceHandler) } - def logLines(): Seq[String] = traceHandler.get.split("\n") + def logLines(): Seq[String] = traceHandler.get.split("\n").toIndexedSeq /** * Verify that the logger set up with `traceLogger` has received a log line with the given @@ -124,7 +128,7 @@ class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { "asdf" + executed + " hi there" } - logger.debugLazy(function) + logger.debugLazy(function()) assert(!executed) } @@ -136,7 +140,7 @@ class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { executed = true "asdf" + executed + " hi there" } - logger.debugLazy(function) + logger.debugLazy(function()) assert(executed) } @@ -263,11 +267,11 @@ class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { val otherFactories = List(LoggerFactory(node = "", level = Some(Level.INFO))) Logger.configure(initialFactories) - assert(Logger.get("").getLevel == Level.DEBUG) + assert(Logger.get("").getLevel() == Level.DEBUG) Logger.withLoggers(otherFactories) { assert(Logger.get("").getLevel() == Level.INFO) } - assert(Logger.get("").getLevel == Level.DEBUG) + assert(Logger.get("").getLevel() == Level.DEBUG) } "configure logging" should { @@ -294,13 +298,13 @@ class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { ) :: Nil ).apply() - assert(log.getLevel == Level.DEBUG) + assert(log.getLevel() == Level.DEBUG) assert(log.getHandlers().length == 1) val handler = log.getHandlers()(0).asInstanceOf[FileHandler] val fileName = folderName + separator + "test.log" assert(handler.filename == fileName) assert(handler.append == false) - assert(handler.getLevel == Level.INFO) + assert(handler.getLevel() == Level.INFO) val formatter = handler.formatter assert( formatter.formatPrefix(javalog.Level.WARNING, "10:55", "hello") == "WARNING 10:55 hello" @@ -325,7 +329,7 @@ class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { ) :: Nil ).apply() - assert(log.getHandlers.length == 1) + assert(log.getHandlers().length == 1) val h = log.getHandlers()(0).asInstanceOf[SyslogHandler] assert(h.dest.asInstanceOf[InetSocketAddress].getHostName == "localhost") assert(h.dest.asInstanceOf[InetSocketAddress].getPort == 212) @@ -365,9 +369,9 @@ class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { ) :: Nil Logger.configure(factories) - assert(Logger.get("").getLevel == Level.INFO) - assert(Logger.get("w3c").getLevel == Level.OFF) - assert(Logger.get("bad_jobs").getLevel == Level.INFO) + assert(Logger.get("").getLevel() == Level.INFO) + assert(Logger.get("w3c").getLevel() == Level.OFF) + assert(Logger.get("bad_jobs").getLevel() == Level.INFO) try { Logger.get("").getHandlers()(0).asInstanceOf[ThrottledHandler] } catch { diff --git a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala index b4c84882b1..06ca8e2781 100644 --- a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala @@ -16,7 +16,8 @@ package com.twitter.logging -import java.net.{DatagramPacket, DatagramSocket} +import java.net.DatagramPacket +import java.net.DatagramSocket import java.util.{logging => javalog} import org.scalatest.wordspec.AnyWordSpec @@ -45,7 +46,7 @@ class SyslogHandlerTest extends AnyWordSpec { syslog.publish(record1) syslog.publish(record2) - SyslogFuture.sync + SyslogFuture.sync() val p = new DatagramPacket(new Array[Byte](1024), 1024) serverSocket.receive(p) assert( @@ -78,7 +79,7 @@ class SyslogHandlerTest extends AnyWordSpec { ).apply() syslog.publish(record1) - SyslogFuture.sync + SyslogFuture.sync() val p = new DatagramPacket(new Array[Byte](1024), 1024) serverSocket.receive(p) assert( @@ -104,7 +105,7 @@ class SyslogHandlerTest extends AnyWordSpec { ).apply() syslog.publish(record1) - SyslogFuture.sync + SyslogFuture.sync() val p = new DatagramPacket(new Array[Byte](1024), 1024) serverSocket.receive(p) assert( From 2e6b8acf30ea70e7fc50f03741b839ba7502a7d7 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Tue, 19 Oct 2021 23:28:23 +0000 Subject: [PATCH 576/796] AppTest: bump graceful shutdown timout Problem Util actions is flakey because of AppTest. Solution / Result: Let's bump the graceful shutdown period before marking the test as flakey JIRA Issues: CSL-11274 Differential Revision: https://phabricator.twitter.biz/D765916 --- util-app/src/test/scala/com/twitter/app/AppTest.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 98e61256e6..51c10b652b 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -1,7 +1,8 @@ package com.twitter.app import com.twitter.app.lifecycle.Event._ -import com.twitter.app.lifecycle.{Event, Observer} +import com.twitter.app.lifecycle.Event +import com.twitter.app.lifecycle.Observer import com.twitter.app.LoadService.Binding import com.twitter.conversions.DurationOps._ import com.twitter.util._ @@ -53,7 +54,7 @@ class WeNeverCloseButWeDoNotCare extends WeNeverClose { } trait ErrorOnExitApp extends App { - override val defaultCloseGracePeriod: Duration = 2.seconds + override val defaultCloseGracePeriod: Duration = 5.seconds override def exitOnError(throwable: Throwable): Unit = { throw throwable From 266b69bd1e23e1fe3bb2b73aa73168ee5f11c8fc Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 20 Oct 2021 20:55:56 +0000 Subject: [PATCH 577/796] {finagle-core,util-stats}: Move LoadedStatsReceiver to util-stats from finagle-core Problem Putting LoadedStatsReceiver in finagle-core means that some things that finagle-core depends on can't use LoadedStatsReceiver. Solution Let's push it down to util-stats, the only additional dependency we need is util-stats => util-app. JIRA Issues: CSL-11393 Differential Revision: https://phabricator.twitter.biz/D763497 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- .../scala/com/twitter/finagle/stats/BUILD | 1 + .../finagle/stats/LoadedStatsReceiver.scala | 30 +++++++++++++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d50d1474e9..3537298493 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,8 @@ New Features java.time.OffsetDateTime. `toInstant`, `toZonedDateTime`, and `toOffsetDateTime` also preserve nanosecond resolution. ``PHAB_ID=D757636`` +* util-stats: Moved `c.t.finagle.stats.LoadedStatsReceiver` and `c.t.finagle.stats.DefaultStatsReceiver` + from the finagle-core module to util-stats. ``PHAB_ID=D763497`` 21.9.0 ------ diff --git a/build.sbt b/build.sbt index 2c2183b66f..e66e4e372b 100644 --- a/build.sbt +++ b/build.sbt @@ -661,7 +661,7 @@ lazy val utilStats = Project( Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.3" % "test") } } - ).dependsOn(utilCore, utilLint) + ).dependsOn(utilApp, utilCore, utilLint) lazy val utilTest = Project( id = "util-test", diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD index e3896b3b00..a57ff9d56b 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD @@ -16,6 +16,7 @@ scala_library( "3rdparty/jvm/com/fasterxml/jackson/module:jackson-module-scala", "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/com/google/code/findbugs:jsr305", + "util/util-app/src/main/scala", "util/util-core:util-core-util", "util/util-lint/src/main/scala/com/twitter/util/lint", ], diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala new file mode 100644 index 0000000000..89fcf746c6 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala @@ -0,0 +1,30 @@ +package com.twitter.finagle.stats + +import com.twitter.app.LoadService + +/** + * A [[com.twitter.finagle.stats.StatsReceiver]] that loads + * all service-loadable receivers and broadcasts stats to them. + */ +object LoadedStatsReceiver extends StatsReceiverProxy { + + /** + * Mutating this value at runtime after it has been initialized should be done + * with great care. If metrics have been created using the prior + * [[StatsReceiver]], updates to those metrics may not be reflected in the + * [[StatsReceiver]] that replaces it. In addition, histograms created with + * the prior [[StatsReceiver]] will not be available. + */ + @volatile var self: StatsReceiver = BroadcastStatsReceiver(LoadService[StatsReceiver]()) +} + +/** + * A "default" StatsReceiver loaded by the + * [[com.twitter.finagle.util.LoadService]] mechanism. + */ +object DefaultStatsReceiver extends StatsReceiverProxy { + def self: StatsReceiver = LoadedStatsReceiver + override def repr: DefaultStatsReceiver.type = this + + def get: StatsReceiver = this +} From 21ae47266f954f5d0ab5a9059b9e8e0ec5fac0dd Mon Sep 17 00:00:00 2001 From: Flavio Brasil Date: Thu, 21 Oct 2021 21:48:52 +0000 Subject: [PATCH 578/796] [util-core][fiber-scheduler] remove async semaphore mechanism **Problem** During the development of the Fiber Scheduler, an async semaphore was used as an attempt to keep the concurrency behavior of tasks coming from redirected `FuturePool`s. The idea is that the async semaphore would limit the concurrency of tasks similarly to how the number of threads in a `FuturePool` limits the number of concurrent tasks. This approach is incorrect because a `FuturePool` limits synchronous concurrency and the async semaphore mechanism limits asynchronous concurrency, which has completely different behavior. **Solution** The async semaphore got introduced with potential future issues in mind and hasn't been proved necessary yet. Given that and the fact that a creating a mechanism to limit synchronous concurrency would require a more delicate change, this phab removes the concurrency limitation mechanism. Differential Revision: https://phabricator.twitter.biz/D762557 --- .../twitter/concurrent/ForkingScheduler.scala | 14 ++----------- .../scala/com/twitter/util/FuturePool.scala | 12 +++-------- .../com/twitter/util/FuturePoolTest.scala | 20 +++++++------------ 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala index 10027f24b4..66e945a45f 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala @@ -1,7 +1,8 @@ package com.twitter.concurrent import com.twitter.util.Future -import java.util.concurrent.{Executor, ExecutorService} +import java.util.concurrent.Executor +import java.util.concurrent.ExecutorService /** * A scheduler that provides methods to fork the execution of @@ -32,17 +33,6 @@ trait ForkingScheduler extends Scheduler { */ def fork[T](executor: Executor)(f: => Future[T]): Future[T] - /** - * Creates a wrapper of this scheduler with a fixed max - * concurrency. The original forking scheduler remains - * unchanged and is used as the underlying implementation - * of the returned scheduler. - * - * @param v the max concurrency - * @return the wrapper with a fixed max concurrency - */ - def withMaxConcurrency(v: Int): ForkingScheduler - /** * Creates an `ExecutorService` wrapper for this forking * scheduler. The original forking scheduler remains diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index 72901a10f7..dd53b758c8 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -1,6 +1,7 @@ package com.twitter.util -import com.twitter.concurrent.{ForkingScheduler, NamedPoolThreadFactory} +import com.twitter.concurrent.ForkingScheduler +import com.twitter.concurrent.NamedPoolThreadFactory import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.{Future => _, _} import scala.runtime.NonLocalReturnControl @@ -138,14 +139,7 @@ class ExecutorServiceFuturePool protected[this] ( private[this] lazy val forkingScheduler: ForkingScheduler = { ForkingScheduler() .filter(_.redirectFuturePools()) - .map { s => - executor match { - case e: ThreadPoolExecutor => - s.withMaxConcurrency(e.getMaximumPoolSize) - case _ => - s - } - }.getOrElse(null) + .getOrElse(null) } def apply[T](f: => T): Future[T] = { diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index 89fa97ce47..49ff11149a 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -1,10 +1,13 @@ package com.twitter.util -import com.twitter.concurrent.{Scheduler, ForkingScheduler} +import com.twitter.concurrent.Scheduler +import com.twitter.concurrent.ForkingScheduler import com.twitter.conversions.DurationOps._ import java.util.concurrent.{Future => JFuture, _} import org.scalatest.concurrent.Eventually -import org.scalatest.time.{Millis, Seconds, Span} +import org.scalatest.time.Millis +import org.scalatest.time.Seconds +import org.scalatest.time.Span import scala.runtime.NonLocalReturnControl import scala.util.control.NonFatal import org.scalatest.funsuite.AnyFunSuite @@ -235,7 +238,6 @@ class FuturePoolTest extends AnyFunSuite with Eventually { } class TestForkingScheduler(redirect: Boolean) extends ForkingScheduler { - var concurrency = Option.empty[Int] var forked = Option.empty[Future[_]] override def fork[T](f: => Future[T]) = { val r = f @@ -245,10 +247,6 @@ class FuturePoolTest extends AnyFunSuite with Eventually { override def redirectFuturePools(): Boolean = redirect override def blocking[T](f: => T)(implicit perm: Awaitable.CanAwait): T = f override def flush(): Unit = {} - override def withMaxConcurrency(v: Int): ForkingScheduler = { - concurrency = Some(v) - this - } override def submit(r: Runnable): Unit = r.run() @@ -258,10 +256,9 @@ class FuturePoolTest extends AnyFunSuite with Eventually { } test("FuturePool should be redirected to the forking scheduler if specified by the scheduler") { - val concurrency = 1 val scheduler = new TestForkingScheduler(redirect = true) val prevScheduler = Scheduler() - val executor = Executors.newFixedThreadPool(concurrency) + val executor = Executors.newFixedThreadPool(1) Scheduler.setUnsafe(scheduler) try { val pool = FuturePool(executor) @@ -273,7 +270,6 @@ class FuturePoolTest extends AnyFunSuite with Eventually { } assert(Await.result(result, 1.second) == 1) assert(Await.result(scheduler.forked.get, 1.second) == 1) - assert(scheduler.concurrency == Some(concurrency)) assert(executed) } finally { Scheduler.setUnsafe(prevScheduler) @@ -283,9 +279,8 @@ class FuturePoolTest extends AnyFunSuite with Eventually { test( "FuturePool should not be redirected to the forking scheduler if specified by the scheduler") { - val concurrency = 1 val scheduler = new TestForkingScheduler(redirect = false) - val executor = Executors.newFixedThreadPool(concurrency) + val executor = Executors.newFixedThreadPool(1) val prevScheduler = Scheduler() Scheduler.setUnsafe(scheduler) try { @@ -298,7 +293,6 @@ class FuturePoolTest extends AnyFunSuite with Eventually { } assert(Await.result(result, 1.second) == 1) assert(scheduler.forked.isEmpty) - assert(scheduler.concurrency.isEmpty) assert(executed) } finally { Scheduler.setUnsafe(prevScheduler) From b18c1c48fce1d5e265d64749361ad1105e98f42b Mon Sep 17 00:00:00 2001 From: Flavio Brasil Date: Fri, 22 Oct 2021 02:39:33 +0000 Subject: [PATCH 579/796] [util/finagle-core][fiber-scheduler] don't reject tasks redirected from `FuturePool`s **Problem** The fiber scheduler can reject tasks that are redirected from `FuturePool`s. The idea behind that was to avoid having to support unbounded queuing but that's making the adoption of the new scheduler more difficult. **Solution** Introduce a new `tryFork` method that has a behavior similar to the previous `fork` implementation and allows the caller to detect if the task got rejected without having to handle an exception. The `fork` method is changed to never reject requests even if the scheduler is overloaded. Typically, `tryFork` is called from Finagle's default filter chain, which can reject tasks, and `fork` is called from redirected `FuturePool`s. Users can also call those methods directly but we don't have cases of that yet. **Notes** - With this change, it's safer to redirect `FuturePool`s so I changed the default fiber scheduler configuration to set `redirectFuturePools` to `true` - Ideally, we should revisit this in the future so unbounded queuing doesn't need to be supported but that will require a larger effort to adapt applications Differential Revision: https://phabricator.twitter.biz/D760928 --- .../com/twitter/concurrent/ForkingScheduler.scala | 15 ++++++++++++++- .../scala/com/twitter/util/FuturePoolTest.scala | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala index 66e945a45f..f0625e807a 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala @@ -11,7 +11,8 @@ import java.util.concurrent.ExecutorService trait ForkingScheduler extends Scheduler { /** - * Forks the execution of a `Future` computation. + * Forks the execution of a `Future` computation even if + * the scheduler is overloaded. * * @param f the Future computation to be forked * @tparam T the type of the `Future` computation @@ -20,6 +21,18 @@ trait ForkingScheduler extends Scheduler { */ def fork[T](f: => Future[T]): Future[T] + /** + * Tries to fork the execution of a `Future` computation and + * returns empty in case the scheduler is overloaded. + * + * @param f the Future computation to be forked + * @tparam T the type of the `Future` computation + * @return A `Future` with `None` if the scheduler is overloaded + * or `Some[T]` if the task is successfully forked and + * executed. + */ + def tryFork[T](f: => Future[T]): Future[Option[T]] + /** * Forks the execution of a `Future` computation using the * provided executor. diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index 49ff11149a..382d8231d2 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -244,6 +244,7 @@ class FuturePoolTest extends AnyFunSuite with Eventually { forked = Some(r) r } + override def tryFork[T](f: => Future[T]): Future[Option[T]] = ??? override def redirectFuturePools(): Boolean = redirect override def blocking[T](f: => T)(implicit perm: Awaitable.CanAwait): T = f override def flush(): Unit = {} From 79183eb0469dd06c676afb395db49aae389a9b75 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Wed, 27 Oct 2021 17:28:34 +0000 Subject: [PATCH 580/796] Twitter-oss: Prepare OSS libraries for release 21.10.0 Problem We want to release the next versions of our Twitter OSS libraries - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11391 Differential Revision: https://phabricator.twitter.biz/D770597 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3537298493..6deec5b26b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.10.0 +------- + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 9948ead043..4bba4d6b1d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -40,7 +40,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.9.0" +val utilCore = "com.twitter" %% "util-core" % "21.10.0" ``` # Units diff --git a/build.sbt b/build.sbt index e66e4e372b..a12815844b 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.10.0-SNAPSHOT" +val releaseVersion = "21.10.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.4" From 12cd4ca1415f4c9545eedde5ddde23683d6b904d Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Wed, 27 Oct 2021 20:03:33 +0000 Subject: [PATCH 581/796] Twitter-oss: update OSS libraries post-release to 21.11.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.11.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11391 Differential Revision: https://phabricator.twitter.biz/D771421 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4bba4d6b1d..62fc500208 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index a12815844b..7ca79935c9 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.10.0" +val releaseVersion = "21.11.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.4" From c1d62105bfe7b951b7c65d95fafc5f08e0d16bdb Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Fri, 29 Oct 2021 20:00:03 +0000 Subject: [PATCH 582/796] finagle|util-cache: caffeine cache upgrade to 2.9.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem/Solution We want to keep our dependencies up to date for the ben-manes Caffeine library. Let’s update to version 2.9.2. JIRA Issues: CSL-11430 Differential Revision: https://phabricator.twitter.biz/D771893 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6deec5b26b..488e0cd9e4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-cache: Update Caffeine cache library to version 2.9.2 ``PHAB_ID=D771893`` + 21.10.0 ------- diff --git a/build.sbt b/build.sbt index 7ca79935c9..a2b7ede2aa 100644 --- a/build.sbt +++ b/build.sbt @@ -21,7 +21,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll ( ) val guavaLib = "com.google.guava" % "guava" % "25.1-jre" -val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.9.1" +val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.9.2" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.15.4" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion From ebe60581c9b032d96ca8f628a54ccd82b9cb7531 Mon Sep 17 00:00:00 2001 From: Michael Leinartas Date: Wed, 3 Nov 2021 16:54:25 +0000 Subject: [PATCH 583/796] util-jvm: Ensure relevant JVM stats use counterish-gauge metadata ## Problem Counterish Gauges (Gauge stat types backed by an underlying counter) were introduced recently along with an application of the new stat type to the most commonly used metrics in util-jvm. Several other metrics exposed by util-jvm are also backed by counters but do not reflect the underlying type correctly, leaving the metadata associated with the stats exported by util-jvm unreliable. ## Solution Modify the following counter-backed metrics to use CounterishGauge: - [[ https://docs.oracle.com/javase/8/docs/api/java/lang/management/CompilationMXBean.html#getTotalCompilationTime-- | compilation/time_msec]] - [[ https://docs.oracle.com/javase/7/docs/api/java/lang/management/ClassLoadingMXBean.html#getTotalLoadedClassCount() | classes/total_loaded ]] - [[ https://docs.oracle.com/javase/7/docs/api/java/lang/management/ClassLoadingMXBean.html#getUnloadedClassCount() | classes/total_unloaded ]] - [[ https://twitter.github.io/util/docs/com/twitter/jvm/Safepoint.html | safepoint/sync_time_millis ]] - [[ https://twitter.github.io/util/docs/com/twitter/jvm/Safepoint.html | safepoint/total_time_millis ]] - [[ https://twitter.github.io/util/docs/com/twitter/jvm/Safepoint.html | safepoint/count ]] - gc/cycles and gc/msec (the unscoped variety, matching its collector-scoped equivalent) ##Notes Also stuck in here is a new stat, jvm/spec_version, a gauge which returns the JVM Specification version, a reliable cross-implementation proxy for the major version of the running JVM. This is useful primarily for tracking migration of major JVM versions across many services JIRA Issues: IOP-1269 Differential Revision: https://phabricator.twitter.biz/D710019 --- CHANGELOG.rst | 3 ++ .../main/scala/com/twitter/jvm/JvmStats.scala | 34 ++++++++++++------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 488e0cd9e4..605692ff34 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,9 @@ New Features * util-stats: Moved `c.t.finagle.stats.LoadedStatsReceiver` and `c.t.finagle.stats.DefaultStatsReceiver` from the finagle-core module to util-stats. ``PHAB_ID=D763497`` +* util-jvm: Update exported metric schemas to properly reflect their underlying counter types + ``PHAB_ID=D710019`` + 21.9.0 ------ diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index d06477c068..d017723aa0 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -45,6 +45,7 @@ object JvmStats { val uptime = stats.addGauge("uptime") { runtime.getUptime().toFloat } gauges.add(uptime) gauges.add(stats.addGauge("start_time") { runtime.getStartTime().toFloat }) + gauges.add(stats.addGauge("spec_version") { runtime.getSpecVersion.toFloat }) val os = ManagementFactory.getOperatingSystemMXBean() gauges.add(stats.addGauge("num_cpus") { os.getAvailableProcessors().toFloat }) @@ -67,19 +68,22 @@ object JvmStats { case null => case compilation => val compilationStats = stats.scope("compilation") - gauges.add(compilationStats.addGauge("time_msec") { - compilation.getTotalCompilationTime().toFloat - }) + gauges.add( + compilationStats.metricBuilder(GaugeType).withCounterishGauge.gauge("time_msec") { + compilation.getTotalCompilationTime().toFloat + }) } val classes = ManagementFactory.getClassLoadingMXBean() val classLoadingStats = stats.scope("classes") - gauges.add(classLoadingStats.addGauge("total_loaded") { - classes.getTotalLoadedClassCount().toFloat - }) - gauges.add(classLoadingStats.addGauge("total_unloaded") { - classes.getUnloadedClassCount().toFloat - }) + gauges.add( + classLoadingStats.metricBuilder(GaugeType).withCounterishGauge.gauge("total_loaded") { + classes.getTotalLoadedClassCount().toFloat + }) + gauges.add( + classLoadingStats.metricBuilder(GaugeType).withCounterishGauge.gauge("total_unloaded") { + classes.getUnloadedClassCount().toFloat + }) gauges.add( classLoadingStats.addGauge("current_loaded") { classes.getLoadedClassCount().toFloat } ) @@ -117,9 +121,15 @@ object JvmStats { } val spStats = stats.scope("safepoint") - gauges.add(spStats.addGauge("sync_time_millis") { jvm.safepoint.syncTimeMillis.toFloat }) - gauges.add(spStats.addGauge("total_time_millis") { jvm.safepoint.totalTimeMillis.toFloat }) - gauges.add(spStats.addGauge("count") { jvm.safepoint.count.toFloat }) + gauges.add(spStats.metricBuilder(GaugeType).withCounterishGauge.gauge("sync_time_millis") { + jvm.safepoint.syncTimeMillis.toFloat + }) + gauges.add(spStats.metricBuilder(GaugeType).withCounterishGauge.gauge("total_time_millis") { + jvm.safepoint.totalTimeMillis.toFloat + }) + gauges.add(spStats.metricBuilder(GaugeType).withCounterishGauge.gauge("count") { + jvm.safepoint.count.toFloat + }) ManagementFactory.getPlatformMXBeans(classOf[BufferPoolMXBean]) match { case null => From 0cec34c6484b2ab222032cfccfc8008779ec1f47 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Mon, 8 Nov 2021 15:42:51 +0000 Subject: [PATCH 584/796] 3rdparty: jackson upgrade to 2.13.0 Problem We are on jackson 2.11.x, but we want to upgrade to a new version. Solution Upgrade to 2.13.0 Differential Revision: https://phabricator.twitter.biz/D744627 --- CHANGELOG.rst | 7 +++ build.sbt | 2 +- .../util/jackson/ScalaObjectMapperTest.scala | 59 +++++++++---------- 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 605692ff34..1663fc6a33 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,11 +7,18 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-jackson: The error message when failing to deserialize a character now correctly prints the non-character string. ``PHAB_ID=D744627`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ * util-cache: Update Caffeine cache library to version 2.9.2 ``PHAB_ID=D771893`` +* util: Bump version of Jackson to 2.13.0. ``PHAB_ID=D744627`` + 21.10.0 ------- diff --git a/build.sbt b/build.sbt index a2b7ede2aa..5f65522bad 100644 --- a/build.sbt +++ b/build.sbt @@ -7,7 +7,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "21.11.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.11.4" +val jacksonVersion = "2.13.0" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala index fe99859470..0855c4daf5 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala @@ -1,44 +1,39 @@ package com.twitter.util.jackson -import com.fasterxml.jackson.core.{JsonFactory, JsonParser} +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.databind.module.SimpleModule -import com.fasterxml.jackson.databind.node.{IntNode, TreeTraversingParser} -import com.fasterxml.jackson.databind.{ - DeserializationContext, - DeserializationFeature, - JsonDeserializer, - JsonMappingException, - JsonNode, - MapperFeature, - PropertyNamingStrategy, - ObjectMapper => JacksonObjectMapper -} +import com.fasterxml.jackson.databind.node.IntNode +import com.fasterxml.jackson.databind.node.TreeTraversingParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonMappingException +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.MapperFeature +import com.fasterxml.jackson.databind.PropertyNamingStrategy +import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper} import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.scala.{ - DefaultScalaModule, - ScalaObjectMapper => JacksonScalaObjectMapper -} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.{ScalaObjectMapper => JacksonScalaObjectMapper} import com.twitter.io.Buf import com.twitter.util.Duration -import com.twitter.util.jackson.Obj.{ - NestedCaseClassInObject, - NestedCaseClassInObjectWithNestedCaseClassInObjectParam -} +import com.twitter.util.jackson.Obj.NestedCaseClassInObject +import com.twitter.util.jackson.Obj.NestedCaseClassInObjectWithNestedCaseClassInObjectParam import com.twitter.util.jackson.TypeAndCompanion.NestedCaseClassInCompanion -import com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException.{ - Unspecified, - ValidationError -} +import com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException.Unspecified +import com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException.ValidationError import com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException -import com.twitter.util.jackson.internal.{ - SimplePersonInPackageObject, - SimplePersonInPackageObjectWithoutConstructorParams -} -import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import com.twitter.util.jackson.internal.SimplePersonInPackageObject +import com.twitter.util.jackson.internal.SimplePersonInPackageObjectWithoutConstructorParams +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream import java.time.format.DateTimeFormatter -import java.time.{LocalDateTime, ZoneId} +import java.time.LocalDateTime +import java.time.ZoneId import java.util.concurrent.TimeUnit -import java.util.{TimeZone, UUID} +import java.util.TimeZone +import java.util.UUID import org.junit.Assert.assertNotNull import org.junit.runner.RunWith import org.scalatest.BeforeAndAfterAll @@ -828,7 +823,7 @@ class ScalaObjectMapperTest |} """.stripMargin, withErrors = Seq( - "c: '' is not a valid Character" + "c: '-1' is not a valid Character" ) ) } From 1322d86bf351bdbcbde3bb59db7690e331758b85 Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Mon, 8 Nov 2021 17:17:01 +0000 Subject: [PATCH 585/796] util-core: Future.toCompletableFuture succeeds on failed futures Problem Calling `Future.toCompletableFuture` on a `Future` already completed with an exception would throw the exception to the caller, instead of successfully creating an exceptionally-completed `CompletableFuture` and returning it. Solution Added a code path that correctly creates a `CompletableFuture` from an already-exceptionally-completed `Future`. JIRA Issues: CSL-11338 Differential Revision: https://phabricator.twitter.biz/D776469 --- .../main/scala/com/twitter/util/Future.scala | 42 +++++++++++-------- .../scala/com/twitter/util/FutureTest.scala | 24 +++++++++-- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 6604b9d9da..82e02ae92b 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1,8 +1,12 @@ package com.twitter.util -import com.twitter.concurrent.{Offer, Tx} -import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} -import java.util.concurrent.{CancellationException, CompletableFuture, Future => JavaFuture} +import com.twitter.concurrent.Offer +import com.twitter.concurrent.Tx +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import java.util.concurrent.CancellationException +import java.util.concurrent.CompletableFuture +import java.util.concurrent.{Future => JavaFuture} import java.util.function.BiConsumer import java.util.{List => JList} import scala.collection.compat.immutable.ArraySeq @@ -2164,23 +2168,27 @@ abstract class Future[+A] extends Awaitable[A] { self => * https://download.oracle.com/javase/6/docs/api/java/util/concurrent/Future.html#cancel(boolean) */ def toCompletableFuture[B >: A]: CompletableFuture[B] = { - if (self.isDefined) { - return CompletableFuture.completedFuture(Await.result(self)) - } - - val f = new CompletableFuture[B]() - this.respond { - case Return(result) => f.complete(result) - case Throw(t) => f.completeExceptionally(t) - } + val f = new CompletableFuture[B]() with (Try[A] => Unit) with BiConsumer[B, Throwable] { + // Complete the Java future from a Try + def apply(t: Try[A]): Unit = + t match { + case Return(result) => complete(result) + case Throw(t) => completeExceptionally(t) + } - f.whenComplete(new BiConsumer[B, Throwable] { - def accept(result: B, t: Throwable): Unit = { - if (t != null && t.isInstanceOf[CancellationException]) { + // For propagating cancellation + def accept(r: B, t: Throwable): Unit = + if (t.isInstanceOf[CancellationException]) { self.raise(t) } - } - }) + } + + poll match { + case Some(t) => f(t) // complete + case None => + this.respond(f) // propagate completion forward + f.whenComplete(f) // propagate cancellation backwards + } // we have to return the original one because cancellation isn't propagated // by CompletionStages diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index f981eb7722..fce42e6358 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -1,13 +1,19 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import java.util.concurrent.{ConcurrentLinkedQueue, ExecutionException, Future => JFuture} +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.ExecutionException +import java.util.concurrent.{Future => JFuture} import java.util.concurrent.atomic.AtomicInteger import org.mockito.ArgumentMatchers.any -import org.mockito.Mockito.{never, times, verify, when} +import org.mockito.Mockito.never +import org.mockito.Mockito.times +import org.mockito.Mockito.verify +import org.mockito.Mockito.when import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer -import org.scalacheck.{Arbitrary, Gen} +import org.scalacheck.Arbitrary +import org.scalacheck.Gen import org.scalatestplus.mockito.MockitoSugar import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.jdk.CollectionConverters._ @@ -879,7 +885,7 @@ class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenProp } } - "java future should throw an exception" in { + "java future should throw an exception when failed later" in { val f = new Promise[Int] val jf = fn(f) val e = new RuntimeException() @@ -889,6 +895,16 @@ class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenProp assert(cause == e) } + "java future should throw an exception when failed early" in { + val f = new Promise[Int] + val e = new RuntimeException() + f.setException(e) + val jf = fn(f) + val actual = intercept[ExecutionException] { jf.get() } + val cause = intercept[RuntimeException] { throw actual.getCause } + assert(cause == e) + } + "interrupt Future when cancelled" in { val f = new HandledPromise[Int] val jf = fn(f) From 9c6e01f89d0bd4db57db535000d268bff09de19e Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Tue, 9 Nov 2021 19:24:30 +0000 Subject: [PATCH 586/796] Revert "3rdparty: jackson upgrade to 2.13.0" Problem 2.13.0 introduced a regression, where they were double-serializing. We're worried that this will lead to a substantial performance regression. Github ticket: https://github.com/FasterXML/jackson-databind/issues/3308 Solution This reverts commit e27353b784543196b9f5fd572110e4b4fe04f4c4. JIRA Issues: CSL-11459 Differential Revision: https://phabricator.twitter.biz/D780287 --- CHANGELOG.rst | 7 ------- build.sbt | 2 +- .../com/twitter/util/jackson/ScalaObjectMapperTest.scala | 2 +- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1663fc6a33..605692ff34 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,18 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -Breaking API Changes -~~~~~~~~~~~~~~~~~~~~ - -* util-jackson: The error message when failing to deserialize a character now correctly prints the non-character string. ``PHAB_ID=D744627`` - Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ * util-cache: Update Caffeine cache library to version 2.9.2 ``PHAB_ID=D771893`` -* util: Bump version of Jackson to 2.13.0. ``PHAB_ID=D744627`` - 21.10.0 ------- diff --git a/build.sbt b/build.sbt index 5f65522bad..a2b7ede2aa 100644 --- a/build.sbt +++ b/build.sbt @@ -7,7 +7,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "21.11.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.13.0" +val jacksonVersion = "2.11.4" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala index 0855c4daf5..5a0579a515 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala @@ -823,7 +823,7 @@ class ScalaObjectMapperTest |} """.stripMargin, withErrors = Seq( - "c: '-1' is not a valid Character" + "c: '' is not a valid Character" ) ) } From df5c4d158d0499f3969d75c322ee3313be4641d4 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Wed, 10 Nov 2021 22:01:13 +0000 Subject: [PATCH 587/796] Update PROJECT files Problem/Solution Update PROJECT files. Differential Revision: https://phabricator.twitter.biz/D781463 --- util-app/PROJECT | 3 ++- util-inject/PROJECT | 1 - util-jackson-annotations/PROJECT | 3 ++- util-jackson/PROJECT | 3 ++- util-logging/PROJECT | 3 ++- util-mock/PROJECT | 1 - util-slf4j-api/PROJECT | 3 ++- util-slf4j-jul-bridge/PROJECT | 3 ++- util-validator/PROJECT | 3 ++- 9 files changed, 14 insertions(+), 9 deletions(-) diff --git a/util-app/PROJECT b/util-app/PROJECT index 31678e4ba3..98a843682c 100644 --- a/util-app/PROJECT +++ b/util-app/PROJECT @@ -2,9 +2,10 @@ owners: - banderson - - ccoco - ibennett - koliver - mnakamura - roanta - vkostyukov +watchers: + - csl-team@twitter.com diff --git a/util-inject/PROJECT b/util-inject/PROJECT index 5010c7f26a..30197d422a 100644 --- a/util-inject/PROJECT +++ b/util-inject/PROJECT @@ -1,6 +1,5 @@ owners: - banderson - - ccoco - ibennett - koliver - mnakamura diff --git a/util-jackson-annotations/PROJECT b/util-jackson-annotations/PROJECT index 2a8a1534c3..66d5ecc6bf 100644 --- a/util-jackson-annotations/PROJECT +++ b/util-jackson-annotations/PROJECT @@ -1,8 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - ccoco - ibennett - jyan - mnakamura - roanta +watchers: + - csl-team@twitter.com diff --git a/util-jackson/PROJECT b/util-jackson/PROJECT index 2a8a1534c3..66d5ecc6bf 100644 --- a/util-jackson/PROJECT +++ b/util-jackson/PROJECT @@ -1,8 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - ccoco - ibennett - jyan - mnakamura - roanta +watchers: + - csl-team@twitter.com diff --git a/util-logging/PROJECT b/util-logging/PROJECT index 24faddeed5..137e3d71c4 100644 --- a/util-logging/PROJECT +++ b/util-logging/PROJECT @@ -2,8 +2,9 @@ owners: - banderson - - ccoco - ibennett - mnakamura - roanta - vkostyukov +watchers: + - csl-team@twitter.com diff --git a/util-mock/PROJECT b/util-mock/PROJECT index 5010c7f26a..30197d422a 100644 --- a/util-mock/PROJECT +++ b/util-mock/PROJECT @@ -1,6 +1,5 @@ owners: - banderson - - ccoco - ibennett - koliver - mnakamura diff --git a/util-slf4j-api/PROJECT b/util-slf4j-api/PROJECT index ec3a2d20d5..223b72d975 100644 --- a/util-slf4j-api/PROJECT +++ b/util-slf4j-api/PROJECT @@ -2,7 +2,8 @@ owners: - banderson - - ccoco - ibennett - mnakamura - roanta +watchers: + - csl-team@twitter.com diff --git a/util-slf4j-jul-bridge/PROJECT b/util-slf4j-jul-bridge/PROJECT index ec3a2d20d5..223b72d975 100644 --- a/util-slf4j-jul-bridge/PROJECT +++ b/util-slf4j-jul-bridge/PROJECT @@ -2,7 +2,8 @@ owners: - banderson - - ccoco - ibennett - mnakamura - roanta +watchers: + - csl-team@twitter.com diff --git a/util-validator/PROJECT b/util-validator/PROJECT index 2a8a1534c3..acd2e0abaa 100644 --- a/util-validator/PROJECT +++ b/util-validator/PROJECT @@ -1,8 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - ccoco - ibennett - jyan - mnakamura - roanta +watchers: + - csl-team@twitter.com \ No newline at end of file From f2758b4213b5bb19e0532cf845fa31ae3407a989 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Thu, 11 Nov 2021 00:11:53 +0000 Subject: [PATCH 588/796] util-stats: Add Stringly Expression constructor Problem It can be troublesome to pass around MetricBuilders or StatsReceiver for users to create expressions. Solution Support creating expression from strings. Result Using string to create expressions could be a bit error-prone, however, the convenience wins the tradeoff comparison here. We added some validation, but it isn't going to guard against all errors since many metrics are created lazily. JIRA Issues: CSL-11283 Differential Revision: https://phabricator.twitter.biz/D777021 --- .../finagle/stats/exp/Expression.scala | 26 ++++++++++++++++-- .../stats/exp/ExpressionSchemaTest.scala | 27 ++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index d7b125995a..38999d0a34 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -1,10 +1,11 @@ package com.twitter.finagle.stats.exp -import com.twitter.finagle.stats.MetricBuilder.HistogramType -import com.twitter.finagle.stats.exp.Expression.HistogramComponent +import com.twitter.finagle.stats.LoadedStatsReceiver import com.twitter.finagle.stats.Metadata import com.twitter.finagle.stats.MetricBuilder +import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.StatsReceiver +import com.twitter.finagle.stats.exp.Expression.HistogramComponent import scala.annotation.varargs private[twitter] object Expression { @@ -25,6 +26,7 @@ private[twitter] object Expression { case MetricExpression(metricBuilder) => Set(metricBuilder.statsReceiver) case HistogramExpression(metricBuilder, _) => Set(metricBuilder.statsReceiver) case ConstantExpression(_) => Set.empty + case StringExpression(_) => Set(LoadedStatsReceiver) case NoExpression => Set.empty } @@ -65,6 +67,20 @@ private[twitter] object Expression { NoExpression } } + + /** + * Represent an expression constructed from metrics in strings, user would need to provide + * role and service name if needed to the [[ExpressionSchema]], otherwise remain unspecified. + * + * @note [[StringExpression]] is for conveniently creating expressions without getting a hold of + * [[StatsReceiver]] or [[MetricBuilder]], if they present or easy to access, users should + * prefer creating expression via Metric instances. + * StringExpressions are not guaranteed to be real metrics, please proceed with proper + * validation and testing. + * + * @param exprs a sequence of metrics in strings + */ + def apply(exprs: Seq[String]): Expression = StringExpression(exprs) } /** @@ -112,6 +128,12 @@ case class HistogramExpression private[exp] ( component: Either[HistogramComponent, Double]) extends Expression +/** + * Represent an expression constructed from metrics in strings + * @see Expression#apply + */ +case class StringExpression private[exp] (exprs: Seq[String]) extends Expression + case object NoExpression extends Expression { @varargs override def func(name: String, rest: Expression*): Expression = NoExpression diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 4f41cad293..102c3c4b18 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -140,14 +140,14 @@ class ExpressionSchemaTest extends AnyFunSuite { test("expressions with the same key - name, labels and namespaces") { new Ctx { - val successRate1 = ExpressionSchema("success_rate", Expression(successMb)) + val successRate1 = ExpressionSchema("success", Expression(successMb)) .withNamespace("section1") .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) .withUnit(Percentage) .withDescription("The success rate of the slow query") .build() - val successRate2 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) + val successRate2 = ExpressionSchema("success", Expression(successMb).divide(sum)) .withNamespace("section1") .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) .withUnit(Percentage) @@ -158,10 +158,29 @@ class ExpressionSchemaTest extends AnyFunSuite { // the second attempt will be a Throw assert(sr.expressions.values.size == 1) assert( - sr.expressions(nameToKey("success_rate", Map(), Seq("section1"))).expr == Expression( - successMb)) + sr.expressions(nameToKey("success", Map(), Seq("section1"))).expr == Expression(successMb)) assert(successRate1.isReturn) assert(successRate2.isThrow) } } + + test("StringExpression nested in FunctionExpression") { + val loadedSr = LoadedStatsReceiver.self + val sr = new InMemoryStatsReceiver + LoadedStatsReceiver.self = sr + val successRate = ExpressionSchema( + "success_rate", + Expression(100).multiply(Expression(Seq("clnt", "aclient", "success")).divide(Expression( + Seq("clnt", "aclient", "success")).plus(Expression(Seq("clnt", "aclient", "failures"))))) + ).withRole(Client).withServiceName("aclient").build() + + assert(sr.expressions.values.size == 1) + assert( + sr.expressions.contains( + ExpressionSchemaKey( + "success_rate", + Map("role" -> "Client", "service_name" -> "aclient"), + Seq()))) + LoadedStatsReceiver.self = loadedSr + } } From 35f262f23aaad1301ccf563210d4af670051ec95 Mon Sep 17 00:00:00 2001 From: Lily He Date: Mon, 15 Nov 2021 17:07:33 +0000 Subject: [PATCH 589/796] util-jackson: Block unsafe polymorphic base type deserialization Problem: Jackson ObjectMapper can suffer from CVEs that exploit the permissive nature of class-name-based polymorphic deserialization. This is a concern if JSON is accepted by untrusted clients and polymorphic handling is enabled. The two ways of enabling polymorphic handling are `enableDefaultTyping()` and annotating properties with `@JsonTypeInfo`. Solution: Util-Jackson's ScalaObjectMapper is not at risk for this type of CVE because default typing is not enabled by default. Out of security precaution, enable BLOCK_UNSAFE_POLYMORPHIC_BASE_TYPES in ScalaObjectMapper. Result: Turning on this mapper feature blocks the use of a set of "unsafe" base types. Even if `enableDefaultTyping` is configured on a custom ScalaObjectMapper, the mapper will be protected from this CVE. JIRA Issues: CSL-11410 Differential Revision: https://phabricator.twitter.biz/D780863 --- CHANGELOG.rst | 4 +++ .../util/jackson/ScalaObjectMapper.scala | 29 +++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 605692ff34..9c2d49b6e2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,10 @@ Runtime Behavior Changes * util-cache: Update Caffeine cache library to version 2.9.2 ``PHAB_ID=D771893`` +* util-jackson: Enable `BLOCK_UNSAFE_POLYMORPHIC_BASE_TYPES` in ScalaObjectMapper to + guard against Remote Code Execution (RCE) security vulnerability. This blocks + polymorphic deserialization from unsafe base types. ``PHAB_ID=D780863`` + 21.10.0 ------- diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala index 7e73ca44d8..6db1814f41 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala @@ -3,21 +3,27 @@ package com.twitter.util.jackson import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.core.json.JsonWriteFeature -import com.fasterxml.jackson.core.util.{DefaultIndenter, DefaultPrettyPrinter} -import com.fasterxml.jackson.core.{JsonFactory, JsonFactoryBuilder, JsonParser, TSFBuilder} +import com.fasterxml.jackson.core.util.DefaultIndenter +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.core.JsonFactoryBuilder +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.core.TSFBuilder import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper, _} -import com.fasterxml.jackson.dataformat.yaml.{YAMLFactory, YAMLFactoryBuilder} +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.dataformat.yaml.YAMLFactoryBuilder import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule -import com.fasterxml.jackson.module.scala.{ - DefaultScalaModule, - ScalaObjectMapper => JacksonScalaObjectMapper -} +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import com.fasterxml.jackson.module.scala.{ScalaObjectMapper => JacksonScalaObjectMapper} import com.twitter.io.Buf import com.twitter.util.jackson.caseclass.CaseClassJacksonModule -import com.twitter.util.jackson.serde.{DefaultSerdeModule, LongKeyDeserializers} +import com.twitter.util.jackson.serde.DefaultSerdeModule +import com.twitter.util.jackson.serde.LongKeyDeserializers import com.twitter.util.validation.ScalaValidator -import java.io.{ByteArrayOutputStream, InputStream, OutputStream} +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.io.OutputStream import java.nio.ByteBuffer object ScalaObjectMapper { @@ -392,6 +398,11 @@ object ScalaObjectMapper { this.additionalMapperConfigurationFns.foreach(_(underlying)) underlying.setPropertyNamingStrategy(this.propertyNamingStrategy) + // Block use of a set of "unsafe" base types such as java.lang.Object + // to prevent exploitation of Remote Code Execution (RCE) vulnerability + // This line can be removed when this feature is enabled by default in Jackson 3 + underlying.enable(MapperFeature.BLOCK_UNSAFE_POLYMORPHIC_BASE_TYPES) + this.jacksonModules.foreach(underlying.registerModule) underlying From 23d1570c6c571934e448e3c36bf67b949f625bec Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 15 Nov 2021 17:44:27 +0000 Subject: [PATCH 590/796] {finagle, util}: add yufang to PROJECT files Problem/Solution add yufang to PROJECT Differential Revision: https://phabricator.twitter.biz/D783073 --- PROJECT.yml | 1 + util-app/PROJECT | 1 + util-inject/PROJECT | 1 + util-jackson-annotations/PROJECT | 3 +++ util-jackson/PROJECT | 3 +++ util-logging/PROJECT | 1 + util-mock/PROJECT | 1 + util-security/PROJECT | 1 + util-slf4j-api/PROJECT | 2 ++ util-slf4j-jul-bridge/PROJECT | 2 ++ util-validator/PROJECT | 3 +++ util-zk-test/PROJECT | 2 ++ util-zk/PROJECT | 1 + 13 files changed, 22 insertions(+) diff --git a/PROJECT.yml b/PROJECT.yml index 44c4a877f0..6e0ace2668 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -6,5 +6,6 @@ owners: - mnakamura - roanta - vkostyukov + - yufang watchers: - csl-team:ldap diff --git a/util-app/PROJECT b/util-app/PROJECT index 98a843682c..fa9fb8e822 100644 --- a/util-app/PROJECT +++ b/util-app/PROJECT @@ -7,5 +7,6 @@ owners: - mnakamura - roanta - vkostyukov + - yufang watchers: - csl-team@twitter.com diff --git a/util-inject/PROJECT b/util-inject/PROJECT index 30197d422a..348b67af27 100644 --- a/util-inject/PROJECT +++ b/util-inject/PROJECT @@ -5,5 +5,6 @@ owners: - mnakamura - roanta - vkostyukov + - yufang watchers: - csl-team:ldap diff --git a/util-jackson-annotations/PROJECT b/util-jackson-annotations/PROJECT index 66d5ecc6bf..bd3e45cf4e 100644 --- a/util-jackson-annotations/PROJECT +++ b/util-jackson-annotations/PROJECT @@ -1,9 +1,12 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: + - banderson - ibennett - jyan - mnakamura - roanta + - vkostyukov + - yufang watchers: - csl-team@twitter.com diff --git a/util-jackson/PROJECT b/util-jackson/PROJECT index 66d5ecc6bf..bd3e45cf4e 100644 --- a/util-jackson/PROJECT +++ b/util-jackson/PROJECT @@ -1,9 +1,12 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: + - banderson - ibennett - jyan - mnakamura - roanta + - vkostyukov + - yufang watchers: - csl-team@twitter.com diff --git a/util-logging/PROJECT b/util-logging/PROJECT index 137e3d71c4..a482ca9a20 100644 --- a/util-logging/PROJECT +++ b/util-logging/PROJECT @@ -6,5 +6,6 @@ owners: - mnakamura - roanta - vkostyukov + - yufang watchers: - csl-team@twitter.com diff --git a/util-mock/PROJECT b/util-mock/PROJECT index 30197d422a..348b67af27 100644 --- a/util-mock/PROJECT +++ b/util-mock/PROJECT @@ -5,5 +5,6 @@ owners: - mnakamura - roanta - vkostyukov + - yufang watchers: - csl-team:ldap diff --git a/util-security/PROJECT b/util-security/PROJECT index fddaac65e3..66ca946cfc 100644 --- a/util-security/PROJECT +++ b/util-security/PROJECT @@ -7,5 +7,6 @@ owners: - bdimmick - roanta - vkostyukov + - yufang - csl-core-security:phab - platform-security:ldap diff --git a/util-slf4j-api/PROJECT b/util-slf4j-api/PROJECT index 223b72d975..a482ca9a20 100644 --- a/util-slf4j-api/PROJECT +++ b/util-slf4j-api/PROJECT @@ -5,5 +5,7 @@ owners: - ibennett - mnakamura - roanta + - vkostyukov + - yufang watchers: - csl-team@twitter.com diff --git a/util-slf4j-jul-bridge/PROJECT b/util-slf4j-jul-bridge/PROJECT index 223b72d975..a482ca9a20 100644 --- a/util-slf4j-jul-bridge/PROJECT +++ b/util-slf4j-jul-bridge/PROJECT @@ -5,5 +5,7 @@ owners: - ibennett - mnakamura - roanta + - vkostyukov + - yufang watchers: - csl-team@twitter.com diff --git a/util-validator/PROJECT b/util-validator/PROJECT index acd2e0abaa..2541e2a188 100644 --- a/util-validator/PROJECT +++ b/util-validator/PROJECT @@ -1,9 +1,12 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: + - banderson - ibennett - jyan - mnakamura - roanta + - vkostyukov + - yufang watchers: - csl-team@twitter.com \ No newline at end of file diff --git a/util-zk-test/PROJECT b/util-zk-test/PROJECT index a8f458ef04..f7f3d2b311 100644 --- a/util-zk-test/PROJECT +++ b/util-zk-test/PROJECT @@ -4,3 +4,5 @@ owners: - koliver - mnakamura - roanta + - vkostyukov + - yufang \ No newline at end of file diff --git a/util-zk/PROJECT b/util-zk/PROJECT index 50aa1a2a50..6199433a06 100644 --- a/util-zk/PROJECT +++ b/util-zk/PROJECT @@ -5,6 +5,7 @@ owners: - mnakamura - roanta - vkostyukov + - yufang watchers: - coord-team@twitter.com - csl-team:ldap From b0160db7f78152cb5cf3cc0cb8a361dac13f27cc Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Mon, 15 Nov 2021 18:26:19 +0000 Subject: [PATCH 591/796] doc: Remove reference of AbstractApplication in doc Problem/Solution Remove the reference of AbstractApplication in our user documentation. Differential Revision: https://phabricator.twitter.biz/D783059 --- doc/src/sphinx/util-stats/faq.rst | 38 ++++++++++++++----------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/doc/src/sphinx/util-stats/faq.rst b/doc/src/sphinx/util-stats/faq.rst index caa336d258..874892f641 100644 --- a/doc/src/sphinx/util-stats/faq.rst +++ b/doc/src/sphinx/util-stats/faq.rst @@ -10,15 +10,11 @@ The code was moved to `util `_ in order to allow usage without depending on finagle-core. To ease the transition, the package namespace was left as is. -Why were there three(!) different implementations? +Why were there two(!) different implementations? -------------------------------------------------- -This is somewhat historical, as early on in Twitter, Commons Stats -was created for Java developers and Ostrich was created for Scala developers. -Code initially was written directly to the implementations and there were -(and still are) some subtle differences in their behavior. -``StatsReceiver`` was written in order to bridge the gap between the -two implementations. `finagle-stats` was written quite a bit later in order to -improve upon both of these implementations in terms of +This is somewhat historical, as early on in Twitter, Ostrich was created for Scala developers. +`finagle-stats` was written quite a bit later in order to +improve upon its implementation in terms of allocation efficiency and histogram precision. Why is `NullStatsReceiver` being used? @@ -34,19 +30,19 @@ and `Maven Central `_ Why migrate to finagle-stats? ----------------------------- -The `finagle-stats` library is designed to replace the other two libraries -formerly supported by Twitter: Ostrich and Commons Stats. The comparison table below -illustrates the advantages of `finagle-stats` over the older libraries. +The `finagle-stats` library is designed to replace Ostrich which was +formerly supported by Twitter. The comparison table below +illustrates the advantages of `finagle-stats` over Ostrich. -+------------------------------+---------------+----------+-----------------+ -| | Commons Stats | Ostrich | finagle-stats | -+==============================+===============+==========+=================+ -| Allocations per `Stat.add` | 16 bytes | 0 bytes | 0 bytes | -+------------------------------+---------------+----------+-----------------+ -| Highest histogram resolution | p99 | p9999 | p9999 | -+------------------------------+---------------+----------+-----------------+ -| Sampling error | N/A | ±5% | ±0.5% | -+------------------------------+---------------+----------+-----------------+ ++------------------------------+----------+-----------------+ +| | Ostrich | finagle-stats | ++==============================+==========+=================+ +| Allocations per `Stat.add` | 0 bytes | 0 bytes | ++------------------------------+----------+-----------------+ +| Highest histogram resolution | p9999 | p9999 | ++------------------------------+----------+-----------------+ +| Sampling error | ±5% | ±0.5% | ++------------------------------+----------+-----------------+ `finagle-stats` allows you to visualize and download the full details of `Stat` histograms. This is available on the @@ -55,7 +51,7 @@ admin page. Are metric values reset or latched after an observability collection? --------------------------------------------------------------------- -``Stats`` are latched by all three implementations. +``Stats`` are latched by all two implementations. This means that for each observability collection window, you will see that window’s distribution of values. From 529293ac952a3321e6ca08b87721143b478b41c2 Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Mon, 15 Nov 2021 19:07:44 +0000 Subject: [PATCH 592/796] util-core: fix Var.collectIndependent Problem Var.collectIndependent allows us for collecting over a set of independent Vars in a stack safe manner. However, the implementation was broken. In particular, we failed to add observations to all the composite Vars since the `iterator` was lazy. With the previous implementation, we would get the following behavior: ``` val v1 = Var(1) val v2 = Var(2) val v3 = Var.collectIndependent(Seq[Var[Int]](v1, v2)) v3.changes.respond(println) // (1, null) v2() = 3 // would not even fire a new update! ``` Solution Rework the implementation to add handlers eagerly to the entire set of Vars we are collecting over and only publish an update when we have at least observed all the Vars. Differential Revision: https://phabricator.twitter.biz/D781639 --- .../src/main/scala/com/twitter/util/Var.scala | 60 ++++++++++--------- .../test/scala/com/twitter/util/VarTest.scala | 46 ++++++++++++-- 2 files changed, 72 insertions(+), 34 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 2169bcba96..3a9b470780 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -1,10 +1,12 @@ package com.twitter.util -import java.util.concurrent.atomic.{AtomicLong, AtomicReference, AtomicReferenceArray} +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.atomic.AtomicReference import java.util.{List => JList} import scala.annotation.tailrec import scala.collection.compat._ import scala.collection.immutable +import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.Buffer import scala.jdk.CollectionConverters._ import scala.language.higherKinds @@ -328,16 +330,11 @@ object Var { } yield left ++ right } - tree(0, vs.length).map { ts => - val b = factory.newBuilder - b ++= ts - b.result() - } + tree(0, vs.length).map(_.to(factory)) } /** * Collect a collection of Vars into a Var of collection. - * Var.collect can result in a stack overflow if called with a large sequence. * Var.collectIndependent breaks composition with respect to update propagation. * That is, collectIndependent can fail to correctly update interdependent vars, * but is safe for independent vars. @@ -362,33 +359,40 @@ object Var { )( implicit factory: Factory[T, CC[T]] ): Var[CC[T]] = - async(factory.newBuilder.result()) { v => + async(factory.newBuilder.result()) { u => val N = vars.size - val cur = new AtomicReferenceArray[T](N) + + // `filling` represents whether or not we have gone through our collection + // of `Var`s and added observations. Once we have "filled" the `cur` array + // for the first time, we can publish an initial update to `u`. Subsequent, + // updates to each constintuent Var are guarded by a lock on the `cur` and + // we ensure that we publish the update before any new updates can come + // through. + // + // @note there is still a subtle race where we can be in the middle + // of "filling" and receive updates on previously filled Vars. However, this + // is an acceptable race since technically we haven't added observations to all + // of the constituent Vars yet. + // + // @note "filling" only works with the guarantee that the initial `observe` is + // synchronous. This should be the case with Vars since they have an initial value. @volatile var filling = true - def build() = { - val b = factory.newBuilder - var i = 0 - while (i < N) { - b += cur.get(i) - i += 1 - } - b.result() - } + val cur = new Array[T](N) - def publish(i: Int, newValue: T) = { - cur.set(i, newValue) - if (!filling) v() = build() + def publish(i: Int, newValue: T): Unit = cur.synchronized { + cur(i) = newValue + if (!filling) u() = cur.to(factory) } - val closes = vars.iterator.zipWithIndex.map { - case (v, i) => v.observe(0, Observer(newValue => publish(i, newValue))) - }.toSeq - - filling = false - v() = build() + val closables = new ArrayBuffer[Closable](N) + val iter = vars.iterator.zipWithIndex + while (iter.hasNext) { + val (v, i) = iter.next() + if (i == N - 1) filling = false + closables += v.observe(newValue => publish(i, newValue)) + } - Closable.all(closes: _*) + Closable.all(closables: _*) } /** diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 9330e4d973..983928a64c 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -274,18 +274,52 @@ class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { assert(ref.get == Seq((1, 2), (2, 4))) } - test("Var.collectIndependent: not overflowing the stack") { - val vars: Seq[Var[Int]] = for (i <- 1 to 10000) yield { - Var(i) // vars(i-1).map(_+2) - } + test("Var.collect overflows the stack") { + val vars: Seq[Var[Int]] = for (i <- 1 to 10 * 1000) yield Var(i) intercept[java.lang.StackOverflowError] { val v = Var.collect(vars) - v.observe { x => } + v.observe { x => /* do nothing */ } } + } + test("Var.collectIndependent does NOT overflow the stack") { + val vars: Seq[Var[Int]] = for (i <- 1 to 10 * 1000) yield Var(i) val v = Var.collectIndependent(vars) - v.observe { x => } + v.observe { x => /* do nothing */ } + assert(v.sample == Seq.tabulate(10 * 1000)(i => i + 1)) + } + + test("Var.collectIndependent") { + val v1 = Var(10) + val v2 = Var(5) + val v3 = Var.collectIndependent(Seq[Var[Int]](v1, v2)) + + assert(v3.sample == Seq(10, 5)) + + v1() = 11 + assert(v3.sample == Seq(11, 5)) + + v2() = 6 + assert(v3.sample == Seq(11, 6)) + } + + test("difference between collect and collectIndependent") { + val v1 = Var(1) + val v2 = v1.map(_ * 2) + val vCollect = Var.collect(Seq(v1, v2)).map { case Seq(a, b) => (a, b) } + val vCollectIndependent = Var.collectIndependent(Seq(v1, v2)).map { case Seq(a, b) => (a, b) } + + val refCollect = new AtomicReference[Seq[(Int, Int)]] + vCollect.changes.build.register(Witness(refCollect)) + + val refCollectIndependent = new AtomicReference[Seq[(Int, Int)]] + vCollectIndependent.changes.build.register(Witness(refCollectIndependent)) + + v1() = 2 + + assert(refCollect.get == Vector((1, 2), (2, 4))) + assert(refCollectIndependent.get == Vector((1, 2), (2, 2), (2, 4))) } /** From 022d3567adc50c8620031c269c04633a7b93c52b Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 16 Nov 2021 21:36:28 +0000 Subject: [PATCH 593/796] finagle-core: Add failures as an metric expression Problem/Solution We would like to have failures as an expression, for client and server, to be auto rendered in the dashboards. JIRA Issues: CSL-11437 Differential Revision: https://phabricator.twitter.biz/D783013 --- .../com/twitter/finagle/stats/exp/Expression.scala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index 38999d0a34..f1499dddd5 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -23,7 +23,7 @@ private[twitter] object Expression { case (acc, expr) => getStatsReceivers(expr) ++ acc } - case MetricExpression(metricBuilder) => Set(metricBuilder.statsReceiver) + case MetricExpression(metricBuilder, _) => Set(metricBuilder.statsReceiver) case HistogramExpression(metricBuilder, _) => Set(metricBuilder.statsReceiver) case ConstantExpression(_) => Set.empty case StringExpression(_) => Set(LoadedStatsReceiver) @@ -57,12 +57,14 @@ private[twitter] object Expression { /** * Create an single expression wrapping a counter or gauge. + * @param metadata MetricBuilder instance + * @param showRollup If set true, inform the expression handler to render all tail metrics */ - def apply(metadata: Metadata): Expression = { + def apply(metadata: Metadata, showRollup: Boolean = false): Expression = { metadata.toMetricBuilder match { case Some(metricBuilder) => require(metricBuilder.metricType != HistogramType, "provide a component for histogram") - MetricExpression(metricBuilder) + MetricExpression(metricBuilder, showRollup) case None => NoExpression } @@ -117,7 +119,8 @@ case class FunctionExpression private[exp] (fnName: String, exprs: Seq[Expressio /** * Represents the leaf metrics */ -case class MetricExpression private[exp] (metricBuilder: MetricBuilder) extends Expression +case class MetricExpression private[exp] (metricBuilder: MetricBuilder, showRollup: Boolean) + extends Expression /** * Represent a histogram expression with specified component, for example the average, or a percentile From c721de6fc8c329770827aa36be80f731834be8f1 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 16 Nov 2021 22:23:17 +0000 Subject: [PATCH 594/796] util-core: Fix 2.13 build Problem / Solution 2.13 build is broken because of collection API changes. Differential Revision: https://phabricator.twitter.biz/D783829 --- util-core/src/main/scala/com/twitter/util/Var.scala | 2 +- util-core/src/test/scala/com/twitter/util/VarTest.scala | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 3a9b470780..fc30dcedbe 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -392,7 +392,7 @@ object Var { closables += v.observe(newValue => publish(i, newValue)) } - Closable.all(closables: _*) + Closable.all(closables.toSeq: _*) } /** diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 983928a64c..e50e4017a9 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -287,7 +287,7 @@ class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { val vars: Seq[Var[Int]] = for (i <- 1 to 10 * 1000) yield Var(i) val v = Var.collectIndependent(vars) v.observe { x => /* do nothing */ } - assert(v.sample == Seq.tabulate(10 * 1000)(i => i + 1)) + assert(v.sample() == Seq.tabulate(10 * 1000)(i => i + 1)) } test("Var.collectIndependent") { @@ -295,13 +295,13 @@ class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { val v2 = Var(5) val v3 = Var.collectIndependent(Seq[Var[Int]](v1, v2)) - assert(v3.sample == Seq(10, 5)) + assert(v3.sample() == Seq(10, 5)) v1() = 11 - assert(v3.sample == Seq(11, 5)) + assert(v3.sample() == Seq(11, 5)) v2() = 6 - assert(v3.sample == Seq(11, 6)) + assert(v3.sample() == Seq(11, 6)) } test("difference between collect and collectIndependent") { From bb92fd69e2a28bcd639b396dab20100a5c453958 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Wed, 17 Nov 2021 16:43:06 +0000 Subject: [PATCH 595/796] util-stats: add testing suite for metric expressions Problem: It's currently difficult to test for correct implementation of metric expressions with a short development loop (ie without having to deploy and check the admin endpoint). Solution: Build a testing suite that demands you define only the statsReceiver and the downstream label for metric labels. This suite will make it easy to get a full summary of registered expressions, assert specific expressions are instrumented, and confirm a set of expected expressions matches the actual expressions registered on the stats receiver. JIRA Issues: CSL-11282 Differential Revision: https://phabricator.twitter.biz/D783532 --- build.sbt | 2 +- .../finagle/stats/InMemoryStatsReceiver.scala | 3 +- .../main/scala/com/twitter/util/testing/BUILD | 2 + .../util/testing/ExpressionTestMixin.scala | 60 +++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 util-test/src/main/scala/com/twitter/util/testing/ExpressionTestMixin.scala diff --git a/build.sbt b/build.sbt index a2b7ede2aa..72cfec7d25 100644 --- a/build.sbt +++ b/build.sbt @@ -676,7 +676,7 @@ lazy val utilTest = Project( "org.scalatestplus" %% "junit-4-12" % "3.1.2.0", "org.scalatestplus" %% "mockito-1-10" % "3.1.0.0" ) - ).dependsOn(utilCore, utilLogging) + ).dependsOn(utilCore, utilLogging, utilStats) lazy val utilThrift = Project( id = "util-thrift", diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index 775e50fc5b..ebe74d4246 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -236,7 +236,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } /** - * Clears all registered counters, gauges, stats, and their metadata. + * Clears all registered counters, gauges, stats, expressions, and their metadata. * @note this is not atomic. If new metrics are added while this method is executing, those metrics may remain. */ def clear(): Unit = { @@ -244,6 +244,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { stats.clear() gauges.clear() schemas.clear() + expressions.clear() } private[this] def toHistogramDetail(addedValues: Seq[Float]): HistogramDetail = { diff --git a/util-test/src/main/scala/com/twitter/util/testing/BUILD b/util-test/src/main/scala/com/twitter/util/testing/BUILD index 1c151e7679..ff7788586f 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/main/scala/com/twitter/util/testing/BUILD @@ -11,6 +11,8 @@ scala_library( "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:mockito-1-10", + "util/util-stats/src/main/scala", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], exports = [ "3rdparty/jvm/org/scalatest", diff --git a/util-test/src/main/scala/com/twitter/util/testing/ExpressionTestMixin.scala b/util-test/src/main/scala/com/twitter/util/testing/ExpressionTestMixin.scala new file mode 100644 index 0000000000..2d487a6c43 --- /dev/null +++ b/util-test/src/main/scala/com/twitter/util/testing/ExpressionTestMixin.scala @@ -0,0 +1,60 @@ +package com.twitter.util.testing + +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey +import com.twitter.finagle.stats.InMemoryStatsReceiver +import org.scalatest.BeforeAndAfterEach +import org.scalatest.Suite +import org.scalatest.SuiteMixin +import scala.collection.mutable + +/** + * Testing trait which provides utilities for testing for proper + * [[com.twitter.finagle.stats.exp.Expression]] registration + */ +trait ExpressionTestMixin extends SuiteMixin with BeforeAndAfterEach { this: Suite => + + protected def downstreamLabel: Map[String, String] + protected val sr: InMemoryStatsReceiver + + protected def nameToKey( + name: String, + labels: Map[String, String] = Map(), + namespaces: Seq[String] = Seq() + ): ExpressionSchemaKey = + ExpressionSchemaKey(name, downstreamLabel ++ labels, namespaces) + + override protected def beforeEach(): Unit = { + sr.clear() + super.beforeEach() + } + + /** + * Given the name, labels, and namespaces that make up a given ExpressionSchemaKey, + * assert that expression is registered on the StatsReceiver. + */ + protected def assertExpressionIsRegistered( + name: String, + labels: Map[String, String] = Map(), + namespaces: Seq[String] = Seq() + ): Unit = { + assert(sr.expressions.contains(nameToKey(name, labels, namespaces))) + } + + /** + * Returns a collection of all registered expressions on the provided StatsReceiver + */ + protected def getMetricsReview: mutable.Map[ExpressionSchemaKey, ExpressionSchema] = { + sr.expressions + } + + /** + * Given a collection of expected ExpressionSchemaKeys, ensure that collection matches what is + * registered. + * + * @param expected a Set of [[ExpressionSchemaKey]]s that should be registered on the StatsReceiver + */ + protected def assertExpressionsAsExpected(expected: Set[ExpressionSchemaKey]): Unit = { + assert(sr.expressions.keySet == expected) + } +} From 3643e1e870c1a6d6fb521a5f0e00c0b9e1e7d4fe Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Fri, 19 Nov 2021 00:58:58 +0000 Subject: [PATCH 596/796] util/util-security: Fix Credentials fails to load valid yaml credentials Problem com.twitter.util.security.Credentials fails to load valid yaml credentials, and it loads invalid yaml credentials. For example, `key:!value` is accepted as a valid Map("key", "!value"), but `{"key": "value"}` is rejected as invalid. Solution Replace the buggy custom parser with a vetted 3rdparty one (snakeyaml) Result Users can load valid yaml files, and invalid yaml files are rejected. JIRA Issues: PSEC-10357 Differential Revision: https://phabricator.twitter.biz/D617641 --- CHANGELOG.rst | 12 +++++ build.sbt | 5 +- .../scala/com/twitter/util/security/BUILD | 1 + .../twitter/util/security/Credentials.scala | 50 +++++++------------ .../util/security/CredentialsTest.scala | 26 +++++----- 5 files changed, 49 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9c2d49b6e2..f03d2721d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,18 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ +* util-security: Use snakeyaml to parse yaml instead of a buggy custom yaml + parser. This means that thrown IOExceptions have been replaced by + YAMLExceptions. Additionally, the parser member has been limited to private visibility. ``PHAB_ID=D617641`` + +New Features +~~~~~~~~~~~~ + +* util-security: Any valid yaml / json file with string keys and values can + be loaded with `com.twitter.util.security.Credentials`. ``PHAB_ID=D617641`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index 72cfec7d25..f0af560136 100644 --- a/build.sbt +++ b/build.sbt @@ -25,6 +25,8 @@ val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.9.2" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.15.4" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion +val snakeyaml = "org.yaml" % "snakeyaml" % "1.24" + def travisTestJavaOptions: Seq[String] = { // We have some custom configuration for the Travis environment @@ -623,7 +625,8 @@ lazy val utilSecurity = Project( name := "util-security", libraryDependencies ++= Seq( scalacheckLib, - "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test" + "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", + snakeyaml ) ).dependsOn(utilCore, utilLogging, utilSecurityTestCerts % "test") diff --git a/util-security/src/main/scala/com/twitter/util/security/BUILD b/util-security/src/main/scala/com/twitter/util/security/BUILD index 75f995aca1..4ca8f81b88 100644 --- a/util-security/src/main/scala/com/twitter/util/security/BUILD +++ b/util-security/src/main/scala/com/twitter/util/security/BUILD @@ -8,6 +8,7 @@ scala_library( strict_deps = True, tags = ["bazel-compatible"], dependencies = [ + "3rdparty/jvm/org/yaml:snakeyaml", "util/util-core:util-core-util", "util/util-logging/src/main/scala/com/twitter/logging", ], diff --git a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala index 24cb8a7685..376bc3a903 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala @@ -16,48 +16,36 @@ package com.twitter.util.security -import java.io.{File, IOException} - +import java.io.File +import org.yaml.snakeyaml.Yaml +import scala.collection.JavaConverters._ import scala.io.Source -import scala.jdk.CollectionConverters._ -import scala.util.parsing.combinator._ -import scala.util.matching.Regex /** * Simple helper to read authentication credentials from a text file. * - * The file's format is assumed to be trivialized yaml, containing lines of the form ``key: value``. - * Keys can be any word character or '-' and values can be any character except new lines. + * The file's format is assumed to be yaml, containing string keys and values. */ object Credentials { - object parser extends RegexParsers { - - override val whiteSpace: Regex = "(?:\\s+|#.*\\r?\\n)+".r - - private[this] val key = "[\\w-]+".r - private[this] val value = ".+".r - - def auth: Parser[Tuple2[String, String]] = key ~ ":" ~ value ^^ { - case k ~ ":" ~ v => (k, v) - case _ => throw new MatchError("Failed case matching on auth parser") - } - def content: Parser[Map[String, String]] = rep(auth) ^^ { auths => Map(auths: _*) } - - def apply(in: String): Map[String, String] = { - parseAll(content, in) match { - case Success(result, _) => result - case x: Failure => throw new IOException(x.toString) - case x: Error => throw new IOException(x.toString) - } - } + private[this] val parser: ThreadLocal[Yaml] = new ThreadLocal[Yaml] { + override def initialValue(): Yaml = new Yaml() } - def apply(file: File): Map[String, String] = parser(Source.fromFile(file).mkString) + def byName(name: String): Map[String, String] = { + apply(new File(sys.env.getOrElse("KEY_FOLDER", "/etc/keys"), name)) + } - def apply(data: String): Map[String, String] = parser(data) + def apply(file: File): Map[String, String] = { + val fileSource = Source.fromFile(file) + try apply(fileSource.mkString) + finally fileSource.close() + } - def byName(name: String): Map[String, String] = { - apply(new File(System.getenv().asScala.getOrElse("KEY_FOLDER", "/etc/keys"), name)) + def apply(data: String): Map[String, String] = { + val result: java.util.Map[String, Any] = parser.get.load(data) + Option(result) + .map(_.asScala.toMap.mapValues(v => if (v == null) "null" else v.toString)).getOrElse( + Map.empty) } } diff --git a/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala b/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala index 32360795cd..46ad0da301 100644 --- a/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala +++ b/util-security/src/test/scala/com/twitter/util/security/CredentialsTest.scala @@ -21,48 +21,48 @@ import org.scalatest.funsuite.AnyFunSuite class CredentialsTest extends AnyFunSuite with Checkers { test("parse a simple auth file") { - val content = "username: root\npassword: hellokitty\n" + val content = "username: root\npassword: not_a_password\n" val result = Credentials(content) - assert(result == Map("username" -> "root", "password" -> "hellokitty")) + assert(result == Map("username" -> "root", "password" -> "not_a_password")) } test("parse a more complex auth file") { val content = """# random comment -username:root -password : last_0f-the/international:playboys +username: root +password: not_a-real/pr0d:password # more stuff - moar :ok +moar: ok """ assert( Credentials(content) == Map( "username" -> "root", - "password" -> "last_0f-the/international:playboys", + "password" -> "not_a-real/pr0d:password", "moar" -> "ok" ) ) } test("work for java peeps too") { - val content = "username: root\npassword: hellokitty\n" + val content = "username: root\npassword: not_a_password\n" val jmap = new Credentials().read(content) assert(jmap.size() == 2) assert(jmap.get("username") == "root") - assert(jmap.get("password") == "hellokitty") + assert(jmap.get("password") == "not_a_password") } test("handle \r\n line breaks") { - val content = "username: root\r\npassword: hellokitty\r\n" - assert(Credentials(content) == Map("username" -> "root", "password" -> "hellokitty")) + val content = "username: root\r\npassword: not_a_password\r\n" + assert(Credentials(content) == Map("username" -> "root", "password" -> "not_a_password")) } test("handle special chars") { - val pass = (0 to 127) + val pass = (32 to 126) .map(_.toChar) - .filter(c => c != '\r' && c != '\n') + .filter(c => c != '\r' && c != '\n' && c != '\'') .mkString - val content = s"username: root\npassword: $pass\n" + val content = s"username: root\npassword: '$pass'\n" assert(Credentials(content) == Map("username" -> "root", "password" -> pass)) } } From 7dfd6efc728ec44cfc551e00fa01469a830787c4 Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Fri, 19 Nov 2021 19:45:37 +0000 Subject: [PATCH 597/796] util/util-security: scala2.13 compat for MapValues Problem scala2.13 introduce a breaking change to mapValues to return a view instead of a map. This broke the util-security build which was updated in D617641 to use mapValues. Solution Force the view into a map with `toMap` Result util-security can build for scala2.13 JIRA Issues: PSEC-10357 Differential Revision: https://phabricator.twitter.biz/D787935 --- .../src/main/scala/com/twitter/util/security/Credentials.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala index 376bc3a903..9093a3b17b 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala @@ -44,7 +44,7 @@ object Credentials { def apply(data: String): Map[String, String] = { val result: java.util.Map[String, Any] = parser.get.load(data) Option(result) - .map(_.asScala.toMap.mapValues(v => if (v == null) "null" else v.toString)).getOrElse( + .map(_.asScala.toMap.mapValues(v => if (v == null) "null" else v.toString).toMap).getOrElse( Map.empty) } } From 78ed6237ba8077457b8ff1bec294c615f75baee5 Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Tue, 23 Nov 2021 19:00:17 +0000 Subject: [PATCH 598/796] Twitter-oss: Prepare OSS libraries for release 21.11.0 Problem We want to release the next versions of our Twitter OSS libraries 21.11.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11489 Differential Revision: https://phabricator.twitter.biz/D789826 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f03d2721d6..dcbcbdf2f4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.11.0 +------- + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ * util-security: Use snakeyaml to parse yaml instead of a buggy custom yaml diff --git a/README.md b/README.md index 62fc500208..48b30b12cb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -40,7 +40,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.10.0" +val utilCore = "com.twitter" %% "util-core" % "21.11.0" ``` # Units diff --git a/build.sbt b/build.sbt index f0af560136..80ed57930e 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.11.0-SNAPSHOT" +val releaseVersion = "21.11.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.4" From e3a8e580a3480ee9d28e1169d46a969eedd1cc7a Mon Sep 17 00:00:00 2001 From: Dorothy Ordogh Date: Tue, 23 Nov 2021 20:19:35 +0000 Subject: [PATCH 599/796] Twitter-oss: update OSS libraries post-release to 21.12.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 21.12.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 48b30b12cb..adff67a751 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 80ed57930e..a601528bbc 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.11.0" +val releaseVersion = "21.12.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.4" From 15f144f6852bb29d7f204a231d733e693c62d0dd Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 3 Dec 2021 18:11:29 +0000 Subject: [PATCH 600/796] csl: Bump SBT to 1.5.5 Problem / Solution Bump SBT to 1.5.5 for CSL libraries and upgrade selected plugins. JIRA Issues: CSL-11423 Differential Revision: https://phabricator.twitter.biz/D792862 --- .github/workflows/continuous-integration.yml | 2 +- build.sbt | 4 +++- project/build.properties | 2 +- project/plugins.sbt | 8 ++++---- project/unidoc.sbt | 2 +- sbt | 4 ++-- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index eab6daf38c..271afeb91c 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -3,7 +3,7 @@ name: continuous integration env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI - SBT_VERSION: 1.5.3 + SBT_VERSION: 1.5.5 defaults: run: diff --git a/build.sbt b/build.sbt index a601528bbc..ea5e60c789 100644 --- a/build.sbt +++ b/build.sbt @@ -213,7 +213,9 @@ val sharedSettings = val sharedScala3EnabledSettings = baseSettings ++ defaultScala3EnabledSettings ++ - Seq(libraryDependencies ++= scala3Dependencies) + Seq(libraryDependencies ++= scala3Dependencies) ++ + Seq(excludeDependencies ++= Seq("org.scala-lang.modules" % "scala-collection-compat_3")) + lazy val noPublishSettings = Seq( publish / skip := true diff --git a/project/build.properties b/project/build.properties index 67d27a1dfe..10fd9eee04 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.3 +sbt.version=1.5.5 diff --git a/project/plugins.sbt b/project/plugins.sbt index b14aeff542..d959da10ca 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,7 +1,7 @@ resolvers += Classpaths.sbtPluginReleases resolvers += Resolver.sonatypeRepo("snapshots") -addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1") -addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.0") -addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.29") +addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.1") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3") +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.31") diff --git a/project/unidoc.sbt b/project/unidoc.sbt index 5818ca1f65..a394be19d8 100644 --- a/project/unidoc.sbt +++ b/project/unidoc.sbt @@ -1 +1 @@ -addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1") +addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3") diff --git a/sbt b/sbt index 2441ede7e4..dd557c21d8 100755 --- a/sbt +++ b/sbt @@ -2,9 +2,9 @@ set -eo pipefail -sbtver="1.5.3" +sbtver="1.5.5" sbtjar="sbt-launch-$sbtver.jar" -sbtsha128="1ca2d0ee419a1f82f512f2aa8556d6e262b37961" +sbtsha128="3beb23e861de5021bd80ca52113b075321580265" sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From 86c9f422c5edc76bd5aebafaf5f322835c402373 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Fri, 3 Dec 2021 21:14:38 +0000 Subject: [PATCH 601/796] finagle|finatra|scrooge|ts|util: Add build version to GH Actions build ccache key Problem Some generated files are cached in the build cache but the thrift files have since been removed, which causes build failures. Solution Add a way to easily invalidate the build cache so if we experience something like this again, we can simply bump the version number and move on to a successful build. Differential Revision: https://phabricator.twitter.biz/D792859 --- .github/workflows/continuous-integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 271afeb91c..2908f5390b 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -4,6 +4,7 @@ env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI SBT_VERSION: 1.5.5 + BUILD_VERSION: "v1" # bump this if builds are failing due to a bad cache defaults: run: @@ -41,7 +42,7 @@ jobs: ~/.ivy2/cache ~/.ivy2/local/com.twitter ~/.sbt - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ env.SBT_VERSION }}-${{ matrix.scala }}-${{ matrix.java }} + key: ${{ runner.os }}-build-${{ env.BUILD_VERSION }}-${{ env.cache-name }}-${{ env.SBT_VERSION }}-${{ matrix.scala }}-${{ matrix.java }} - name: update cache run: | if [ -f ~/.ivy2/cache ]; then From caf528e2ff7a7686e4f9cc7fe95553669fa064b8 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 7 Dec 2021 18:17:19 +0000 Subject: [PATCH 602/796] util-core: Improve performance and simplify Var.collects Problem `Var.collectIndependent` is pretty slow as it has to copy an entire result (array) on every update of every individual var. Solution Use similar technic from Future.collect and avoid intermediate copies of the result by wrapping a mutable array with an immutable interface. Result This also drops higher-kinded API for both Activity's and Var's collect methods. It can't be a generic CC[X] anymore and has to be Seq, just like in Future.collect. JIRA Issues: CSL-11377 Differential Revision: https://phabricator.twitter.biz/D795123 --- CHANGELOG.rst | 7 +++ .../scala/com/twitter/util/Activity.scala | 45 +++++++------------ .../src/main/scala/com/twitter/util/Var.scala | 43 +++++++++--------- .../test/scala/com/twitter/util/VarTest.scala | 17 ------- 4 files changed, 43 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dcbcbdf2f4..d71354bfd0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ +* util-core: `Activity.collect*` and `Var.collect*` are now implemented in terms of known collection + type `scala.collection.Seq` versus HKT `CC[X]` before. This allows for certain performance + enhancements as well as makes it more aligned with the `Future.collect` APIs. + ``PHAB_ID=D795123`` + 21.11.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/Activity.scala b/util-core/src/main/scala/com/twitter/util/Activity.scala index 75675b4285..5fbadd9a3a 100644 --- a/util-core/src/main/scala/com/twitter/util/Activity.scala +++ b/util-core/src/main/scala/com/twitter/util/Activity.scala @@ -1,14 +1,11 @@ package com.twitter.util import java.util.{List => JList} - +import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.Buffer import scala.jdk.CollectionConverters._ -import scala.language.higherKinds -import scala.reflect.ClassTag import scala.util.control.NonFatal -import scala.Iterable -import scala.collection.compat._ +import scala.collection.{Seq => AnySeq} /** * Activity, like [[com.twitter.util.Var Var]], is a value that varies over @@ -195,11 +192,7 @@ object Activity { * * @example def collect[T](activities: Coll[Activity[T]]): Activity[Coll[T]] */ - def collect[T: ClassTag, CC[X] <: Iterable[X]]( - acts: CC[Activity[T]] - )( - implicit factory: Factory[T, CC[T]] - ): Activity[CC[T]] = { + def collect[T](acts: AnySeq[Activity[T]]): Activity[Seq[T]] = { collect(acts, false) } @@ -213,32 +206,26 @@ object Activity { * * @example def collectIndependent[T](activities: Coll[Activity[T]]): Activity[Coll[T]] */ - def collectIndependent[T: ClassTag, CC[X] <: Iterable[X]]( - acts: CC[Activity[T]] - )( - implicit factory: Factory[T, CC[T]] - ): Activity[CC[T]] = { + def collectIndependent[T](acts: AnySeq[Activity[T]]): Activity[Seq[T]] = { collect(acts, true) } - private[this] def collect[T: ClassTag, CC[X] <: Iterable[X]]( - acts: CC[Activity[T]], + private[this] def collect[T]( + acts: AnySeq[Activity[T]], collectIndependent: Boolean - )( - implicit factory: Factory[T, CC[T]] - ): Activity[CC[T]] = { + ): Activity[Seq[T]] = { if (acts.isEmpty) - return Activity.value(factory.fromSpecific(Nil)) + return Activity.value(Seq.empty[T]) - val states: Iterable[Var[State[T]]] = acts.map(_.run) - val stateVar: Var[Iterable[State[T]]] = if (collectIndependent) { + val states: AnySeq[Var[State[T]]] = acts.map(_.run) + val stateVar: Var[Seq[State[T]]] = if (collectIndependent) { Var.collectIndependent(states) } else { Var.collect(states) } - def flip(states: Iterable[State[T]]): State[CC[T]] = { - val notOk = states find { + def flip(states: AnySeq[State[T]]): State[Seq[T]] = { + val notOk = states.find { case Pending | Failed(_) => true case Ok(_) => false } @@ -250,16 +237,16 @@ object Activity { case Some(_) => assert(false) } - val ts = factory.newBuilder - states foreach { + val ts = new ArrayBuffer[T](states.size) + states.foreach { case Ok(t) => ts += t case _ => assert(false) } - Ok(ts.result()) + Ok(ts.toSeq) } - Activity(stateVar map flip) + Activity(stateVar.map(flip)) } /** diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index fc30dcedbe..d5719372ca 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -4,14 +4,13 @@ import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicReference import java.util.{List => JList} import scala.annotation.tailrec -import scala.collection.compat._ +import scala.collection.{Seq => AnySeq} +import scala.collection.compat.immutable.ArraySeq import scala.collection.immutable import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.Buffer import scala.jdk.CollectionConverters._ import scala.language.higherKinds -import scala.reflect.ClassTag -import scala.Iterable /** * Vars are values that vary over time. To create one, you must give it an @@ -311,12 +310,8 @@ object Var { * // refCollectIndependent == Vector((1,2), (2,2), (2,4)) * }}} */ - def collect[T: ClassTag, CC[X] <: Iterable[X]]( - vars: CC[Var[T]] - )( - implicit factory: Factory[T, CC[T]] - ): Var[CC[T]] = { - val vs = vars.toArray + def collect[T](vars: AnySeq[Var[T]]): Var[Seq[T]] = { + val vs = vars.toIndexedSeq def tree(begin: Int, end: Int): Var[Seq[T]] = if (begin == end) Var(Seq.empty) @@ -330,7 +325,7 @@ object Var { } yield left ++ right } - tree(0, vs.length).map(_.to(factory)) + tree(0, vs.length) } /** @@ -354,12 +349,8 @@ object Var { * // refCollectIndependent == Vector((1,2), (2,2), (2,4)) * }}} */ - def collectIndependent[T: ClassTag, CC[X] <: Iterable[X]]( - vars: CC[Var[T]] - )( - implicit factory: Factory[T, CC[T]] - ): Var[CC[T]] = - async(factory.newBuilder.result()) { u => + def collectIndependent[T](vars: AnySeq[Var[T]]): Var[Seq[T]] = + async(Seq.empty[T]) { u => val N = vars.size // `filling` represents whether or not we have gone through our collection @@ -377,19 +368,25 @@ object Var { // @note "filling" only works with the guarantee that the initial `observe` is // synchronous. This should be the case with Vars since they have an initial value. @volatile var filling = true - val cur = new Array[T](N) + val cur = new Array[Any](N) - def publish(i: Int, newValue: T): Unit = cur.synchronized { - cur(i) = newValue - if (!filling) u() = cur.to(factory) + def publishAt(i: Int): T => Unit = { newValue => + cur.synchronized { + cur(i) = newValue + if (!filling) { + u() = ArraySeq.unsafeWrapArray(cur).asInstanceOf[ArraySeq[T]] + } + } } val closables = new ArrayBuffer[Closable](N) - val iter = vars.iterator.zipWithIndex + var i = 0 + val iter = vars.iterator while (iter.hasNext) { - val (v, i) = iter.next() + val v = iter.next() if (i == N - 1) filling = false - closables += v.observe(newValue => publish(i, newValue)) + closables += v.observe(publishAt(i)) + i += 1 } Closable.all(closables.toSeq: _*) diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index e50e4017a9..3bf1c3796a 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -243,23 +243,6 @@ class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { } } - // This is either very neat or very horrendous, - // depending on your point of view. - test("Var.collect[Set]") { - val vars = Seq(Var(1), Var(2), Var(3)) - - val coll = Var.collect(vars.map { v => v: Var[Int] }.toSet) - val ref = new AtomicReference[Set[Int]] - coll.changes.register(Witness(ref)) - assert(ref.get == Set(1, 2, 3)) - - vars(1).update(1) - assert(ref.get == Set(1, 3)) - - vars(1).update(999) - assert(ref.get == Set(1, 999, 3)) - } - test("Var.collect: ordering") { val v1 = Var(1) val v2 = v1.map(_ * 2) From 6b16fd1b99a7e702198135c6937aef13433b6fa7 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 9 Dec 2021 22:43:43 +0000 Subject: [PATCH 603/796] util-security: Make sure we return the right dynamic type Problem / Solution Due to unforseen consequences, we need to make sure the dynamic type returned by `Credentials.apply(String)` returns an `immutable.HashMap`. Differential Revision: https://phabricator.twitter.biz/D797920 --- .../twitter/util/security/Credentials.scala | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala index 9093a3b17b..6bb8ceef27 100644 --- a/util-security/src/main/scala/com/twitter/util/security/Credentials.scala +++ b/util-security/src/main/scala/com/twitter/util/security/Credentials.scala @@ -19,6 +19,7 @@ package com.twitter.util.security import java.io.File import org.yaml.snakeyaml.Yaml import scala.collection.JavaConverters._ +import scala.collection.immutable.HashMap import scala.io.Source /** @@ -27,9 +28,6 @@ import scala.io.Source * The file's format is assumed to be yaml, containing string keys and values. */ object Credentials { - private[this] val parser: ThreadLocal[Yaml] = new ThreadLocal[Yaml] { - override def initialValue(): Yaml = new Yaml() - } def byName(name: String): Map[String, String] = { apply(new File(sys.env.getOrElse("KEY_FOLDER", "/etc/keys"), name)) @@ -42,10 +40,16 @@ object Credentials { } def apply(data: String): Map[String, String] = { - val result: java.util.Map[String, Any] = parser.get.load(data) - Option(result) - .map(_.asScala.toMap.mapValues(v => if (v == null) "null" else v.toString).toMap).getOrElse( - Map.empty) + val parser = new Yaml() + val result: java.util.Map[String, Any] = parser.load(data) + if (result == null) Map.empty + else { + val builder = HashMap.newBuilder[String, String] + result.forEach { (k, v) => + builder += k -> String.valueOf(v) + } + builder.result() + } } } From e27ca40872445a4e2b01be0526ac78bbe0a7c865 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 10 Dec 2021 19:44:36 +0000 Subject: [PATCH 604/796] util-core: Small performance improvements to Var Problem / Solution Misc perf improvements to Vars. JIRA Issues: CSL-11377 Differential Revision: https://phabricator.twitter.biz/D797736 --- .../src/main/scala/com/twitter/util/Var.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index d5719372ca..5f48be10ea 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -7,7 +7,6 @@ import scala.annotation.tailrec import scala.collection.{Seq => AnySeq} import scala.collection.compat.immutable.ArraySeq import scala.collection.immutable -import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.Buffer import scala.jdk.CollectionConverters._ import scala.language.higherKinds @@ -379,17 +378,17 @@ object Var { } } - val closables = new ArrayBuffer[Closable](N) + val closables = new Array[Any](N) var i = 0 val iter = vars.iterator while (iter.hasNext) { val v = iter.next() if (i == N - 1) filling = false - closables += v.observe(publishAt(i)) + closables(i) = v.observe(publishAt(i)) i += 1 } - Closable.all(closables.toSeq: _*) + Closable.all(ArraySeq.unsafeWrapArray(closables).asInstanceOf[ArraySeq[Closable]]: _*) } /** @@ -485,15 +484,17 @@ private object UpdatableVar { def :=(newv: T): State[T] = copy(value = newv, version = version + 1) } - implicit def order[T]: Ordering[Party[T]] = new Ordering[Party[T]] { + private val anyOrder: Ordering[Party[Any]] = new Ordering[Party[Any]] { // This is safe because observers are compared // only from the same counter. - def compare(a: Party[T], b: Party[T]): Int = { - val c1 = a.depth compare b.depth + def compare(a: Party[Any], b: Party[Any]): Int = { + val c1 = java.lang.Integer.compare(a.depth, b.depth) if (c1 != 0) return c1 - a.n compare b.n + java.lang.Long.compare(a.n, b.n) } } + + implicit def order[T]: Ordering[Party[T]] = anyOrder.asInstanceOf[Ordering[Party[T]]] } private[util] class UpdatableVar[T](init: T) extends Var[T] with Updatable[T] with Extractable[T] { From 87760497359a87bc3bc101b98fdee9042caff3fa Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 13 Dec 2021 19:12:18 +0000 Subject: [PATCH 605/796] util-core: Optimize Closable.sequence for the case of 2 Problem Closable.sequence is often used with just two underlying closables (very popular in Var) in the collection. The generic implementation (in terms of Seq) is pretty inefficient for that use case. Solution Provide a Closable.sequence implementation for just two elements. JIRA Issues: CSL-11377 Differential Revision: https://phabricator.twitter.biz/D797566 --- .../scala/com/twitter/util/Closable.scala | 41 ++++++++++-- .../scala/com/twitter/util/ClosableTest.scala | 64 ++++++++++++------- 2 files changed, 79 insertions(+), 26 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Closable.scala b/util-core/src/main/scala/com/twitter/util/Closable.scala index d909bfc6d5..f365715e0a 100644 --- a/util-core/src/main/scala/com/twitter/util/Closable.scala +++ b/util-core/src/main/scala/com/twitter/util/Closable.scala @@ -1,10 +1,15 @@ package com.twitter.util -import java.lang.ref.{PhantomReference, Reference, ReferenceQueue} +import java.lang.ref.PhantomReference +import java.lang.ref.Reference +import java.lang.ref.ReferenceQueue import java.{util => ju} -import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} -import java.util.logging.{Level, Logger} -import scala.annotation.{tailrec, varargs} +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicReference +import java.util.logging.Level +import java.util.logging.Logger +import scala.annotation.tailrec +import scala.annotation.varargs import scala.util.control.{NonFatal => NF} /** @@ -99,6 +104,34 @@ object Closable { } } + /** + * Sequential composition: create a new Closable which, when + * closed, closes all of the underlying ones in sequence: that is, + * resource ''n+1'' is not closed until resource ''n'' is. + * + * @return the first failed [[Future]] should any of the `Closables` + * result in a failed [[Future]]. + * + * @note as with all `Closables`, the `deadline` passed to `close` + * is advisory. + */ + def sequence(a: Closable, b: Closable): Closable = new Closable { + def close(deadline: Time): Future[Unit] = { + val first = safeClose(a, deadline) + + def onFirstComplete(t: Try[Unit]): Future[Unit] = { + val second = safeClose(b, deadline) + if (t.isThrow) if (second.isDefined) first else second.transform(_ => first) + else second + } + + first.poll match { + case Some(t) => onFirstComplete(t) + case None => first.transform(onFirstComplete) + } + } + } + /** * Sequential composition: create a new Closable which, when * closed, closes all of the underlying ones in sequence: that is, diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index 178593abf8..bed043c65b 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -1,7 +1,8 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ -import org.scalatest.concurrent.{Eventually, IntegrationPatience} +import org.scalatest.concurrent.Eventually +import org.scalatest.concurrent.IntegrationPatience import org.scalatest.funsuite.AnyFunSuite class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience { @@ -105,13 +106,40 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience ) } + test("Cloasable.make catches NonFatals and translates to failed Futures") { + val throwing = Closable.make { _ => throw new Exception("lolz") } + + var failed = 0 + val f = throwing.close().respond { + case Throw(_) => failed = -1 + case _ => failed = 1 + } + + val ex = intercept[Exception] { + Await.result(f, 2.seconds) + } + assert(ex.getMessage.contains("lolz")) + assert(failed == -1) + } +} + +abstract class ClosableSequenceTest extends AnyFunSuite with Eventually with IntegrationPatience { + + def sequenceTwo(a: Closable, b: Closable): Closable + + private[this] def assertIsDone(f: Future[Unit]): Unit = + assert(f.isDefined) + + private[this] def assertIsNotDone(f: Future[Unit]): Unit = + assert(!f.isDefined) + test("Closable.sequence") { val p1, p2 = new Promise[Unit] var n1, n2 = 0 val c1 = Closable.make(_ => { n1 += 1; p1 }) val c2 = Closable.make(_ => { n2 += 1; p2 }) - val c = Closable.sequence(c1, c2) + val c = sequenceTwo(c1, c2) assert(n1 == 0) assert(n2 == 0) @@ -143,7 +171,7 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience } // test with a failed close first - val f1 = Closable.sequence(c1, c2).close() + val f1 = sequenceTwo(c1, c2).close() assert(n1 == 1) assert(n2 == 1) val x1 = intercept[RuntimeException] { @@ -152,7 +180,7 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience assert(x1.getMessage.contains("n1=1")) // then test in reverse order, failed close last - val f2 = Closable.sequence(c2, c1).close() + val f2 = sequenceTwo(c2, c1).close() assert(n1 == 2) assert(n2 == 2) val x2 = intercept[RuntimeException] { @@ -161,7 +189,7 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience assert(x2.getMessage.contains("n1=2")) // multiple failures, returns the first failure - val f3 = Closable.sequence(c1, c1).close() + val f3 = sequenceTwo(c1, c1).close() assert(n1 == 4) val x3 = intercept[RuntimeException] { Await.result(f3, 5.seconds) @@ -172,7 +200,7 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience // last closable is satisfied val p = new Promise[Unit]() val c3 = Closable.make(_ => p) - val f4 = Closable.sequence(c1, c3).close() + val f4 = sequenceTwo(c1, c3).close() assert(n1 == 5) assert(!f4.isDefined) p.setDone() @@ -184,7 +212,7 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience test("Closable.sequence lifts synchronous exceptions") { val throwing = Closable.make(_ => sys.error("lolz")) - val f = Closable.sequence(Closable.nop, throwing).close() + val f = sequenceTwo(Closable.nop, throwing).close() val ex = intercept[Exception] { Await.result(f, 5.seconds) } @@ -195,25 +223,17 @@ class ClosableTest extends AnyFunSuite with Eventually with IntegrationPatience assert( Future .value(1) - .map(_ => Closable.sequence(Closable.nop, Closable.nop).close().isDefined) + .map(_ => sequenceTwo(Closable.nop, Closable.nop).close().isDefined) .poll .contains(Return.True) ) } +} - test("Cloasable.make catches NonFatals and translates to failed Futures") { - val throwing = Closable.make { _ => throw new Exception("lolz") } - - var failed = 0 - val f = throwing.close().respond { - case Throw(_) => failed = -1 - case _ => failed = 1 - } +final class ClosableSequence2Test extends ClosableSequenceTest { + def sequenceTwo(a: Closable, b: Closable): Closable = Closable.sequence(a, b) +} - val ex = intercept[Exception] { - Await.result(f, 2.seconds) - } - assert(ex.getMessage.contains("lolz")) - assert(failed == -1) - } +final class ClosableSequenceNTest extends ClosableSequenceTest { + def sequenceTwo(a: Closable, b: Closable): Closable = Closable.sequence(Seq(a, b): _*) } From c39786a4ab3ed9e115fbd617e63adb42f0344a41 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Mon, 13 Dec 2021 19:21:07 +0000 Subject: [PATCH 606/796] util/util-core: skip scheduler on empty waitqueue Problem: Empty waitqueues are submitted to the scheduler. This will happen when a promise with no continuations is satisfied Solution: Skip empty waitqueues Result: Lets also make WaitQueue.Empty an object to make debugging references easier JIRA Issues: CSL-11517 Differential Revision: https://phabricator.twitter.biz/D796346 --- .../main/scala/com/twitter/util/Promise.scala | 9 +++-- .../twitter/concurrent/SchedulerTest.scala | 1 - .../scala/com/twitter/util/PromiseTest.scala | 38 +++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 24521a06a5..3374f52349 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -92,8 +92,11 @@ object Promise { loop(this, WaitQueue.empty) } - final def runInScheduler(t: Try[A]): Unit = - Scheduler.submit(new Runnable() { def run(): Unit = WaitQueue.this.run(t) }) + final def runInScheduler(t: Try[A]): Unit = { + if (this ne WaitQueue.Empty) { + Scheduler.submit(() => WaitQueue.this.run(t)) + } + } @tailrec private def run(t: Try[A]): Unit = @@ -107,7 +110,7 @@ object Promise { private[util] object WaitQueue { - val Empty: WaitQueue[Nothing] = new WaitQueue[Nothing] { + object Empty extends WaitQueue[Nothing] { final def first: K[Nothing] = throw new IllegalStateException("WaitQueue.Empty") diff --git a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala index ee2a5a60c0..523c83b92a 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SchedulerTest.scala @@ -150,7 +150,6 @@ abstract class LocalSchedulerTest(lifo: Boolean) extends AnyFunSuite with Matche val record = sampleBlockingFraction(0.0) assert(record == null) } - } class LocalSchedulerFifoTest extends LocalSchedulerTest(false) diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index e310bbcc25..430399383a 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -1,10 +1,20 @@ package com.twitter.util +import com.twitter.concurrent.Scheduler +import com.twitter.concurrent.LocalScheduler import com.twitter.conversions.DurationOps._ import org.scalatest.funsuite.AnyFunSuite class PromiseTest extends AnyFunSuite { + class TrackingScheduler extends LocalScheduler(false) { + @volatile var submitted: Int = 0 + override def submit(r: Runnable): Unit = { + submitted = submitted + 1 + super.submit(r) + } + } + test("Promise.detach should not detach other attached promises") { val p = new Promise[Unit] val attached1 = Promise.attached(p) @@ -305,4 +315,32 @@ class PromiseTest extends AnyFunSuite { assert("main thread" == Await.result(p2, 3.seconds)) assert("main thread" == Await.result(p, 3.seconds)) } + + test("promise with no continuations is not submitted to the scheduler") { + val old = Scheduler() + try { + val sched = new TrackingScheduler() + Scheduler.setUnsafe(sched) + val p = new Promise[Unit]() + p.setDone() + assert(sched.submitted == 0) + } finally { + Scheduler.setUnsafe(old) + } + } + + test("registered continuation on satisfied promise is submitted to the scheduler") { + val old = Scheduler() + try { + val sched = new TrackingScheduler() + Scheduler.setUnsafe(sched) + val p = new Promise[Unit]() + p.setDone() + + p.ensure(()) + assert(sched.submitted == 1) + } finally { + Scheduler.setUnsafe(old) + } + } } From ffd37d6b56619ec8fd7a1084dab16e39477ae622 Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Wed, 15 Dec 2021 20:17:00 +0000 Subject: [PATCH 607/796] Twitter-oss: Prepare OSS libraries for release 21.12.00 Problem We want to release the next versions of our Twitter OSS libraries 21.12.00 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D802007 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d71354bfd0..31a777aa57 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +21.12.0 +------- + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ * util-core: `Activity.collect*` and `Var.collect*` are now implemented in terms of known collection diff --git a/README.md b/README.md index adff67a751..5c264fd295 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -40,7 +40,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.11.0" +val utilCore = "com.twitter" %% "util-core" % "21.12.0" ``` # Units diff --git a/build.sbt b/build.sbt index ea5e60c789..41cd5742cf 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.12.0-SNAPSHOT" +val releaseVersion = "21.12.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.4" From b05df7ff87f8ceb45a598190c40e574a13de3090 Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Wed, 15 Dec 2021 21:30:00 +0000 Subject: [PATCH 608/796] Twitter-oss: update OSS libraries post-release to 22.1.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.1.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D802418 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c264fd295..6e2145f967 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 41cd5742cf..7b36a4dc5b 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "21.12.0" +val releaseVersion = "22.1.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.4" From eaa0641c151c71454b27e12f0d6ae667f64fa06f Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 16 Dec 2021 19:01:18 +0000 Subject: [PATCH 609/796] csl: Bye Kevin Differential Revision: https://phabricator.twitter.biz/D802469 --- PROJECT.yml | 1 - util-app/PROJECT | 1 - util-benchmark/PROJECT | 1 - util-core/PROJECT | 1 - util-inject/PROJECT | 1 - util-mock/PROJECT | 1 - util-stats/PROJECT | 4 +--- util-zk-test/PROJECT | 1 - 8 files changed, 1 insertion(+), 10 deletions(-) diff --git a/PROJECT.yml b/PROJECT.yml index 6e0ace2668..ec21b4ac20 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -2,7 +2,6 @@ owners: - banderson - - koliver - mnakamura - roanta - vkostyukov diff --git a/util-app/PROJECT b/util-app/PROJECT index fa9fb8e822..a482ca9a20 100644 --- a/util-app/PROJECT +++ b/util-app/PROJECT @@ -3,7 +3,6 @@ owners: - banderson - ibennett - - koliver - mnakamura - roanta - vkostyukov diff --git a/util-benchmark/PROJECT b/util-benchmark/PROJECT index eb8a36aae8..85f4cb0288 100644 --- a/util-benchmark/PROJECT +++ b/util-benchmark/PROJECT @@ -2,7 +2,6 @@ owners: - banderson - - koliver - mnakamura - roanta - vkostyukov diff --git a/util-core/PROJECT b/util-core/PROJECT index f6d911c574..79d52f717c 100644 --- a/util-core/PROJECT +++ b/util-core/PROJECT @@ -3,7 +3,6 @@ owners: - banderson - hallam - - koliver - mnakamura - roanta - vkostyukov diff --git a/util-inject/PROJECT b/util-inject/PROJECT index 348b67af27..fbed8cad0c 100644 --- a/util-inject/PROJECT +++ b/util-inject/PROJECT @@ -1,7 +1,6 @@ owners: - banderson - ibennett - - koliver - mnakamura - roanta - vkostyukov diff --git a/util-mock/PROJECT b/util-mock/PROJECT index 348b67af27..fbed8cad0c 100644 --- a/util-mock/PROJECT +++ b/util-mock/PROJECT @@ -1,7 +1,6 @@ owners: - banderson - ibennett - - koliver - mnakamura - roanta - vkostyukov diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 0ebef1dd2f..51de3ca05d 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -3,9 +3,7 @@ owners: - banderson - beisenman - - koliver - - mdannenberg - mnakamura - roanta - vkostyukov - - yufang \ No newline at end of file + - yufang diff --git a/util-zk-test/PROJECT b/util-zk-test/PROJECT index f7f3d2b311..a4d97bddf7 100644 --- a/util-zk-test/PROJECT +++ b/util-zk-test/PROJECT @@ -1,7 +1,6 @@ owners: - coord-review:ldap - banderson - - koliver - mnakamura - roanta - vkostyukov From 78e3aa30d065f5b29403fa49d47404b9d7a5945c Mon Sep 17 00:00:00 2001 From: Gary Grossman Date: Fri, 17 Dec 2021 20:02:22 +0000 Subject: [PATCH 610/796] finatra | util: Update json4s version to 4.0.3 and factor into val **Problem** The latest version of json4s is 4.0.3. finatra depends on json4s 3.6.11. util-jackson depends on json4s 3.6.11, and util-validator depends on json4s 3.6.7. (Seems like an accident that these are different.) Other popular libraries, such as jwt-scala, now depend on json4s 4.0.3. This creates binary compatibility problems when Twitter Util is used in conjunction with these libraries: ``` [error] * org.json4s:json4s-core_2.12:4.0.3 (early-semver) is selected over {3.6.11, 3.6.11, 3.6.11, 3.6.7} [error] +- com.github.jwt-scala:jwt-json4s-common_2.12:9.0.2 (depends on 4.0.3) [error] +- com.twitter:util-validator_2.12:21.11.0 (depends on 3.6.7) [error] +- com.twitter:util-jackson_2.12:21.11.0 (depends on 3.6.11) [error] +- com.twitter:finatra-validation_2.12:21.11.0 (depends on 3.6.11) [error] +- com.twitter:finatra-jackson_2.12:21.11.0 (depends on 3.6.11) [error] ``` **Solution** Twitter Util should bump to json4s 4.0.3. I know that all libraries in the Twitter Stack bump these dependencies at the same time, so I have also provided a PR for Finatra. bijection is already on json4s 4.0.3. scrooge and finagle do not depends on json4s. sbt test succeeds with json4s 4.0.3 and no other code changes. I took the liberty of creating a val json4sVersion instead of having the version number separately in util-jackson and util-validator... this should help to keep them in sync in the future. **Result** Closes twitter/util#297 Closes twitter/finatra#572 Signed-off-by: Joy Bestourous Differential Revision: https://phabricator.twitter.biz/D803847 --- build.sbt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 7b36a4dc5b..5b8b1c0c46 100644 --- a/build.sbt +++ b/build.sbt @@ -8,6 +8,7 @@ val releaseVersion = "22.1.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.11.4" +val json4sVersion = "4.0.3" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" @@ -505,7 +506,7 @@ lazy val utilJackson = Project( "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion, "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava"), "jakarta.validation" % "jakarta.validation-api" % "3.0.0", - "org.json4s" %% "json4s-core" % "3.6.11", + "org.json4s" %% "json4s-core" % json4sVersion, "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion % "test", scalacheckLib, "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", @@ -725,7 +726,7 @@ lazy val utilValidator = Project( "jakarta.validation" % "jakarta.validation-api" % "3.0.0", "org.hibernate.validator" % "hibernate-validator" % "7.0.1.Final", "org.glassfish" % "jakarta.el" % "4.0.0", - "org.json4s" %% "json4s-core" % "3.6.7", + "org.json4s" %% "json4s-core" % json4sVersion, "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion % "test", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" From d9e680fd274a4114286ad716c674db834ffe77e1 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 10 Jan 2022 19:50:39 +0000 Subject: [PATCH 611/796] util-core: Tiny closable optimization Problem Most of the time, Closable combinators are dealing with const futures (`Future.Done`) so we're paying the price of `Some` wrapper on each `ConstFuture.poll`. Solution Compare against `Future.Done` instead of polling the Future. JIRA Issues: CSL-11377 Differential Revision: https://phabricator.twitter.biz/D808640 --- .../main/scala/com/twitter/util/Closable.scala | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Closable.scala b/util-core/src/main/scala/com/twitter/util/Closable.scala index f365715e0a..c28fd0fd40 100644 --- a/util-core/src/main/scala/com/twitter/util/Closable.scala +++ b/util-core/src/main/scala/com/twitter/util/Closable.scala @@ -94,8 +94,8 @@ object Closable { def checkNext(): Future[Unit] = { if (!iter.hasNext) Future.Done else { - iter.next().poll match { - case Some(Return(_)) => checkNext() + iter.next() match { + case Future.Done => checkNext() case _ => Future.join(fs) } } @@ -125,10 +125,8 @@ object Closable { else second } - first.poll match { - case Some(t) => onFirstComplete(t) - case None => first.transform(onFirstComplete) - } + if (first eq Future.Done) onFirstComplete(Try.Unit) + else first.transform(onFirstComplete) } } @@ -170,10 +168,8 @@ object Closable { val firstClose = safeClose(hd, deadline) // eagerness is needed by `Var`, see RB_ID=557464 - firstClose.poll match { - case Some(t) => onTry(t) - case None => firstClose.transform(onTry) - } + if (firstClose eq Future.Done) onTry(Try.Unit) + else firstClose.transform(onTry) } def close(deadline: Time): Future[Unit] = From a01c937a76f54990e2d4face89d194efac9316fd Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 10 Jan 2022 23:40:54 +0000 Subject: [PATCH 612/796] util-core: Optimize UpdatableVar for lower allocations Problem UpdatableVar is quite heavy on allocations every time it's updated / observer / closed. Solution Inline state under UpdatableVar, make state mutable and use plain old synchronization to avoid atomic wrappers. Also, replace SortedSet (backed by RBT) with a hand-rolled implementation based off of linked list. JIRA Issues: CSL-11377 Differential Revision: https://phabricator.twitter.biz/D797868 --- .../src/main/scala/com/twitter/util/Var.scala | 127 ++++++++++++------ .../test/scala/com/twitter/util/VarTest.scala | 34 +++++ 2 files changed, 122 insertions(+), 39 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 5f48be10ea..ef16e3e9b9 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -1,13 +1,11 @@ package com.twitter.util -import java.util.concurrent.atomic.AtomicLong -import java.util.concurrent.atomic.AtomicReference import java.util.{List => JList} import scala.annotation.tailrec import scala.collection.{Seq => AnySeq} import scala.collection.compat.immutable.ArraySeq -import scala.collection.immutable import scala.collection.mutable.Buffer +import scala.collection.mutable.ListBuffer import scala.jdk.CollectionConverters._ import scala.language.higherKinds @@ -474,72 +472,123 @@ object Var { private object UpdatableVar { import Var.Observer - case class Party[T](obs: Observer[T], depth: Int, n: Long) { + final case class Party[T](observer: Observer[T], depth: Int, sequence: Long) { @volatile var active: Boolean = true } - case class State[T](value: T, version: Long, parties: immutable.SortedSet[Party[T]]) { - def -(p: Party[T]): State[T] = copy(parties = parties - p) - def +(p: Party[T]): State[T] = copy(parties = parties + p) - def :=(newv: T): State[T] = copy(value = newv, version = version + 1) - } - - private val anyOrder: Ordering[Party[Any]] = new Ordering[Party[Any]] { - // This is safe because observers are compared - // only from the same counter. - def compare(a: Party[Any], b: Party[Any]): Int = { + val partyOrder: Ordering[Party[_]] = new Ordering[Party[_]] { + def compare(a: Party[_], b: Party[_]): Int = { val c1 = java.lang.Integer.compare(a.depth, b.depth) if (c1 != 0) return c1 - java.lang.Long.compare(a.n, b.n) + java.lang.Long.compare(a.sequence, b.sequence) } } - - implicit def order[T]: Ordering[Party[T]] = anyOrder.asInstanceOf[Ordering[Party[T]]] } private[util] class UpdatableVar[T](init: T) extends Var[T] with Updatable[T] with Extractable[T] { import UpdatableVar._ import Var.Observer - private[this] val n = new AtomicLong(0) - private[this] val state = new AtomicReference(State[T](init, 0, immutable.SortedSet.empty)) - - @tailrec - private[this] def cas(next: State[T] => State[T]): State[T] = { - val from = state.get - val to = next(from) - if (state.compareAndSet(from, to)) to else cas(next) - } - - def apply(): T = state.get.value + // The state must be mutated or read under a synchronized block on 'this'. The value can be read + // w/o an external synchronization. + @volatile private[this] var value = init + private[this] var version = 0L + private[this] var partySequence = 0L + // Exposed for testing. + // + // Parties is a sorted set. We implement it ourselves (out top of vanilla list) at the cost + // of O(n) insert/remove in order to save on memory footprint. Similar to Future's WaitQueue, + // there just a handful of observers registering to each Var so plain old linked list works + // the best. + @volatile private[util] var parties = List.empty[Party[T]] + + def apply(): T = value + + def update(newValue: T): Unit = { + val v = synchronized { + version += 1 + value = newValue + version + } - def update(newv: T): Unit = synchronized { - val State(value, version, parties) = cas(_ := newv) - for (p @ Party(obs, _, _) <- parties) { + // Another party maybe be racing to register while we're updating with the new value. + // This is not a big deal b/c observers prevent double-updates by design (via versions). + parties.foreach { p => // An antecedent update may have closed the current // party (e.g. flatMap does this); we need to check that // the party is active here in order to prevent stale updates. - if (p.active) - obs.publish(this, value, version) + if (p.active) p.observer.publish(this, newValue, v) } } protected def observe(depth: Int, obs: Observer[T]): Closable = { obs.claim(this) - val party = Party(obs, depth, n.getAndIncrement()) - val State(value, version, _) = cas(_ + party) - obs.publish(this, value, version) + + val (p, curValue, curVersion) = synchronized { + val party = Party(obs, depth, partySequence) + insertParty(party) + partySequence += 1 + (party, value, version) + } + + obs.publish(this, curValue, curVersion) new Closable { def close(deadline: Time): Future[Unit] = { - party.active = false - cas(_ - party) + p.active = false + UpdatableVar.this.synchronized { + removeParty(p) + } Future.Done } } } - override def toString: String = "Var(" + state.get.value + ")@" + hashCode + // Must be called while synchronized on this. + def removeParty(p: Party[T]): Unit = { + @tailrec + def loop(current: List[Party[T]], result: ListBuffer[Party[T]]): List[Party[T]] = + current match { + case Nil => result.toList + case q :: next => + if (q eq p) (result ++= next).toList + else loop(next, result += q) + } + + parties = parties match { + case q :: next if q eq p => next + case ps => loop(ps, ListBuffer.empty) + } + } + + // Must be called while synchronized on this. + def insertParty(p: Party[T]): Unit = { + @tailrec + def loop(current: List[Party[T]], result: ListBuffer[Party[T]]): List[Party[T]] = + current match { + case Nil => (result += p).toList + case q :: next => + val order = partyOrder.compare(p, q) + if (order < 0) { + // p < q, insert here + result += p + (result ++= current).toList + } else if (order > 0) { + // p > q, the insert position is not found yet + loop(next, result += q) + } else { + // p == q, ignore p and return the original queue + parties + } + } + + parties = parties match { + case Nil => p :: Nil + case ps => loop(ps, ListBuffer.empty) + } + } + + override def toString: String = "Var(" + value + ")@" + hashCode } /** diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 3bf1c3796a..01d6c1b487 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -1,7 +1,9 @@ package com.twitter.util +import com.twitter.util.UpdatableVar.Party import java.util.concurrent.atomic.AtomicReference import org.scalacheck.Arbitrary.arbitrary +import org.scalacheck.Gen import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable import org.scalatest.funsuite.AnyFunSuite @@ -482,4 +484,36 @@ class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { } } } + + test("Hand-rolled observers queue behaves exactly like SortedSet") { + implicit val anyOrder: Ordering[Party[Any]] = + UpdatableVar.partyOrder.asInstanceOf[Ordering[Party[Any]]] + + val arbParties = for { + n <- Gen.choose(1, 100) + depths <- Gen.listOfN(n, Gen.posNum[Int]) + seqs <- Gen.listOfN(n, Gen.posNum[Int]) + } yield { + depths.zip(seqs).map { + case (d, s) => Party[Any](null, d, s) + } + } + + forAll(arbParties) { parties => + var expected = scala.collection.immutable.SortedSet.empty[Party[Any]] + expected ++= parties + + val v = new UpdatableVar[Any](null) + parties.foreach(p => v.insertParty(p)) + + // test insert + assert(expected.toList == v.parties) + + expected -= parties.head + v.removeParty(parties.head) + + // test remove + assert(expected.toList == v.parties) + } + } } From 4ddeaf899564a895d92d77fe7919a46ad7c100a3 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 12 Jan 2022 18:43:58 +0000 Subject: [PATCH 613/796] 3rdparty: Revert "Revert "3rdparty: jackson upgrade to 2.13.0"" & Upgrade to 2.13.1 Problem We're on an old version of jackson, and we'd like to upgrade to a new version. We tried in the past to upgrade to 2.13.0, but it introduced a regression, where they were double-serializing. We believe 2.13.1 no longer has that regression, so it should be safe to upgrade. Solution This reverts commit e27353b784543196b9f5fd572110e4b4fe04f4c4. JIRA Issues: CSL-11573 Differential Revision: https://phabricator.twitter.biz/D808049 --- CHANGELOG.rst | 12 ++++++++++++ build.sbt | 2 +- .../twitter/util/jackson/ScalaObjectMapperTest.scala | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 31a777aa57..1777e84aa8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,11 +7,22 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-jackson: The error message when failing to deserialize a character now correctly prints the non-character string. ``PHAB_ID=D808049`` + +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Bump version of Jackson to 2.13.1. ``PHAB_ID=D808049`` + 21.12.0 ------- Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ + * util-core: `Activity.collect*` and `Var.collect*` are now implemented in terms of known collection type `scala.collection.Seq` versus HKT `CC[X]` before. This allows for certain performance enhancements as well as makes it more aligned with the `Future.collect` APIs. @@ -22,6 +33,7 @@ Breaking API Changes Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ + * util-security: Use snakeyaml to parse yaml instead of a buggy custom yaml parser. This means that thrown IOExceptions have been replaced by YAMLExceptions. Additionally, the parser member has been limited to private visibility. ``PHAB_ID=D617641`` diff --git a/build.sbt b/build.sbt index 5b8b1c0c46..c446b73c5e 100644 --- a/build.sbt +++ b/build.sbt @@ -7,7 +7,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "22.1.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.11.4" +val jacksonVersion = "2.13.1" val json4sVersion = "4.0.3" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala index 5a0579a515..0855c4daf5 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala @@ -823,7 +823,7 @@ class ScalaObjectMapperTest |} """.stripMargin, withErrors = Seq( - "c: '' is not a valid Character" + "c: '-1' is not a valid Character" ) ) } From 68c5a8adba471ab03b05ffcfba738972e671cc61 Mon Sep 17 00:00:00 2001 From: Gary Grossman Date: Wed, 12 Jan 2022 21:20:38 +0000 Subject: [PATCH 614/796] {util,scrooge,twitter-server,finagle,finatra}: Upgrade sbt to 1.6.1 Problem / Solution Update to sbt 1.6.1 Signed-off-by: Moses Nakamura JIRA Issues: CSL-11546 Differential Revision: https://phabricator.twitter.biz/D810355 --- .github/workflows/continuous-integration.yml | 2 +- project/build.properties | 2 +- sbt | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 2908f5390b..8a50a1be8a 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -3,7 +3,7 @@ name: continuous integration env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI - SBT_VERSION: 1.5.5 + SBT_VERSION: 1.6.1 BUILD_VERSION: "v1" # bump this if builds are failing due to a bad cache defaults: diff --git a/project/build.properties b/project/build.properties index 10fd9eee04..3161d2146c 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.5.5 +sbt.version=1.6.1 diff --git a/sbt b/sbt index dd557c21d8..1d62b30232 100755 --- a/sbt +++ b/sbt @@ -2,9 +2,9 @@ set -eo pipefail -sbtver="1.5.5" +sbtver="1.6.1" sbtjar="sbt-launch-$sbtver.jar" -sbtsha128="3beb23e861de5021bd80ca52113b075321580265" +sbtsha128="b69ab7f114d09d859b5e349acdb05d32c9d49072" sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From cdf573eaec382c3c076c82692f676c808d806b6d Mon Sep 17 00:00:00 2001 From: Kyle Bahr Date: Wed, 12 Jan 2022 23:01:18 +0000 Subject: [PATCH 615/796] Twitter-oss: Prepare OSS libraries for release 22.1.0 Problem We want to release the next versions of our Twitter OSS libraries 22.1.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11595 Differential Revision: https://phabricator.twitter.biz/D812047 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1777e84aa8..7e8789bddc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.1.0 +------ + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 6e2145f967..f51b1a0353 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -40,7 +40,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "21.12.0" +val utilCore = "com.twitter" %% "util-core" % "22.1.0" ``` # Units diff --git a/build.sbt b/build.sbt index c446b73c5e..29301fa579 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.1.0-SNAPSHOT" +val releaseVersion = "22.1.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.1" From 2211f75bf5c9384326e9d289fea4caa6ed82df38 Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Wed, 12 Jan 2022 23:50:57 +0000 Subject: [PATCH 616/796] util-core: Return suppressed exception information in Closable.sequence Problem Closable.sequence returns the first failed Future[Unit] result of closing the Closables, and in the case of multiple failures, the subsequent exceptions are lost. Solution Add the subsequent failures to the list of suppressed exceptions attached to the first failed Future[Unit] result. Result Users will be able to see all failures in the case that there are multiple failures in Closable.sequence. JIRA Issues: CSL-11522 Differential Revision: https://phabricator.twitter.biz/D809561 --- CHANGELOG.rst | 2 ++ .../scala/com/twitter/util/Closable.scala | 29 ++++++++++++++----- .../scala/com/twitter/util/ClosableTest.scala | 3 +- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7e8789bddc..42b3796324 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -20,6 +20,8 @@ Runtime Behavior Changes * util: Bump version of Jackson to 2.13.1. ``PHAB_ID=D808049`` +* util-core: Return suppressed exception information in Closable.sequence ``PHAB_ID=D809561`` + 21.12.0 ------- diff --git a/util-core/src/main/scala/com/twitter/util/Closable.scala b/util-core/src/main/scala/com/twitter/util/Closable.scala index c28fd0fd40..39388fa647 100644 --- a/util-core/src/main/scala/com/twitter/util/Closable.scala +++ b/util-core/src/main/scala/com/twitter/util/Closable.scala @@ -110,7 +110,8 @@ object Closable { * resource ''n+1'' is not closed until resource ''n'' is. * * @return the first failed [[Future]] should any of the `Closables` - * result in a failed [[Future]]. + * result in a failed [[Future]]. Any subsequent failures are + * included as suppressed exceptions. * * @note as with all `Closables`, the `deadline` passed to `close` * is advisory. @@ -121,8 +122,17 @@ object Closable { def onFirstComplete(t: Try[Unit]): Future[Unit] = { val second = safeClose(b, deadline) - if (t.isThrow) if (second.isDefined) first else second.transform(_ => first) - else second + if (t.isThrow) { + if (second eq Future.Done) first + else { + second.transform { + case Return(_) => first + case Throw(secondThrow) => + t.throwable.addSuppressed(secondThrow) + first + } + } + } else second } if (first eq Future.Done) onFirstComplete(Try.Unit) @@ -136,7 +146,8 @@ object Closable { * resource ''n+1'' is not closed until resource ''n'' is. * * @return the first failed [[Future]] should any of the `Closables` - * result in a failed [[Future]]. + * result in a failed [[Future]]. Any subsequent failures are + * included as suppressed exceptions. * * @note as with all `Closables`, the `deadline` passed to `close` * is advisory. @@ -146,11 +157,11 @@ object Closable { private final def closeSeq( deadline: Time, closables: Seq[Closable], - firstFailure: Option[Future[Unit]] + firstFailure: Option[Throw[Unit]] ): Future[Unit] = closables match { case Seq() => firstFailure match { - case Some(f) => f + case Some(t) => Future.const(t) case None => Future.Done } case Seq(hd, tl @ _*) => @@ -159,8 +170,10 @@ object Closable { case Return(_) => firstFailure case t @ Throw(_) => firstFailure match { - case Some(_) => firstFailure - case None => Some(Future.const(t)) + case Some(firstThrow) => + firstThrow.throwable.addSuppressed(t.throwable) + firstFailure + case None => Some(t) } } closeSeq(deadline, tl, failure) diff --git a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala index bed043c65b..20c4cd3d41 100644 --- a/util-core/src/test/scala/com/twitter/util/ClosableTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ClosableTest.scala @@ -188,13 +188,14 @@ abstract class ClosableSequenceTest extends AnyFunSuite with Eventually with Int } assert(x2.getMessage.contains("n1=2")) - // multiple failures, returns the first failure + // multiple failures, returns the first failure, and a list of suppressed failures. val f3 = sequenceTwo(c1, c1).close() assert(n1 == 4) val x3 = intercept[RuntimeException] { Await.result(f3, 5.seconds) } assert(x3.getMessage.contains("n1=3")) + assert(x3.getSuppressed.head.getMessage.contains("n1=4")) // verify that first failure is returned, but only after the // last closable is satisfied From 5b61b8452bbbdcac95dd0e9eb019a01736246262 Mon Sep 17 00:00:00 2001 From: Kyle Bahr Date: Thu, 13 Jan 2022 19:56:30 +0000 Subject: [PATCH 617/796] Twitter-oss: update OSS libraries post-release to 22.2.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11595 Differential Revision: https://phabricator.twitter.biz/D812865 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f51b1a0353..a1bba1be72 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 29301fa579..882c090ea1 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.1.0" +val releaseVersion = "22.2.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.1" From ee8015f429546927287a088d5313a513f990b229 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Wed, 19 Jan 2022 04:50:43 +0000 Subject: [PATCH 618/796] util-core: Optimize Closable.all for lower allocations Problem / Solution There is no need to create an intermediate sequence of Future[Unit] to hold results of Closable.all, we can just keep closing them one by one, until non-Future.Done future is observed (should be very rare). JIRA Issues: CSL-11377 Differential Revision: https://phabricator.twitter.biz/D814733 --- .../main/scala/com/twitter/util/Closable.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Closable.scala b/util-core/src/main/scala/com/twitter/util/Closable.scala index 39388fa647..5b75b52302 100644 --- a/util-core/src/main/scala/com/twitter/util/Closable.scala +++ b/util-core/src/main/scala/com/twitter/util/Closable.scala @@ -88,19 +88,19 @@ object Closable { */ @varargs def all(closables: Closable*): Closable = new Closable { def close(deadline: Time): Future[Unit] = { - val fs = closables.map { closable => safeClose(closable, deadline) } - val iter = fs.iterator + val iter = closables.iterator @tailrec - def checkNext(): Future[Unit] = { - if (!iter.hasNext) Future.Done + def checkNext(async: List[Future[Unit]]): List[Future[Unit]] = { + if (!iter.hasNext) async else { - iter.next() match { - case Future.Done => checkNext() - case _ => Future.join(fs) + safeClose(iter.next(), deadline) match { + case Future.Done => checkNext(async) + case f => checkNext(f :: async) } } } - checkNext() + + Future.join(checkNext(Nil)) } } From 07b5c2ef2f7843cc1c8e3cf7b017a98ee1f0d0bc Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Wed, 19 Jan 2022 23:04:13 +0000 Subject: [PATCH 619/796] csl: Update Caffeine Cache library to 2.9.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem/Solution We want to keep our dependencies up to date for the ben-manes Caffeine library. Let’s update to version 2.9.3. JIRA Issues: CSL-11548 Differential Revision: https://phabricator.twitter.biz/D815761 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 42b3796324..07dbf72098 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Bump version of Caffeine to 2.9.3. ``PHAB_ID=D815761`` + 22.1.0 ------ diff --git a/build.sbt b/build.sbt index 882c090ea1..9eb84ec3d8 100644 --- a/build.sbt +++ b/build.sbt @@ -22,7 +22,7 @@ val zkDependency = "org.apache.zookeeper" % "zookeeper" % zkVersion excludeAll ( ) val guavaLib = "com.google.guava" % "guava" % "25.1-jre" -val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.9.2" +val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.9.3" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.15.4" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion From 62ddb5010885a1d244813bba089452ff01a1cf5e Mon Sep 17 00:00:00 2001 From: Flavio Brasil Date: Wed, 26 Jan 2022 02:26:13 +0000 Subject: [PATCH 620/796] [util-core][finagle-core][fiber-scheduler] introduce withMaxSyncConcurrency **Problem** When a `FuturePool` is redirected to a `ForkingScheduler`, the characteristics of the original thread pool are lost. The synchronous concurrency limit imposed be the number of threads and the queue size aren't used. **Solution** Introduce a new method `withMaxSyncConcurrency`, which is used to create a new `ForkingScheduler` wrapper with bounded concurrency and queue. Differential Revision: https://phabricator.twitter.biz/D791343 --- .../twitter/concurrent/ForkingScheduler.scala | 20 +++++++ .../scala/com/twitter/util/FuturePool.scala | 9 +++- .../com/twitter/util/FuturePoolTest.scala | 53 +++++++++++++++++-- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala index f0625e807a..467489178d 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/ForkingScheduler.scala @@ -46,6 +46,26 @@ trait ForkingScheduler extends Scheduler { */ def fork[T](executor: Executor)(f: => Future[T]): Future[T] + /** + * Creates a wrapper of this scheduler with a fixed max + * synchronous concurrency. The concurrency limit is + * applied to synchronous execution so tasks can be + * running in parallel if they're suspended at asynchronous + * boundaries but once they are scheduled for execution + * only `concurrency` tasks can be running in parallel. + * + * The original forking scheduler remains + * unchanged and is used as the underlying implementation + * of the returned scheduler. + * + * @param concurrency the max concurrency + * @param maxWaiters max number of pending tasks. When this limit + * is reached, the scheduler starts to return + * `RejectedExecutionException`s + * @return the wrapper with a fixed max concurrency + */ + def withMaxSyncConcurrency(concurrency: Int, maxWaiters: Int): ForkingScheduler + /** * Creates an `ExecutorService` wrapper for this forking * scheduler. The original forking scheduler remains diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index dd53b758c8..f3f9d20715 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -139,7 +139,14 @@ class ExecutorServiceFuturePool protected[this] ( private[this] lazy val forkingScheduler: ForkingScheduler = { ForkingScheduler() .filter(_.redirectFuturePools()) - .getOrElse(null) + .map { s => + executor match { + case e: ThreadPoolExecutor => + s.withMaxSyncConcurrency(e.getMaximumPoolSize, e.getQueue.remainingCapacity()) + case _ => + s + } + }.getOrElse(null) } def apply[T](f: => T): Future[T] = { diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index 382d8231d2..722c08b8c0 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -238,6 +238,8 @@ class FuturePoolTest extends AnyFunSuite with Eventually { } class TestForkingScheduler(redirect: Boolean) extends ForkingScheduler { + var concurrency = Option.empty[Int] + var maxWaiters = Option.empty[Int] var forked = Option.empty[Future[_]] override def fork[T](f: => Future[T]) = { val r = f @@ -248,6 +250,11 @@ class FuturePoolTest extends AnyFunSuite with Eventually { override def redirectFuturePools(): Boolean = redirect override def blocking[T](f: => T)(implicit perm: Awaitable.CanAwait): T = f override def flush(): Unit = {} + override def withMaxSyncConcurrency(v: Int, w: Int): ForkingScheduler = { + concurrency = Some(v) + maxWaiters = Some(w) + this + } override def submit(r: Runnable): Unit = r.run() @@ -256,10 +263,12 @@ class FuturePoolTest extends AnyFunSuite with Eventually { override def numDispatches: Long = ??? } - test("FuturePool should be redirected to the forking scheduler if specified by the scheduler") { + test( + "FuturePool should be redirected to the forking scheduler if specified by the scheduler - unbounded queue") { + val concurrency = 1 val scheduler = new TestForkingScheduler(redirect = true) val prevScheduler = Scheduler() - val executor = Executors.newFixedThreadPool(1) + val executor = Executors.newFixedThreadPool(concurrency) Scheduler.setUnsafe(scheduler) try { val pool = FuturePool(executor) @@ -271,6 +280,41 @@ class FuturePoolTest extends AnyFunSuite with Eventually { } assert(Await.result(result, 1.second) == 1) assert(Await.result(scheduler.forked.get, 1.second) == 1) + assert(scheduler.concurrency == Some(concurrency)) + assert(scheduler.maxWaiters == Some(Int.MaxValue)) + assert(executed) + } finally { + Scheduler.setUnsafe(prevScheduler) + executor.shutdown() + } + } + + test( + "FuturePool should be redirected to the forking scheduler if specified by the scheduler - bounded queue") { + val concurrency = 1 + val maxWaiters = 10 + val scheduler = new TestForkingScheduler(redirect = true) + val prevScheduler = Scheduler() + val executor = + new ThreadPoolExecutor( + concurrency, + concurrency, + 0L, + TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(maxWaiters)) + Scheduler.setUnsafe(scheduler) + try { + val pool = FuturePool(executor) + var executed = false + val result = + pool { + executed = true + 1 + } + assert(Await.result(result, 1.second) == 1) + assert(Await.result(scheduler.forked.get, 1.second) == 1) + assert(scheduler.concurrency == Some(concurrency)) + assert(scheduler.maxWaiters == Some(maxWaiters)) assert(executed) } finally { Scheduler.setUnsafe(prevScheduler) @@ -280,8 +324,9 @@ class FuturePoolTest extends AnyFunSuite with Eventually { test( "FuturePool should not be redirected to the forking scheduler if specified by the scheduler") { + val concurrency = 1 val scheduler = new TestForkingScheduler(redirect = false) - val executor = Executors.newFixedThreadPool(1) + val executor = Executors.newFixedThreadPool(concurrency) val prevScheduler = Scheduler() Scheduler.setUnsafe(scheduler) try { @@ -294,6 +339,8 @@ class FuturePoolTest extends AnyFunSuite with Eventually { } assert(Await.result(result, 1.second) == 1) assert(scheduler.forked.isEmpty) + assert(scheduler.concurrency.isEmpty) + assert(scheduler.maxWaiters.isEmpty) assert(executed) } finally { Scheduler.setUnsafe(prevScheduler) From cd99f16542959d926d43a5f5b498c2627038664b Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 28 Jan 2022 00:15:00 +0000 Subject: [PATCH 621/796] util/util-slf4j-api: API updates and deprecation removal Problem The current usage pattern that Scala users should prefer using the `Logging` trait is a bit flawed as it doesn't account for the very common and correct case of creating a Logger once in a companion object for use in a companion class. This would mirror the Java best practice of defining a logger instance as a static member of a Java class. This is generally preferred as a best practice because there is typically no reason to instantiate a new logger instance every time the class it is defined in is instantiated. Related, since loggers are named after the class passed to the constructor, the API doesn't currently attempt to normalize the name of the logger when given a class that represents a Scala singleton object (see: https://docs.scala-lang.org/tour/singleton-objects.html) meaning that a logger created in a companion object can end up with the trailing `$` symbol which Scala uses for singleton object class names. Also, the Logger functions were not optimized in the same way as the Logging trait (exposing call-by-name versions) since Scala users were meant to prefer the Logging trait which did have these variations whereas Java users could prefer the Logger functions directly. These variations should also exist on the Logger for Scala users. Additionally, `c.t.util.logging.Loggers` was deprecated some time ago as it is no longer necessary for bridging the Scala object constructors for Java users. So let's finally remove it. Solution Add `call-by-name` method variations to the Logger to provide the same behavior available in the Logging trait. Normalize a logger name when a Scala singleton object class is passed to drop the `$` suffix. Remove the deprecated `Loggers` class as Java users can use the apply methods in `Logger` now directly without additional ceremony. Result More utility for Scala users for the Logger. Logger names will be more consistent across a singleton object or a class. And reduction of tech debt through the removal of deprecated code. Differential Revision: https://phabricator.twitter.biz/D820252 --- CHANGELOG.rst | 7 + .../com/twitter/util/logging/Logger.scala | 268 +++++++++++++++++- .../com/twitter/util/logging/Loggers.scala | 53 ---- .../com/twitter/util/logging/Logging.scala | 5 +- .../util/logging/LoggersCompilationTest.java | 45 --- .../twitter/util/logging/LoggingTest.scala | 2 +- .../util/logging/MarkerLoggingTest.scala | 2 +- 7 files changed, 274 insertions(+), 108 deletions(-) delete mode 100644 util-slf4j-api/src/main/scala/com/twitter/util/logging/Loggers.scala delete mode 100644 util-slf4j-api/src/test/java/com/twitter/util/logging/LoggersCompilationTest.java diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 07dbf72098..82a788d1a2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,13 @@ Unreleased Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ +* util-slf4j-api: Update the `Logger` API to include "call-by-name" method + variations akin to the `Logging` trait. When creating a `Logger` from a + Scala singleton object class, the resultant logger name will no longer include + the `$` suffix. Remove the deprecated `Loggers` object which is no longer + needed for Java compatibility as users can now directly use the `Logger` + apply functions with no additional ceremony. ``PHAB_ID=D820252`` + * util: Bump version of Caffeine to 2.9.3. ``PHAB_ID=D815761`` 22.1.0 diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala index 295f2e8ca5..05d63455e0 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logger.scala @@ -1,8 +1,10 @@ package com.twitter.util.logging import org.slf4j -import org.slf4j.{LoggerFactory, Marker} -import scala.reflect.{ClassTag, classTag} +import org.slf4j.LoggerFactory +import org.slf4j.Marker +import scala.reflect.ClassTag +import scala.reflect.classTag /** * Companion object for [[com.twitter.util.logging.Logger]] which provides @@ -56,14 +58,18 @@ object Logger { /** * Create a [[com.twitter.util.logging.Logger]] named for the given class. * @param clazz class to use for naming the underlying Logger. - * @see Java users see $GetLoggerClassScaldocLink + * @note Scala singleton object classes are converted to loggers named without the `$` suffix. + * @see Java users see $GetLoggerClassScaladocLink + * @see [[https://docs.scala-lang.org/tour/singleton-objects.html]] * * {{{ * val logger = Logger(classOf[MyClass]) * }}} */ def apply(clazz: Class[_]): Logger = { - new Logger(LoggerFactory.getLogger(clazz)) + if (isSingletonObject(clazz)) + new Logger(LoggerFactory.getLogger(objectClazzName(clazz))) + else new Logger(LoggerFactory.getLogger(clazz)) } /** @@ -82,13 +88,18 @@ object Logger { * Create a [[com.twitter.util.logging.Logger]] for the runtime class wrapped * by the implicit `scala.reflect.ClassTag` denoting the runtime class `T`. * @tparam T the runtime class type + * @note Scala singleton object classes are converted to loggers named without the `$` suffix. + * @see [[https://docs.scala-lang.org/tour/singleton-objects.html]] * * {{{ * val logger = Logger[MyClass] * }}} */ def apply[T: ClassTag]: Logger = { - new Logger(LoggerFactory.getLogger(classTag[T].runtimeClass)) + val clazz = classTag[T].runtimeClass + if (isSingletonObject(clazz)) + new Logger(LoggerFactory.getLogger(objectClazzName(clazz))) + else new Logger(LoggerFactory.getLogger(clazz)) } /** @@ -117,6 +128,34 @@ object Logger { */ def getLogger(underlying: slf4j.Logger): Logger = Logger(underlying) + + /** + * Returns true if the given instance is a Scala singleton object, false otherwise. + * EXPOSED FOR TESTING + * @see [[https://docs.scala-lang.org/tour/singleton-objects.html]] + */ + private[logging] def isSingletonObject(clazz: Class[_]): Boolean = { + // While Scala's reflection utilities offer this in a succinct and convenient + // method, the startup costs are quite high and in testing, the results not always + // accurate. The approach used here, while fragile to Scala internal changes, was + // deemed worth the tradeoff. This assumes that singleton objects have class names that + // end with $ and have a field on them called MODULE$. For example `object Foo` + // has the class name `Foo` and a `MODULE$` field. + // This works as of Scala 2.12, 2.13 and 3.0.2-RC1. + val className = clazz.getName + if (!className.endsWith("$")) + return false + + try { + clazz.getField("MODULE$") // this throws if the field doesn't exist + true + } catch { + case _: NoSuchFieldException => false + } + } + + // EXPOSED FOR TESTING + private[logging] def objectClazzName(clazz: Class[_]) = clazz.getName.stripSuffix("$") } /** @@ -126,6 +165,8 @@ object Logger { * to support it's usage through the [[com.twitter.util.logging.Logging]] trait when the trait is mixed * into a [[https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html java.io.Serializable]] class. * + * Note, logging methods have a call-by-name variation which are intended for use from Scala. + * * @define isLevelEnabled * * Determines if the named log level is enabled. Returns `true` if enabled, `false` otherwise. @@ -153,6 +194,18 @@ object Logger { * * Log the given parameterized message at the named log level using the given * args taking into consideration the given `org.slf4j.Marker` data. See [[https://www.slf4j.org/faq.html#logging_performance Parameterized Message Logging]] + * + * @define logResult + * + * Log the given message at the named log level formatted with the result of the + * passed in function using the underlying logger. The incoming string message should + * contain a single `%s` which will be replaced with the result[T] of the given function. + * + * Example: + * + * {{{ + * infoResult("The answer is: %s") {"42"} + * }}} */ @SerialVersionUID(1L) final class Logger private (underlying: slf4j.Logger) extends Serializable { @@ -180,18 +233,34 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { def trace(message: String): Unit = if (underlying.isTraceEnabled) underlying.trace(message) + /** $log */ + def trace(message: => Any): Unit = + if (underlying.isTraceEnabled) underlying.trace(toString(message)) + /** $log */ def trace(message: String, cause: Throwable): Unit = if (underlying.isTraceEnabled) underlying.trace(message, cause) + /** $log */ + def trace(message: => Any, cause: Throwable): Unit = + if (underlying.isTraceEnabled) underlying.trace(toString(message), cause) + /** $logMarker */ def trace(marker: Marker, message: String): Unit = if (underlying.isTraceEnabled(marker)) underlying.trace(marker, message) + /** $logMarker */ + def trace(marker: Marker, message: => Any): Unit = + if (underlying.isTraceEnabled(marker)) underlying.trace(marker, toString(message)) + /** $logMarker */ def trace(marker: Marker, message: String, cause: Throwable): Unit = if (underlying.isTraceEnabled(marker)) underlying.trace(marker, message, cause) + /** $logMarker */ + def trace(marker: Marker, message: => Any, cause: Throwable): Unit = + if (underlying.isTraceEnabled(marker)) underlying.trace(marker, toString(message), cause) + /** $logWith */ def traceWith(message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -200,6 +269,14 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { if (underlying.isTraceEnabled) underlying.trace(message, args: _*) } + /** $logWith */ + def traceWith(message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isTraceEnabled) underlying.trace(toString(message)) + } else { + if (underlying.isTraceEnabled) underlying.trace(toString(message), args: _*) + } + /** $logWithMarker */ def traceWith(marker: Marker, message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -208,6 +285,21 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { if (underlying.isTraceEnabled(marker)) underlying.trace(marker, message, args: _*) } + /** $logWithMarker */ + def traceWith(marker: Marker, message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isTraceEnabled(marker)) underlying.trace(marker, toString(message)) + } else { + if (underlying.isTraceEnabled(marker)) underlying.trace(marker, toString(message), args: _*) + } + + /** $logResult */ + def traceResult[T](message: => String)(fn: => T): T = { + val result = fn + trace(message.format(result)) + result + } + /* Debug */ /** $isLevelEnabled */ @@ -221,18 +313,34 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { def debug(message: String): Unit = if (underlying.isDebugEnabled) underlying.debug(message) + /** $log */ + def debug(message: => Any): Unit = + if (underlying.isDebugEnabled) underlying.debug(toString(message)) + /** $log */ def debug(message: String, cause: Throwable): Unit = if (underlying.isDebugEnabled) underlying.debug(message, cause) + /** $log */ + def debug(message: => Any, cause: Throwable): Unit = + if (underlying.isDebugEnabled) underlying.debug(toString(message), cause) + /** $logMarker */ def debug(marker: Marker, message: String): Unit = if (underlying.isDebugEnabled(marker)) underlying.debug(marker, message) + /** $logMarker */ + def debug(marker: Marker, message: => Any): Unit = + if (underlying.isDebugEnabled(marker)) underlying.debug(marker, toString(message)) + /** $logMarker */ def debug(marker: Marker, message: String, cause: Throwable): Unit = if (underlying.isDebugEnabled(marker)) underlying.debug(marker, message, cause) + /** $logMarker */ + def debug(marker: Marker, message: => Any, cause: Throwable): Unit = + if (underlying.isDebugEnabled(marker)) underlying.debug(marker, toString(message), cause) + /** $logWith */ def debugWith(message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -241,6 +349,14 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { if (underlying.isDebugEnabled) underlying.debug(message, args: _*) } + /** $logWith */ + def debugWith(message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isDebugEnabled) underlying.debug(toString(message)) + } else { + if (underlying.isDebugEnabled) underlying.debug(toString(message), args: _*) + } + /** $logWithMarker */ def debugWith(marker: Marker, message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -249,6 +365,21 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { if (underlying.isDebugEnabled(marker)) underlying.debug(marker, message, args: _*) } + /** $logWithMarker */ + def debugWith(marker: Marker, message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isDebugEnabled(marker)) underlying.debug(marker, toString(message)) + } else { + if (underlying.isDebugEnabled(marker)) underlying.debug(marker, toString(message), args: _*) + } + + /** $logResult */ + def debugResult[T](message: => String)(fn: => T): T = { + val result = fn + debug(message.format(result)) + result + } + /* Info */ /** $isLevelEnabled */ @@ -262,18 +393,34 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { def info(message: String): Unit = if (underlying.isInfoEnabled) underlying.info(message) + /** $log */ + def info(message: => Any): Unit = + if (underlying.isInfoEnabled) underlying.info(toString(message)) + /** $log */ def info(message: String, cause: Throwable): Unit = if (underlying.isInfoEnabled) underlying.info(message, cause) + /** $log */ + def info(message: => Any, cause: Throwable): Unit = + if (underlying.isInfoEnabled) underlying.info(toString(message), cause) + /** $logMarker */ def info(marker: Marker, message: String): Unit = if (underlying.isInfoEnabled(marker)) underlying.info(marker, message) + /** $logMarker */ + def info(marker: Marker, message: => Any): Unit = + if (underlying.isInfoEnabled(marker)) underlying.info(marker, toString(message)) + /** $logMarker */ def info(marker: Marker, message: String, cause: Throwable): Unit = if (underlying.isInfoEnabled(marker)) underlying.info(marker, message, cause) + /** $logMarker */ + def info(marker: Marker, message: => Any, cause: Throwable): Unit = + if (underlying.isInfoEnabled(marker)) underlying.info(marker, toString(message), cause) + /** $logWith */ def infoWith(message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -282,6 +429,14 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { if (underlying.isInfoEnabled) underlying.info(message, args: _*) } + /** $logWith */ + def infoWith(message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isInfoEnabled) underlying.info(toString(message)) + } else { + if (underlying.isInfoEnabled) underlying.info(toString(message), args: _*) + } + /** $logWithMarker */ def infoWith(marker: Marker, message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -290,6 +445,21 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { if (underlying.isInfoEnabled(marker)) underlying.info(marker, message, args: _*) } + /** $logWithMarker */ + def infoWith(marker: Marker, message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isInfoEnabled(marker)) underlying.info(marker, toString(message)) + } else { + if (underlying.isInfoEnabled(marker)) underlying.info(marker, toString(message), args: _*) + } + + /** $logResult */ + def infoResult[T](message: => String)(fn: => T): T = { + val result = fn + info(message.format(result)) + result + } + /* Warn */ /** $isLevelEnabled */ @@ -303,18 +473,34 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { def warn(message: String): Unit = if (underlying.isWarnEnabled) underlying.warn(message) + /** $log */ + def warn(message: => Any): Unit = + if (underlying.isWarnEnabled) underlying.warn(toString(message)) + /** $log */ def warn(message: String, cause: Throwable): Unit = if (underlying.isWarnEnabled) underlying.warn(message, cause) + /** $log */ + def warn(message: => Any, cause: Throwable): Unit = + if (underlying.isWarnEnabled) underlying.warn(toString(message), cause) + /** $logMarker */ def warn(marker: Marker, message: String): Unit = if (underlying.isWarnEnabled(marker)) underlying.warn(marker, message) + /** $logMarker */ + def warn(marker: Marker, message: => Any): Unit = + if (underlying.isWarnEnabled(marker)) underlying.warn(marker, toString(message)) + /** $logMarker */ def warn(marker: Marker, message: String, cause: Throwable): Unit = if (underlying.isWarnEnabled(marker)) underlying.warn(marker, message, cause) + /** $logMarker */ + def warn(marker: Marker, message: => Any, cause: Throwable): Unit = + if (underlying.isWarnEnabled(marker)) underlying.warn(marker, toString(message), cause) + /** $logWith */ def warnWith(message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -323,6 +509,14 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { if (underlying.isWarnEnabled) underlying.warn(message, args: _*) } + /** $logWith */ + def warnWith(message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isWarnEnabled) underlying.warn(toString(message)) + } else { + if (underlying.isWarnEnabled) underlying.warn(toString(message), args: _*) + } + /** $logWithMarker */ def warnWith(marker: Marker, message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -331,6 +525,21 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { if (underlying.isWarnEnabled(marker)) underlying.warn(marker, message, args: _*) } + /** $logWithMarker */ + def warnWith(marker: Marker, message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isWarnEnabled(marker)) underlying.warn(marker, toString(message)) + } else { + if (underlying.isWarnEnabled(marker)) underlying.warn(marker, toString(message), args: _*) + } + + /** $logResult */ + def warnResult[T](message: => String)(fn: => T): T = { + val result = fn + warn(message.format(result)) + result + } + /* Error */ /** $isLevelEnabled */ @@ -344,18 +553,34 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { def error(message: String): Unit = if (underlying.isErrorEnabled) underlying.error(message) + /** $log */ + def error(message: => Any): Unit = + if (underlying.isErrorEnabled) underlying.error(toString(message)) + /** $log */ def error(message: String, cause: Throwable): Unit = if (underlying.isErrorEnabled) underlying.error(message, cause) + /** $log */ + def error(message: => Any, cause: Throwable): Unit = + if (underlying.isErrorEnabled) underlying.error(toString(message), cause) + /** $logMarker */ def error(marker: Marker, message: String): Unit = if (underlying.isErrorEnabled(marker)) underlying.error(marker, message) + /** $logMarker */ + def error(marker: Marker, message: => Any): Unit = + if (underlying.isErrorEnabled(marker)) underlying.error(marker, toString(message)) + /** $logMarker */ def error(marker: Marker, message: String, cause: Throwable): Unit = if (underlying.isErrorEnabled(marker)) underlying.error(marker, message, cause) + /** $logMarker */ + def error(marker: Marker, message: => Any, cause: Throwable): Unit = + if (underlying.isErrorEnabled(marker)) underlying.error(marker, toString(message), cause) + /** $logWith */ def errorWith(message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -364,6 +589,14 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { if (underlying.isErrorEnabled) underlying.error(message, args: _*) } + /** $logWith */ + def errorWith(message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isErrorEnabled) underlying.error(toString(message)) + } else { + if (underlying.isErrorEnabled) underlying.error(toString(message), args: _*) + } + /** $logWithMarker */ def errorWith(marker: Marker, message: String, args: AnyRef*): Unit = if (args.isEmpty) { @@ -371,4 +604,29 @@ final class Logger private (underlying: slf4j.Logger) extends Serializable { } else { if (underlying.isErrorEnabled(marker)) underlying.error(marker, message, args: _*) } + + /** $logWithMarker */ + def errorWith(marker: Marker, message: => Any, args: AnyRef*): Unit = + if (args.isEmpty) { + if (underlying.isErrorEnabled(marker)) underlying.error(marker, toString(message)) + } else { + if (underlying.isErrorEnabled(marker)) underlying.error(marker, toString(message), args: _*) + } + + /** $logResult */ + def errorResult[T](message: => String)(fn: => T): T = { + val result = fn + error(message.format(result)) + result + } + + /* Private */ + + /** + * Convert [[Any]] type to [[String]] to safely deal with nulls in the above functions. + */ + private[this] def toString(obj: Any): String = obj match { + case null => "null" + case _ => obj.toString + } } diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Loggers.scala b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Loggers.scala deleted file mode 100644 index 53317c036d..0000000000 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Loggers.scala +++ /dev/null @@ -1,53 +0,0 @@ -package com.twitter.util.logging - -import org.slf4j - -/** - * For Java usability. - * - * @note Scala users see [[com.twitter.util.logging.Logger]] - * @deprecated Java users can use corresponding methods in [[com.twitter.util.logging.Logger]] - */ -@deprecated("Use com.twitter.util.logging.Logger", "2020-09-30") -object Loggers { - - /** - * Create a [[Logger]] for the given name. - * - * @param name name of the underlying `slf4j.Logger`. - * - * {{{ - * Logger logger = Logger.getLogger("name"); - * }}} - */ - @deprecated("Use com.twitter.util.logging.Logger", "2020-09-30") - def getLogger(name: String): Logger = - Logger(name) - - /** - * Create a [[Logger]] named for the given class. - * - * @param clazz class to use for naming the underlying slf4j.Logger. - * - * {{{ - * Logger logger = Logger.getLogger(classOf[MyClass]); - * }}} - */ - @deprecated("Use com.twitter.util.logging.Logger", "2020-09-30") - def getLogger(clazz: Class[_]): Logger = - Logger(clazz) - - /** - * Create a [[Logger]] wrapping the given underlying - * `org.slf4j.Logger`. - * - * @param underlying an `org.slf4j.Logger` - * - * {{{ - * Logger logger = Logger.getLogger(LoggerFactory.getLogger("name")); - * }}} - */ - @deprecated("Use com.twitter.util.logging.Logger", "2020-09-30") - def getLogger(underlying: slf4j.Logger): Logger = - Logger(underlying) -} diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logging.scala b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logging.scala index 7b2f1d5f1b..2b344b2557 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logging.scala +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/Logging.scala @@ -1,6 +1,6 @@ package com.twitter.util.logging -import org.slf4j.{LoggerFactory, Marker} +import org.slf4j.Marker import scala.language.implicitConversions /** @@ -56,8 +56,7 @@ import scala.language.implicitConversions */ trait Logging { - private[this] final lazy val _logger: Logger = - Logger(LoggerFactory.getLogger(getClass)) + private[this] final lazy val _logger: Logger = Logger(getClass) /** * Return the underlying [[com.twitter.util.logging.Logger]] diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/LoggersCompilationTest.java b/util-slf4j-api/src/test/java/com/twitter/util/logging/LoggersCompilationTest.java deleted file mode 100644 index 1bfd6dea07..0000000000 --- a/util-slf4j-api/src/test/java/com/twitter/util/logging/LoggersCompilationTest.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.twitter.util.logging; - -import org.junit.Test; -import org.slf4j.LoggerFactory; - -import static org.junit.Assert.*; - -public final class LoggersCompilationTest { - - @Test - public void testApply1() { - /* By slf4j Logger */ - - Logger logger = - Loggers.getLogger(LoggerFactory.getLogger(this.getClass())); - - assertEquals( - "com.twitter.util.logging.LoggersCompilationTest", - logger.name()); - } - - @Test - public void testApply2() { - /* By Class */ - - Logger logger = - Loggers.getLogger(LoggersCompilationTest.class); - - assertEquals( - "com.twitter.util.logging.LoggersCompilationTest", - logger.name()); - } - - @Test - public void testApply3() { - /* By String */ - - Logger logger = - Loggers.getLogger("LoggersCompilationTest"); - - assertEquals( - "LoggersCompilationTest", - logger.name()); - } -} diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala index f4bb5e3b85..a5b3f4c511 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/LoggingTest.scala @@ -363,7 +363,7 @@ class LoggingTest extends AnyFunSuite with Matchers with MockitoSugar { val message = "msg" val cause = new RuntimeException("TEST EXCEPTION") val arg1 = "arg1" - val arg2 = Integer.valueOf(1) + val arg2: Integer = Integer.valueOf(1) val arg3 = "arg3" val underlying: org.slf4j.Logger = mock[org.slf4j.Logger] when(isEnabledFn(underlying)).thenReturn(isEnabled) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala index 5215673199..c0d42436fc 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/MarkerLoggingTest.scala @@ -275,7 +275,7 @@ class MarkerLoggingTest extends AnyFunSuite with Matchers with MockitoSugar { val message = "msg" val cause = new RuntimeException("TEST EXCEPTION") val arg1 = "arg1" - val arg2 = new Integer(1) + val arg2: Integer = Integer.valueOf(1) val arg3 = "arg3" val underlying: org.slf4j.Logger = mock[org.slf4j.Logger] when(p(underlying)).thenReturn(isEnabled) From ad617fe92445371dc5647add7a7143450492c137 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Wed, 2 Feb 2022 22:54:17 +0000 Subject: [PATCH 622/796] util-jvm: Register JVM expressions to StatsReceiver Problem/Solution Register JVM expression including heap usages, metaspace usages, memory pool usages, and open file descriptor count in StatsReceiver. JIRA Issues: CSL-11632 Differential Revision: https://phabricator.twitter.biz/D820472 --- CHANGELOG.rst | 7 ++ .../main/scala/com/twitter/jvm/JvmStats.scala | 68 +++++++++++----- .../scala/com/twitter/jvm/JvmStatsTest.scala | 77 ++++++++++++++++--- 3 files changed, 121 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 82a788d1a2..a60daf774b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-jvm: Register JVM expression including memory pool usages (including code cache, compressed class space, + eden space, sheap, metaspace, survivor space, and old gen) and open file descriptors count in StatsReceiver. + ``PHAB_ID=D820472`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index d017723aa0..7cd025aad1 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -2,6 +2,7 @@ package com.twitter.jvm import com.twitter.conversions.StringOps._ import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.Bytes import com.twitter.finagle.stats.Milliseconds import com.twitter.finagle.stats.StatsReceiver import com.twitter.finagle.stats.exp.Expression @@ -25,9 +26,10 @@ object JvmStats { def heap = mem.getHeapMemoryUsage() val heapStats = stats.scope("heap") + val heapUsedGauge = heapStats.addGauge("used") { heap.getUsed().toFloat } gauges.add(heapStats.addGauge("committed") { heap.getCommitted().toFloat }) gauges.add(heapStats.addGauge("max") { heap.getMax().toFloat }) - gauges.add(heapStats.addGauge("used") { heap.getUsed().toFloat }) + gauges.add(heapUsedGauge) def nonHeap = mem.getNonHeapMemoryUsage() val nonHeapStats = stats.scope("nonheap") @@ -51,8 +53,17 @@ object JvmStats { gauges.add(stats.addGauge("num_cpus") { os.getAvailableProcessors().toFloat }) os match { case unix: com.sun.management.UnixOperatingSystemMXBean => - gauges.add(stats.addGauge("fd_count") { unix.getOpenFileDescriptorCount.toFloat }) + val fileDescriptorCountGauge = stats.addGauge("fd_count") { + unix.getOpenFileDescriptorCount.toFloat + } + gauges.add(fileDescriptorCountGauge) gauges.add(stats.addGauge("fd_limit") { unix.getMaxFileDescriptorCount.toFloat }) + // register expression + ExpressionSchema("file_descriptors", Expression(fileDescriptorCountGauge.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withDescription( + "Total file descriptors used by the service. If it continuously increasing over time, then potentially files or connections aren't being closed") + .build() case _ => } @@ -100,8 +111,18 @@ object JvmStats { } if (pool.getUsage != null) { def usage = pool.getUsage // this is a snapshot, we can't reuse the value - gauges.add(currentMem.addGauge(name, "used") { usage.getUsed.toFloat }) + val usageGauge = currentMem.addGauge(name, "used") { usage.getUsed.toFloat } + gauges.add(usageGauge) gauges.add(currentMem.addGauge(name, "max") { usage.getMax.toFloat }) + + // register memory usage expression + ExpressionSchema("memory_pool", Expression(usageGauge.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withLabel("kind", name) + .withUnit(Bytes) + .withDescription( + s"The current estimate of the amount of space within the $name memory pool holding allocated objects in bytes") + .build() } } gauges.add(postGCStats.addGauge("used") { @@ -165,7 +186,8 @@ object JvmStats { .withLabel(ExpressionSchema.Role, "jvm") .withLabel("gc_pool", name) .withUnit(Milliseconds) - .withDescription(s"The total elapsed time spent doing collections for the $name gc pool") + .withDescription( + s"The total elapsed time spent doing collections for the $name gc pool in milliseconds") .build() gauges.add(poolCycles) @@ -180,21 +202,6 @@ object JvmStats { gcPool.map(_.getCollectionTime).filter(_ > 0).sum.toFloat } - ExpressionSchema("jvm_uptime", Expression(uptime.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withUnit(Milliseconds) - .withDescription("The uptime of the JVM in MS") - .build() - ExpressionSchema("gc_cycles", Expression(cycles.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withDescription("The total number of collections that have occurred") - .build() - ExpressionSchema("gc_latency", Expression(msec.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withUnit(Milliseconds) - .withDescription("The total elapsed time spent doing collections") - .build() - gauges.add(cycles) gauges.add(msec) allocations = new Allocations(gcStats) @@ -208,6 +215,27 @@ object JvmStats { // return ms from ns while retaining precision gauges.add(stats.addGauge("application_time_millis") { jvm.applicationTime.toFloat / 1000000 }) gauges.add(stats.addGauge("tenuring_threshold") { jvm.tenuringThreshold.toFloat }) - } + // register metric expressions + ExpressionSchema("jvm_uptime", Expression(uptime.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withUnit(Milliseconds) + .withDescription("The uptime of the JVM in milliseconds") + .build() + ExpressionSchema("gc_cycles", Expression(cycles.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withDescription("The total number of collections that have occurred") + .build() + ExpressionSchema("gc_latency", Expression(msec.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withUnit(Milliseconds) + .withDescription("The total elapsed time spent doing collections in milliseconds") + .build() + ExpressionSchema("memory_pool", Expression(heapUsedGauge.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withLabel("kind", "Heap") + .withUnit(Bytes) + .withDescription("Heap in use in bytes") + .build() + } } diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala index 441d8ff4f3..e8cc020862 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala @@ -2,26 +2,81 @@ package com.twitter.jvm import com.twitter.finagle.stats.InMemoryStatsReceiver import com.twitter.finagle.stats.exp.ExpressionSchema -import com.twitter.finagle.stats.exp.ExpressionSchemaKey import com.twitter.finagle.stats.exp.MetricExpression import org.scalatest.funsuite.AnyFunSuite class JvmStatsTest extends AnyFunSuite { - def assertMetric(expressionSchema: ExpressionSchema): Unit = { - assert(expressionSchema.expr.isInstanceOf[MetricExpression]) - } + private[this] val receiver = new InMemoryStatsReceiver() + JvmStats.register(receiver) + private[this] val expressions = receiver.getAllExpressionsWithLabel(ExpressionSchema.Role, "jvm") test("expressions are instrumented") { - val receiver = new InMemoryStatsReceiver() - JvmStats.register(receiver) + expressions.foreach { + case (_, expressionSchema) => + assertMetric(expressionSchema) + } + } - val expressions = receiver.getAllExpressionsWithLabel(ExpressionSchema.Role, "jvm") + test("memory_pool expression is registered") { + val memoryPoolExpressionLabels = expressions + .filter { + case (k, _) => k.name == "memory_pool" + }.keySet.map(_.labels) - assert(expressions.size == 7) + assertLabels(memoryPoolExpressionLabels, "role", "jvm") + assertLabels(memoryPoolExpressionLabels, "kind", "Code_Cache") + assertLabels(memoryPoolExpressionLabels, "kind", "Compressed_Class_Space") + assertLabels(memoryPoolExpressionLabels, "kind", "Metaspace") + assertLabels(memoryPoolExpressionLabels, "kind", "PS_Eden_Space") + assertLabels(memoryPoolExpressionLabels, "kind", "Heap") + assertLabels(memoryPoolExpressionLabels, "kind", "PS_Survivor_Space") + assertLabels(memoryPoolExpressionLabels, "kind", "PS_Old_Gen") + } - expressions.foreach { (x: (ExpressionSchemaKey, ExpressionSchema)) => - assertMetric(x._2) - } + test("file_descriptors expression is registered") { + val fileDescriptorsExpressionLabels = expressions + .filter { + case (k, _) => k.name == "file_descriptors" + }.keySet.map(_.labels) + + assertLabels(fileDescriptorsExpressionLabels, "role", "jvm") + } + + test("gc_cycles expression is registered") { + val gcCyclesExpressionLabels = expressions + .filter { + case (k, _) => k.name == "gc_cycles" + }.keySet.map(_.labels) + assert(gcCyclesExpressionLabels.contains(Map("role" -> "jvm", "gc_pool" -> "PS_Scavenge"))) + assert(gcCyclesExpressionLabels.contains(Map("role" -> "jvm", "gc_pool" -> "PS_MarkSweep"))) } + + test("gc_latency expression is registered") { + val gcLatenciesExpressionLabels = expressions + .filter { + case (k, _) => k.name == "gc_latency" + }.keySet.map(_.labels) + assert(gcLatenciesExpressionLabels.contains(Map("role" -> "jvm", "gc_pool" -> "PS_Scavenge"))) + assert(gcLatenciesExpressionLabels.contains(Map("role" -> "jvm", "gc_pool" -> "PS_MarkSweep"))) + } + + test("jvm_uptime expression is registered") { + val jvmUptimeExpressionLabels = expressions + .filter { + case (k, _) => k.name == "jvm_uptime" + }.keySet.map(_.labels) + + assertLabels(jvmUptimeExpressionLabels, "role", "jvm") + } + + private[this] def assertMetric(expressionSchema: ExpressionSchema): Unit = + assert(expressionSchema.expr.isInstanceOf[MetricExpression]) + + private[this] def assertLabels( + labels: scala.collection.Set[Map[String, String]], + key: String, + value: String + ): Unit = + assert(labels.exists(_.get(key).get == value)) } From bbde11622550af943d4d288f093ab6a2b71471fc Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Fri, 4 Feb 2022 20:32:39 +0000 Subject: [PATCH 623/796] util-core: Fix race condition in Var.collectIndependent Problem There is a subtle race condition in the current Var.collectIndepdent implementation where the "async" update may be racing with the very last "sync" one and triggering the observation with "null" in the last position of the collected result. Solution Fix race condition by checking the `filling` state within a synchronized block. JIRA Issues: CSL-11377 Differential Revision: https://phabricator.twitter.biz/D824055 --- util-core/src/main/scala/com/twitter/util/Var.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index ef16e3e9b9..643289fd49 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -364,29 +364,32 @@ object Var { // // @note "filling" only works with the guarantee that the initial `observe` is // synchronous. This should be the case with Vars since they have an initial value. - @volatile var filling = true + var filling = true val cur = new Array[Any](N) def publishAt(i: Int): T => Unit = { newValue => cur.synchronized { cur(i) = newValue + if (filling && i == N - 1) filling = false if (!filling) { - u() = ArraySeq.unsafeWrapArray(cur).asInstanceOf[ArraySeq[T]] + // toSeq does not do a deep copy until 2.13 + val copy = new Array[Any](N) + Array.copy(cur, 0, copy, 0, cur.length) + u() = ArraySeq.unsafeWrapArray(copy).asInstanceOf[Seq[T]] } } } - val closables = new Array[Any](N) + val closables = new Array[Closable](N) var i = 0 val iter = vars.iterator while (iter.hasNext) { val v = iter.next() - if (i == N - 1) filling = false closables(i) = v.observe(publishAt(i)) i += 1 } - Closable.all(ArraySeq.unsafeWrapArray(closables).asInstanceOf[ArraySeq[Closable]]: _*) + Closable.all(ArraySeq.unsafeWrapArray(closables): _*) } /** From b696fdfd9082fc7dfb3468f7aea84ce24677ef82 Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Mon, 7 Feb 2022 17:24:47 +0000 Subject: [PATCH 624/796] util-core: Add Memoize.classValue Problem We'd like to use `java.lang.ClassValue`, but its Java API looks cumbersome in Scala as it requires subclassing and overriding a protected method for every class value. Solution Add a `Memoize.classValue` method for `Class[_] => T` functions that resembles ordinary memoization in its API but uses `ClassValue` internally. Differential Revision: https://phabricator.twitter.biz/D825673 --- CHANGELOG.rst | 2 + .../main/scala/com/twitter/util/Memoize.scala | 17 +++++++ .../scala/com/twitter/util/MemoizeTest.scala | 46 ++++++++++++++++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a60daf774b..0418768ed6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,8 @@ Unreleased New Features ~~~~~~~~~~~~ +* util-core: Added `Memoize.classValue` as a Scala-friendly API for `java.lang.ClassValue`. ``PHAB_ID=D825673`` + * util-jvm: Register JVM expression including memory pool usages (including code cache, compressed class space, eden space, sheap, metaspace, survivor space, and old gen) and open file descriptors count in StatsReceiver. ``PHAB_ID=D820472`` diff --git a/util-core/src/main/scala/com/twitter/util/Memoize.scala b/util-core/src/main/scala/com/twitter/util/Memoize.scala index cb85640dc4..97e9a09bfa 100644 --- a/util-core/src/main/scala/com/twitter/util/Memoize.scala +++ b/util-core/src/main/scala/com/twitter/util/Memoize.scala @@ -164,4 +164,21 @@ object Memoize { case _ => missing(a) } } + + /** + * Thread-safe memoization for a function taking Class objects, + * using [[java.lang.ClassValue]]. + * + * @param f the function to memoize using a `ClassValue` + * @tparam T the return type of the function + * @return a memoized function over classes that stores memoized values + * in a `ClassValue` + */ + def classValue[T](f: Class[_] => T): Class[_] => T = { + val cv = new ClassValue[T] { + def computeValue(c: Class[_]): T = f(c) + } + + cv.get + } } diff --git a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala index c84ca34614..1e72509627 100644 --- a/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/MemoizeTest.scala @@ -2,7 +2,9 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.{TimeUnit, CountDownLatch => JavaCountDownLatch} +import java.util.concurrent.atomic.AtomicReference +import java.util.concurrent.{CountDownLatch => JavaCountDownLatch} +import java.util.concurrent.TimeUnit import org.mockito.Mockito._ import org.scalatest.funsuite.AnyFunSuite @@ -185,4 +187,46 @@ class MemoizeTest extends AnyFunSuite { assert(memoizer.size == memoizer.snap.size) assert(memoizer.snap.size == 1) } + + test("Memoize.classValue: returns the same value for the same input") { + val cv = Memoize.classValue { _ => + new Object() + } + // just some anonymous classes + val clazz1 = new Object() {}.getClass + val clazz2 = new Object() {}.getClass + assert(clazz1 ne clazz2) + + val o11 = cv(clazz1) + val o12 = cv(clazz1) + val o21 = cv(clazz2) + val o22 = cv(clazz2) + assert(o11 eq o12) + assert(o21 eq o22) + assert(o11 ne o21) + } + + test("Memoize.classValue: racing threads observe the same value") { + val Concurrency = 16 + val ready = new JavaCountDownLatch(Concurrency) + val cv = Memoize.classValue { _ => + ready.await(1, TimeUnit.SECONDS) + new Object() + } + + val clazz = new Object() {}.getClass + + val values = new Array[AtomicReference[Object]](Concurrency) + val threads = (0 until Concurrency).map { i => + new Thread(() => { + ready.countDown() + values(i) = new AtomicReference(cv(clazz)) + }) + } + threads.foreach { _.start() } + threads.foreach { _.join(1000L) } + + val v0 = values(0).get() + assert(values.forall(_.get eq v0)) + } } From ab73d64f95ceea48899e7561429d781ad8da5929 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Mon, 7 Feb 2022 20:29:00 +0000 Subject: [PATCH 625/796] util-slf4j-bridge: Add Slf4jBridge trait for handler installation in class construction Problem Installation of the SLF4JBridgeHandler should happen as early as possible, ideally during the construction of the main application. Solution Add a mixin which can be composed into extensions of `c.t.app.App` which will attempt installation of the bridge handler via the `Slf4jBridgeUtility` as part of class construction. Move TwitterServer to use the mixin in util-slf4j-jul-bridge. Result A simple way for extensions of `c.t.app.App` to ensure installation of the SLF4JBridgeHandler is attempted during the constructor of the `App`. Differential Revision: https://phabricator.twitter.biz/D827493 --- CHANGELOG.rst | 4 +++ build.sbt | 29 +++++++++---------- .../main/scala/com/twitter/util/logging/BUILD | 1 + .../twitter/util/logging/Slf4jBridge.scala | 21 ++++++++++++++ .../util/logging/Slf4jBridgeUtility.scala | 8 +++-- 5 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridge.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0418768ed6..fbaed5e2c8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,10 @@ New Features eden space, sheap, metaspace, survivor space, and old gen) and open file descriptors count in StatsReceiver. ``PHAB_ID=D820472`` +* util-slf4j-jul-bridge: Add `Slf4jBridge` trait which can be mixed into extensions of `c.t.app.App` in + order to attempt installation of the `SLF4JBridgeHandler` via the `Slf4jBridgeUtility` in the constructor + of the `c.t.app.App` instance. ``PHAB_ID=D827493`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index 9eb84ec3d8..d3f82e8c85 100644 --- a/build.sbt +++ b/build.sbt @@ -28,7 +28,6 @@ val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.15.4" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion val snakeyaml = "org.yaml" % "snakeyaml" % "1.24" - def travisTestJavaOptions: Seq[String] = { // We have some custom configuration for the Travis environment // https://docs.travis-ci.com/user/environment-variables/#default-environment-variables @@ -214,10 +213,9 @@ val sharedSettings = val sharedScala3EnabledSettings = baseSettings ++ defaultScala3EnabledSettings ++ - Seq(libraryDependencies ++= scala3Dependencies) ++ + Seq(libraryDependencies ++= scala3Dependencies) ++ Seq(excludeDependencies ++= Seq("org.scala-lang.modules" % "scala-collection-compat_3")) - lazy val noPublishSettings = Seq( publish / skip := true ) @@ -245,7 +243,7 @@ def scalatestScalacheckVersionedDep(scalaVersion: String) = { ) } } - + lazy val util = Project( id = "util", base = file(".") @@ -529,7 +527,7 @@ lazy val utilJvm = Project( name := "util-jvm", libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % "test", - ) ++ scalatestMockitoVersionedDep(scalaVersion.value) + ) ++ scalatestMockitoVersionedDep(scalaVersion.value) ).dependsOn(utilApp, utilCore, utilStats) lazy val utilLint = Project( @@ -617,7 +615,7 @@ lazy val utilSlf4jJulBridge = Project( ).settings( name := "util-slf4j-jul-bridge", libraryDependencies ++= Seq(slf4jApi, "org.slf4j" % "jul-to-slf4j" % slf4jVersion) - ).dependsOn(utilCore, utilSlf4jApi) + ).dependsOn(utilCore, utilApp, utilSlf4jApi) lazy val utilSecurity = Project( id = "util-security", @@ -655,18 +653,19 @@ lazy val utilStats = Project( scalacheckLib, "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, - ("com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava")).cross(CrossVersion.for3Use2_13), + ("com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion exclude ("com.google.guava", "guava")) + .cross(CrossVersion.for3Use2_13), "org.mockito" % "mockito-core" % mockitoVersion % "test" ) ++ scalatestMockitoVersionedDep(scalaVersion.value) - ++ scalatestScalacheckVersionedDep(scalaVersion.value) - ++ { - CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, major)) if major <= 12 => - Seq() - case _ => - Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.3" % "test") + ++ scalatestScalacheckVersionedDep(scalaVersion.value) + ++ { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, major)) if major <= 12 => + Seq() + case _ => + Seq("org.scala-lang.modules" %% "scala-parallel-collections" % "1.0.3" % "test") + } } - } ).dependsOn(utilApp, utilCore, utilLint) lazy val utilTest = Project( diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD index 2fad866749..9485be0942 100644 --- a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD @@ -10,6 +10,7 @@ scala_library( dependencies = [ "3rdparty/jvm/org/slf4j:jul-to-slf4j", "3rdparty/jvm/org/slf4j:slf4j-api", + "util/util-app/src/main/scala", "util/util-core/src/main/scala/com/twitter/concurrent", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridge.scala b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridge.scala new file mode 100644 index 0000000000..6fd6ec13b5 --- /dev/null +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridge.scala @@ -0,0 +1,21 @@ +package com.twitter.util.logging + +import com.twitter.app.App + +/** + * Installing the `SLF4JBridgeHandler` should be done as early as possible in order to capture any + * log statements emitted to JUL. Generally it is best to install the handler in the constructor of + * the main class of your application. + * + * This trait can be mixed into a [[com.twitter.app.App]] type and will attempt to install the + * `SLF4JBridgeHandler` via the [[Slf4jBridgeUtility]] as part of the constructor of the resultant + * class. + * @see [[Slf4jBridgeUtility]] + * @see [[org.slf4j.bridge.SLF4JBridgeHandler]] + * @see [[https://www.slf4j.org/legacy.html Bridging legacy APIs]] + */ +trait Slf4jBridge { self: App => + + /** Attempt Slf4jBridgeHandler installation */ + Slf4jBridgeUtility.attemptSlf4jBridgeHandlerInstallation() +} diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala index aa846e736e..a6168573ee 100644 --- a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/Slf4jBridgeUtility.scala @@ -17,6 +17,8 @@ import org.slf4j.bridge.SLF4JBridgeHandler * manner. However, using the [[https://logback.qos.ch/manual/configuration.html#LevelChangePropagator Logback LevelChangePropagator]] * (> 0.9.25) can eliminate this performance impact. * + * @see [[org.slf4j.bridge.SLF4JBridgeHandler]] + * @see [[https://www.slf4j.org/legacy.html Bridging legacy APIs]] * @see [[https://www.slf4j.org/legacy.html#jul-to-slf4j SLF4J: jul-to-slf4j bridge]] */ object Slf4jBridgeUtility extends Logging { @@ -30,7 +32,7 @@ object Slf4jBridgeUtility extends Logging { /* Private */ - private val install = Once { + private[this] val install = Once { if (!SLF4JBridgeHandler.isInstalled && canInstallBridgeHandler) { SLF4JBridgeHandler.removeHandlersForRootLogger() SLF4JBridgeHandler.install() @@ -42,7 +44,7 @@ object Slf4jBridgeUtility extends Logging { * We do not want to install the bridge handler if the * `org.slf4j.impl.JDK14LoggerFactory` exists on the classpath. */ - private def canInstallBridgeHandler: Boolean = { + private[this] def canInstallBridgeHandler: Boolean = { try { Class.forName("org.slf4j.impl.JDK14LoggerFactory", false, this.getClass.getClassLoader) LoggerFactory @@ -53,7 +55,7 @@ object Slf4jBridgeUtility extends Logging { ) false } catch { - case e: ClassNotFoundException => + case _: ClassNotFoundException => true } } From 88d3c95a30888d26f1c9bf8f6bf3b2d7f5e80ec2 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 8 Feb 2022 04:26:35 +0000 Subject: [PATCH 626/796] util-core: c.t.u.Responder is a dead type Problem We have a type called `Responder` that houses the methods that are already defined on Future. This does nothing but bloat the itable of `Promise`, which is the only implementation of the sealed `Responder` trait. Solution Fold the method definitions into `Promise` and call it a day. Differential Revision: https://phabricator.twitter.biz/D824830 --- CHANGELOG.rst | 5 ++ .../main/scala/com/twitter/util/Promise.scala | 78 +++++++++---------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fbaed5e2c8..941547aaca 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -32,6 +32,11 @@ Runtime Behavior Changes * util: Bump version of Caffeine to 2.9.3. ``PHAB_ID=D815761`` +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-core: The `c.t.util.Responder` trait has been removed. ``PHAB_ID=D824830`` + 22.1.0 ------ diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 3374f52349..8833d24223 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -323,45 +323,6 @@ object Promise { private val AlwaysUnit: Any => Unit = _ => () - sealed trait Responder[A] { this: Future[A] => - protected final def continueAll(wq: WaitQueue[A]): Unit = { - var ks = wq - while (ks ne WaitQueue.Empty) { - continue(ks.first) - ks = ks.rest - } - } - - protected def continue(k: K[A]): Unit - - /** - * Note: exceptions in responds are monitored. That is, if the - * computation `k` throws a raw (ie. not encoded in a Future) - * exception, it is handled by the current monitor, see - * [[Monitor]] for details. - */ - def respond(k: Try[A] => Unit): Future[A] = { - continue(new Monitored(Local.save(), k)) - this - } - - def transform[B](f: Try[A] => Future[B]): Future[B] = { - val promise = interrupts[B](this) - - continue(new FutureTransformer(Local.save(), f, promise)) - - promise - } - - protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { - val promise = interrupts[B](this) - - continue(new TryTransformer(Local.save(), f, promise)) - - promise - } - } - // PUBLIC API /** @@ -465,9 +426,44 @@ object Promise { * space leak is incurred from `flatMap` in the tail position since * intermediate promises are merged into the root promise. */ -class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[A]] { +class Promise[A] extends Future[A] with Updatable[Try[A]] { import Promise._ + private final def continueAll(wq: WaitQueue[A]): Unit = { + var ks = wq + while (ks ne WaitQueue.Empty) { + continue(ks.first) + ks = ks.rest + } + } + + /** + * Note: exceptions in responds are monitored. That is, if the + * computation `k` throws a raw (ie. not encoded in a Future) + * exception, it is handled by the current monitor, see + * [[Monitor]] for details. + */ + final def respond(k: Try[A] => Unit): Future[A] = { + continue(new Monitored(Local.save(), k)) + this + } + + final def transform[B](f: Try[A] => Future[B]): Future[B] = { + val promise = interrupts[B](this) + + continue(new FutureTransformer(Local.save(), f, promise)) + + promise + } + + final protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { + val promise = interrupts[B](this) + + continue(new TryTransformer(Local.save(), f, promise)) + + promise + } + // Note: this will always be one of: // - WaitQueue (Waiting) // - Interrupted @@ -776,7 +772,7 @@ class Promise[A] extends Future[A] with Promise.Responder[A] with Updatable[Try[ * * @throws Promise.ImmutableResult if the Promise is already populated */ - def update(result: Try[A]): Unit = { + final def update(result: Try[A]): Unit = { updateIfEmpty(result) || { val current = Await.result(liftToTry) throw ImmutableResult(s"Result set multiple times. Value='$current', New='$result'") From 316948e25bf1e424255658bfd2a695ee48961142 Mon Sep 17 00:00:00 2001 From: Flavio Brasil Date: Tue, 8 Feb 2022 04:34:28 +0000 Subject: [PATCH 627/796] [util-core] lightweight Local getter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Problem** In some contexts, `Local`s are read by the same thread at a high rate. For example, a scheduler worker that is pinned to a single thread might need to access a `Local` every time that a task is submitted to it. Even though the `Local` read is done always by the same thread, the `Local` implementation stills need to go through the `ThreadLocal` map to find the local value. **Solution** Change the encoding of the `ThreadLocal` that holds the `Local.Context` to use a `ContextRef`. When the context needs to be updated, instead of setting the `ThreadLocal` value, the `ContextRef` object is updated. This is safe because `ContextRef` is always accessed by the owning thread. The `ContextRef` value is used to implement a getter that caches the value of the local. On each getter invocation, the implementation checks if the context hasn't changed and uses the cached value if possible, which also avoids the `Context.get` lookup. **Notes** - For performance reasons, the getter implementation doesn't check if it's being accessed by the owning thread. There's an `assert` to make it easier to identify issues in tests. - I'm going to create a separate review to use this new mechanism in the fiber scheduler implementation - It seems that this mechanism could be used to implement the cpu time tracing in the `LocalScheduler` efficiently. **JMH results** ``` [info] Benchmark Mode Cnt Score Error Units [info] LocalBenchmark.get avgt 10 4.196 ± 0.213 ns/op [info] LocalBenchmark.getThreadLocal avgt 10 1.789 ± 0.104 ns/op ``` Differential Revision: https://phabricator.twitter.biz/D820768 --- .../com/twitter/util/LocalBenchmark.scala | 7 +- .../main/scala/com/twitter/util/Local.scala | 99 ++++++++++++++----- .../scala/com/twitter/util/LocalTest.scala | 51 ++++++++++ 3 files changed, 131 insertions(+), 26 deletions(-) diff --git a/util-benchmark/src/main/scala/com/twitter/util/LocalBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/LocalBenchmark.scala index 7b66ab8416..2950cb09fa 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/LocalBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/LocalBenchmark.scala @@ -2,12 +2,14 @@ package com.twitter.util import org.openjdk.jmh.annotations._ -@State(Scope.Benchmark) +@State(Scope.Thread) +@Threads(1) class LocalBenchmark extends StdBenchAnnotations { // We've sampled some service instances and get the result that usually // 10 Locals are constructed and 3 Locals are set. val size = 10 private[this] val locals = Array.fill(size)(new Local[String]) + private[this] val getter = locals(1).threadLocalGetter() @Benchmark def let1(): String = @@ -41,4 +43,7 @@ class LocalBenchmark extends StdBenchAnnotations { @Benchmark def get(): Option[String] = locals(1)() + + @Benchmark + def getThreadLocal(): Option[String] = getter() } diff --git a/util-core/src/main/scala/com/twitter/util/Local.scala b/util-core/src/main/scala/com/twitter/util/Local.scala index d97b078c54..8c144338c3 100644 --- a/util-core/src/main/scala/com/twitter/util/Local.scala +++ b/util-core/src/main/scala/com/twitter/util/Local.scala @@ -4858,6 +4858,7 @@ object Local { * Key used in [[Context]], internal. */ private[util] final class Key + private final class ContextRef(var ctx: Context) /** * Represents the current state of all [[Local locals]] for a given @@ -4866,43 +4867,73 @@ object Local { * This should be treated as an opaque value and direct modifications * and access are considered verboten. */ - private[this] val localCtx = new ThreadLocal[Context] { - override def initialValue(): Context = Context.empty + private[this] val localContextRef = new ThreadLocal[ContextRef] { + override def initialValue() = new ContextRef(Context.empty) } /** * Return a snapshot of the current Local state. */ - def save(): Context = localCtx.get + def save(): Context = saveRef().ctx /** * Restore the Local state to a given Context of values. */ - def restore(saved: Context): Unit = localCtx.set(saved) + def restore(saved: Context): Unit = saveRef().ctx = saved + + private def saveRef(): ContextRef = localContextRef.get() private def set(key: Key, value: Some[_]): Unit = { - localCtx.set(localCtx.get.set(key, value)) + val ref = saveRef() + ref.ctx = ref.ctx.set(key, value) } private def get(key: Key): Option[_] = - localCtx.get.get(key) + save().get(key) + + private def clear(key: Key): Unit = { + val ref = saveRef() + ref.ctx = ref.ctx.remove(key) + } - private def clear(key: Key): Unit = - localCtx.set(localCtx.get.remove(key)) + /** + * A lightweight getter scoped to a specific thread. + * + * IMPORTANT: *MUST* be used only by the thread that + * creates it. + */ + final class ThreadLocalGetter[T] private[Local] (key: Key) { + private[this] val owner = Thread.currentThread() + private[this] val ref = saveRef() + private[this] var contextCache = ref.ctx + private[this] var valueCache = contextCache.get(key) + def apply(): Option[T] = { + assert(Thread.currentThread() == owner) + val curr = ref.ctx + if (curr ne contextCache) { + contextCache = curr + valueCache = contextCache.get(key) + } + valueCache.asInstanceOf[Option[T]] + } + } /** * Clear all locals in the current context. */ - def clear(): Unit = localCtx.set(Context.empty) + def clear(): Unit = restore(Context.empty) /** * Execute a block with the given Locals, restoring current values upon completion. */ def let[U](ctx: Context)(f: => U): U = { - val saved = save() - restore(ctx) + val ref = saveRef() + val saved = ref.ctx + ref.ctx = ctx try f - finally restore(saved) + finally { + ref.ctx = saved + } } /** @@ -4919,10 +4950,13 @@ object Local { def closed[R](fn: () => R): () => R = { val closure = Local.save() () => { - val save = Local.save() - Local.restore(closure) + val ref = saveRef() + val save = ref.ctx + ref.ctx = closure try fn() - finally Local.restore(save) + finally { + ref.ctx = save + } } } } @@ -4968,27 +5002,41 @@ final class Local[T] { */ def apply(): Option[T] = Local.get(key).asInstanceOf[Option[T]] + /** + * Creates a lightweight getter for the current thread. + * + * IMPORTANT: The returned getter *MUST* be used only by + * the thread that creates it. + * + * This method is useful to avoid performance overhead if + * a local needs to be accessed several times by the same + * thread. + */ + def threadLocalGetter(): Local.ThreadLocalGetter[T] = + new Local.ThreadLocalGetter(key) + /** * Execute a block with a specific Local value, restoring the current state * upon completion. */ def let[U](value: T)(f: => U): U = { - val oldCtx = Local.save() + val ref = Local.saveRef() + val oldCtx = ref.ctx val newCtx = oldCtx.set(key, Some(value)) - Local.restore(newCtx) + ref.ctx = newCtx try f finally { - val now = Local.save() + val now = ref.ctx if (newCtx eq now) { // Fast path: no other ctx modifications to worry about - Local.restore(oldCtx) + ref.ctx = oldCtx } else { // Another element was updated in the meantime. We just have to set the old value. val next = oldCtx.get(key) match { case s @ Some(_) => now.set(key, s) case None => now.remove(key) } - Local.restore(next) + ref.ctx = next } } } @@ -4998,22 +5046,23 @@ final class Local[T] { * completion. */ def letClear[U](f: => U): U = { - val oldCtx = Local.save() + val ref = Local.saveRef() + val oldCtx = ref.ctx val newCtx = oldCtx.remove(key) - Local.restore(newCtx) + ref.ctx = newCtx try f finally { - val now = Local.save() + val now = ref.ctx if (newCtx eq now) { // Fast path: no other ctx modifications to worry about - Local.restore(oldCtx) + ref.ctx = oldCtx } else { // Another element was updated in the meantime. We just have to set the old value. val next = oldCtx.get(key) match { case s @ Some(_) => now.set(key, s) case None => now.remove(key) } - Local.restore(next) + ref.ctx = next } } } diff --git a/util-core/src/test/scala/com/twitter/util/LocalTest.scala b/util-core/src/test/scala/com/twitter/util/LocalTest.scala index 1edf886072..ee925fe1fd 100644 --- a/util-core/src/test/scala/com/twitter/util/LocalTest.scala +++ b/util-core/src/test/scala/com/twitter/util/LocalTest.scala @@ -196,6 +196,57 @@ class LocalTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { assert(fn() == 101) } + test("Local.threadLocalGetter returns the same value as local.apply") { + val l = new Local[Int] + val getter = l.threadLocalGetter() + l() = 1 + assert(getter() == l()) + val adder: () => Int = { () => + val rv = 100 + getter().get + l() = 10000 + assert(getter() == l()) + rv + } + val fn = Local.closed(adder) + l() = 100 + assert(getter() == l()) + assert(fn() == 101) + assert(getter() == l()) + assert(l() == Some(100)) + assert(fn() == 101) + } + + test("Local.threadLocalGetter should have a per-thread state") { + val local = new Local[Int] + val getter = local.threadLocalGetter() + var threadValue: Option[Int] = null + + local() = 123 + + val t = new Thread { + override def run() = { + val getter2 = local.threadLocalGetter() + assert(getter2() == None) + local() = 333 + threadValue = getter2() + } + } + + t.start() + t.join() + + assert(local() == Some(123)) + assert(getter() == Some(123)) + assert(threadValue == Some(333)) + } + + test("Local.threadLocalGetter asserts that it used by its owner") { + val l = new Local[Int] + val getter = + Await.result(FuturePool.unboundedPool(l.threadLocalGetter())) + intercept[AssertionError](getter()) + } + implicit lazy val arbKey: Arbitrary[Local.Key] = Arbitrary(new Local.Key) test("Context should have the same behavior with Map") { From 120f892c2227b3934317fc0c0d62afb1222034d5 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 10 Feb 2022 23:09:23 +0000 Subject: [PATCH 628/796] util-jvm: fix flaky JvmStatsTest Problem `JvmStatsTest` is flaky since it depends on the JVM and GC version the test is running on. It passed in source but failed OSS due to running in a different environment with error message: ``` 2022-02-07T21:22:36.8995460Z [info] JvmStatsTest: 2022-02-07T21:22:36.8996550Z [info] - expressions are instrumented 2022-02-07T21:22:36.9456672Z [info] - memory_pool expression is registered *** FAILED *** 2022-02-07T21:22:36.9491043Z [info] labels.exists(((x$6: Map[String,String]) => x$6.get(key).get.==(value))) was false (JvmStatsTest.scala:81) 2022-02-07T21:22:36.9517859Z [info] - file_descriptors expression is registered 2022-02-07T21:22:36.9578658Z [info] - gc_cycles expression is registered *** FAILED *** 2022-02-07T21:22:36.9582297Z [info] Set(Map("role" -> "jvm", "gc_pool" -> "ParNew"), Map("role" -> "jvm", "gc_pool" -> "ConcurrentMarkSweep"), Map("role" -> "jvm")) did not contain Map("role" -> "jvm", "gc_pool" -> "PS_Scavenge") (JvmStatsTest.scala:51) 2022-02-07T21:22:36.9611928Z [info] - gc_latency expression is registered *** FAILED *** 2022-02-07T21:22:36.9615667Z [info] Set(Map("role" -> "jvm", "gc_pool" -> "ConcurrentMarkSweep"), Map("role" -> "jvm"), Map("role" -> "jvm", "gc_pool" -> "ParNew")) did not contain Map("role" -> "jvm", "gc_pool" -> "PS_Scavenge") (JvmStatsTest.scala:60) ``` Solution Remove the assumptions in the tests that varies per JVM/GC implementation. JIRA Issues: CSL-11698 Differential Revision: https://phabricator.twitter.biz/D829391 --- .../scala/com/twitter/jvm/JvmStatsTest.scala | 64 +++---------------- 1 file changed, 8 insertions(+), 56 deletions(-) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala index e8cc020862..c5fcf03a4e 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmStatsTest.scala @@ -18,65 +18,17 @@ class JvmStatsTest extends AnyFunSuite { } } - test("memory_pool expression is registered") { - val memoryPoolExpressionLabels = expressions - .filter { - case (k, _) => k.name == "memory_pool" - }.keySet.map(_.labels) - - assertLabels(memoryPoolExpressionLabels, "role", "jvm") - assertLabels(memoryPoolExpressionLabels, "kind", "Code_Cache") - assertLabels(memoryPoolExpressionLabels, "kind", "Compressed_Class_Space") - assertLabels(memoryPoolExpressionLabels, "kind", "Metaspace") - assertLabels(memoryPoolExpressionLabels, "kind", "PS_Eden_Space") - assertLabels(memoryPoolExpressionLabels, "kind", "Heap") - assertLabels(memoryPoolExpressionLabels, "kind", "PS_Survivor_Space") - assertLabels(memoryPoolExpressionLabels, "kind", "PS_Old_Gen") - } - - test("file_descriptors expression is registered") { - val fileDescriptorsExpressionLabels = expressions - .filter { - case (k, _) => k.name == "file_descriptors" - }.keySet.map(_.labels) - - assertLabels(fileDescriptorsExpressionLabels, "role", "jvm") - } - - test("gc_cycles expression is registered") { - val gcCyclesExpressionLabels = expressions - .filter { - case (k, _) => k.name == "gc_cycles" - }.keySet.map(_.labels) - assert(gcCyclesExpressionLabels.contains(Map("role" -> "jvm", "gc_pool" -> "PS_Scavenge"))) - assert(gcCyclesExpressionLabels.contains(Map("role" -> "jvm", "gc_pool" -> "PS_MarkSweep"))) - } - - test("gc_latency expression is registered") { - val gcLatenciesExpressionLabels = expressions - .filter { - case (k, _) => k.name == "gc_latency" - }.keySet.map(_.labels) - assert(gcLatenciesExpressionLabels.contains(Map("role" -> "jvm", "gc_pool" -> "PS_Scavenge"))) - assert(gcLatenciesExpressionLabels.contains(Map("role" -> "jvm", "gc_pool" -> "PS_MarkSweep"))) - } - - test("jvm_uptime expression is registered") { - val jvmUptimeExpressionLabels = expressions - .filter { - case (k, _) => k.name == "jvm_uptime" - }.keySet.map(_.labels) - - assertLabels(jvmUptimeExpressionLabels, "role", "jvm") + test("expressions are registered") { + assertExpressionLabels("memory_pool") + assertExpressionLabels("file_descriptors") + assertExpressionLabels("gc_cycles") + assertExpressionLabels("gc_latency") + assertExpressionLabels("jvm_uptime") } private[this] def assertMetric(expressionSchema: ExpressionSchema): Unit = assert(expressionSchema.expr.isInstanceOf[MetricExpression]) - private[this] def assertLabels( - labels: scala.collection.Set[Map[String, String]], - key: String, - value: String - ): Unit = - assert(labels.exists(_.get(key).get == value)) + private[this] def assertExpressionLabels(expressionKey: String): Unit = + assert(expressions.keysIterator.exists(_.name == expressionKey)) } From ef6e3fbd79e703865f533020a42973eb5e4be70e Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Tue, 22 Feb 2022 20:21:49 +0000 Subject: [PATCH 629/796] {util,twitter-server}: Make StringExpression higher fidelity Problem StringExpression is currently incompatible with rating because we don't know whether it's backed by a counter or not! Solution Add a boolean as a quick hack to let us know whether the metric is a counter or not. This API may change rapidly as we learn more about how it should be used. JIRA Issues: CSL-11715 Differential Revision: https://phabricator.twitter.biz/D832571 --- .../scala/com/twitter/finagle/stats/exp/Expression.scala | 6 +++--- .../twitter/finagle/stats/exp/ExpressionSchemaTest.scala | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index f1499dddd5..04771beb1b 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -26,7 +26,7 @@ private[twitter] object Expression { case MetricExpression(metricBuilder, _) => Set(metricBuilder.statsReceiver) case HistogramExpression(metricBuilder, _) => Set(metricBuilder.statsReceiver) case ConstantExpression(_) => Set.empty - case StringExpression(_) => Set(LoadedStatsReceiver) + case StringExpression(_, _) => Set(LoadedStatsReceiver) case NoExpression => Set.empty } @@ -82,7 +82,7 @@ private[twitter] object Expression { * * @param exprs a sequence of metrics in strings */ - def apply(exprs: Seq[String]): Expression = StringExpression(exprs) + def apply(exprs: Seq[String], isCounter: Boolean): Expression = StringExpression(exprs, isCounter) } /** @@ -135,7 +135,7 @@ case class HistogramExpression private[exp] ( * Represent an expression constructed from metrics in strings * @see Expression#apply */ -case class StringExpression private[exp] (exprs: Seq[String]) extends Expression +case class StringExpression private[exp] (exprs: Seq[String], isCounter: Boolean) extends Expression case object NoExpression extends Expression { @varargs diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 102c3c4b18..e230064109 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -170,8 +170,10 @@ class ExpressionSchemaTest extends AnyFunSuite { LoadedStatsReceiver.self = sr val successRate = ExpressionSchema( "success_rate", - Expression(100).multiply(Expression(Seq("clnt", "aclient", "success")).divide(Expression( - Seq("clnt", "aclient", "success")).plus(Expression(Seq("clnt", "aclient", "failures"))))) + Expression(100).multiply( + Expression(Seq("clnt", "aclient", "success"), isCounter = true).divide( + Expression(Seq("clnt", "aclient", "success"), isCounter = true).plus( + Expression(Seq("clnt", "aclient", "failures"), isCounter = true)))) ).withRole(Client).withServiceName("aclient").build() assert(sr.expressions.values.size == 1) From 1a0d87c3f2809b6ef23ad87c382dadb14e10d688 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 24 Feb 2022 01:13:00 +0000 Subject: [PATCH 630/796] util-jackson: Use PropertyNamingStrategies for static instances Problem In release 2.12, Jackson introduced `PropertyNamingStrategies` as a work-around for [#2715](https://github.com/FasterXML/jackson-databind/issues/2715) which deprecated using the like `PropertyNamingStrategy` static fields. Solution Move to using the equivalent static fields in `PropertyNamingStrategies`. Result Less deprection tech-debt. Differential Revision: https://phabricator.twitter.biz/D838232 --- .../util/jackson/ScalaObjectMapper.scala | 28 +++++++++---------- .../util/jackson/ScalaObjectMapperTest.scala | 9 +++--- .../twitter/util/jackson/caseclasses.scala | 22 ++++++++------- 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala index 6db1814f41..3925995598 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala @@ -40,7 +40,7 @@ object ScalaObjectMapper { /** The default [[PropertyNamingStrategy]] for a [[ScalaObjectMapper]] */ private[twitter] val DefaultPropertyNamingStrategy: PropertyNamingStrategy = - PropertyNamingStrategy.SNAKE_CASE + PropertyNamingStrategies.SNAKE_CASE /** The default [[JsonInclude.Include]] for serialization for a [[ScalaObjectMapper]] */ private[twitter] val DefaultSerializationInclude: JsonInclude.Include = @@ -125,29 +125,29 @@ object ScalaObjectMapper { /** * Utility to create a new [[ScalaObjectMapper]] explicitly configured with - * [[PropertyNamingStrategy.LOWER_CAMEL_CASE]] as a `PropertyNamingStrategy` wrapping the + * [[PropertyNamingStrategies.LOWER_CAMEL_CASE]] as a `PropertyNamingStrategy` wrapping the * given [[JacksonScalaObjectMapperType]]. * * @note the `underlying` mapper is copied (not mutated) to produce the new [[ScalaObjectMapper]] - * with a [[PropertyNamingStrategy.LOWER_CAMEL_CASE]] PropertyNamingStrategy. + * with a [[PropertyNamingStrategies.LOWER_CAMEL_CASE]] PropertyNamingStrategy. */ def camelCaseObjectMapper(underlying: JacksonScalaObjectMapperType): ScalaObjectMapper = { val objectMapperCopy = ObjectMapperCopier.copy(underlying) - objectMapperCopy.setPropertyNamingStrategy(PropertyNamingStrategy.LOWER_CAMEL_CASE) + objectMapperCopy.setPropertyNamingStrategy(PropertyNamingStrategies.LOWER_CAMEL_CASE) new ScalaObjectMapper(objectMapperCopy) } /** * Utility to create a new [[ScalaObjectMapper]] explicitly configured with - * [[PropertyNamingStrategy.SNAKE_CASE]] as a `PropertyNamingStrategy` wrapping the + * [[PropertyNamingStrategies.SNAKE_CASE]] as a `PropertyNamingStrategy` wrapping the * given [[JacksonScalaObjectMapperType]]. * * @note the `underlying` mapper is copied (not mutated) to produce the new [[ScalaObjectMapper]] - * with a [[PropertyNamingStrategy.SNAKE_CASE]] PropertyNamingStrategy. + * with a [[PropertyNamingStrategies.SNAKE_CASE]] PropertyNamingStrategy. */ def snakeCaseObjectMapper(underlying: JacksonScalaObjectMapperType): ScalaObjectMapper = { val objectMapperCopy = ObjectMapperCopier.copy(underlying) - objectMapperCopy.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) + objectMapperCopy.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) new ScalaObjectMapper(objectMapperCopy) } @@ -158,7 +158,7 @@ object ScalaObjectMapper { * For example, * {{{ * ScalaObjectMapper.builder - * .withPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy) + * .withPropertyNamingStrategy(new PropertyNamingStrategies.UpperCamelCaseStrategy) * .withNumbersAsStrings(true) * .withAdditionalJacksonModules(...) * .objectMapper @@ -169,7 +169,7 @@ object ScalaObjectMapper { * {{{ * val builder = * ScalaObjectMapper.builder - * .withPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy) + * .withPropertyNamingStrategy(new PropertyNamingStrategies.UpperCamelCaseStrategy) * .withNumbersAsStrings(true) * .withAdditionalJacksonModules(...) * @@ -187,7 +187,7 @@ object ScalaObjectMapper { * For example, * {{{ * ScalaObjectMapper.builder - * .withPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy) + * .withPropertyNamingStrategy(new PropertyNamingStrategies.UpperCamelCaseStrategy) * .withNumbersAsStrings(true) * .withAdditionalJacksonModules(...) * .objectMapper @@ -198,7 +198,7 @@ object ScalaObjectMapper { * {{{ * val builder = * ScalaObjectMapper.builder - * .withPropertyNamingStrategy(new PropertyNamingStrategy.UpperCamelCaseStrategy) + * .withPropertyNamingStrategy(new PropertyNamingStrategies.UpperCamelCaseStrategy) * .withNumbersAsStrings(true) * .withAdditionalJacksonModules(...) * @@ -240,14 +240,14 @@ object ScalaObjectMapper { /** * Creates a new [[ScalaObjectMapper]] explicitly configured with - * [[PropertyNamingStrategy.LOWER_CAMEL_CASE]] as a `PropertyNamingStrategy`. + * [[PropertyNamingStrategies.LOWER_CAMEL_CASE]] as a `PropertyNamingStrategy`. */ final def camelCaseObjectMapper: ScalaObjectMapper = ScalaObjectMapper.camelCaseObjectMapper(jacksonScalaObjectMapper) /** * Creates a new [[ScalaObjectMapper]] explicitly configured with - * [[PropertyNamingStrategy.SNAKE_CASE]] as a `PropertyNamingStrategy`. + * [[PropertyNamingStrategies.SNAKE_CASE]] as a `PropertyNamingStrategy`. */ final def snakeCaseObjectMapper: ScalaObjectMapper = ScalaObjectMapper.snakeCaseObjectMapper(jacksonScalaObjectMapper) @@ -256,7 +256,7 @@ object ScalaObjectMapper { /** * Configure a [[PropertyNamingStrategy]] for this [[Builder]]. - * @note the default is [[PropertyNamingStrategy.SNAKE_CASE]] + * @note the default is [[PropertyNamingStrategies.SNAKE_CASE]] * @see [[ScalaObjectMapper.DefaultPropertyNamingStrategy]] */ final def withPropertyNamingStrategy(propertyNamingStrategy: PropertyNamingStrategy): Builder = diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala index 0855c4daf5..11b0a1970f 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.JsonDeserializer import com.fasterxml.jackson.databind.JsonMappingException import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.MapperFeature +import com.fasterxml.jackson.databind.PropertyNamingStrategies import com.fasterxml.jackson.databind.PropertyNamingStrategy import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper} import com.fasterxml.jackson.dataformat.yaml.YAMLFactory @@ -384,7 +385,7 @@ class ScalaObjectMapperTest val jacksonScalaObjectMapper: JacksonScalaObjectMapperType = new JacksonObjectMapper() with JacksonScalaObjectMapper jacksonScalaObjectMapper.registerModule(DefaultScalaModule) - jacksonScalaObjectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.LOWER_CAMEL_CASE) + jacksonScalaObjectMapper.setPropertyNamingStrategy(PropertyNamingStrategies.LOWER_CAMEL_CASE) // default for ScalaObjectMapper#Builder is snake_case which should not get applied here // since this method only wraps and does not mutate the underlying mapper @@ -454,7 +455,7 @@ class ScalaObjectMapperTest // test with default scala module val defaultScalaObjectMapper = new JacksonObjectMapper with JacksonScalaObjectMapper defaultScalaObjectMapper.registerModule(DefaultScalaModule) - defaultScalaObjectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) + defaultScalaObjectMapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) deserialize[BasicDate](defaultScalaObjectMapper, expectedStr) should equal(expectedInstance) // case insensitive mapper feature does not pertain to scala enumerations (only Java enums) @@ -566,7 +567,7 @@ class ScalaObjectMapperTest // test with default scala module val defaultScalaObjectMapper = new JacksonObjectMapper with JacksonScalaObjectMapper defaultScalaObjectMapper.registerModule(DefaultScalaModule) - defaultScalaObjectMapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) + defaultScalaObjectMapper.setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) deserialize[Page[Person]](defaultScalaObjectMapper, input) should equal(result) // test with ScalaObjectMapper @@ -934,7 +935,7 @@ class ScalaObjectMapperTest ) { val objMapper = new JacksonObjectMapper with JacksonScalaObjectMapper objMapper.registerModule(DefaultScalaModule) - objMapper.setPropertyNamingStrategy(new PropertyNamingStrategy.SnakeCaseStrategy) + objMapper.setPropertyNamingStrategy(new PropertyNamingStrategies.SnakeCaseStrategy) val response = objMapper.writeValueAsString(NamingStrategyJsonProperty("abc")) response should equal("""{"long_field_name":"abc"}""") diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala index 0c1f19436d..1915f7a1dc 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala @@ -5,25 +5,27 @@ import com.fasterxml.jackson.annotation._ import com.fasterxml.jackson.core.JsonParser import com.fasterxml.jackson.core.`type`.TypeReference import com.fasterxml.jackson.databind._ -import com.fasterxml.jackson.databind.annotation.{JsonDeserialize, JsonNaming} -import com.fasterxml.jackson.databind.deser.std.NumberDeserializers.{ - BigDecimalDeserializer, - NumberDeserializer -} +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import com.fasterxml.jackson.databind.annotation.JsonNaming +import com.fasterxml.jackson.databind.deser.std.NumberDeserializers.BigDecimalDeserializer +import com.fasterxml.jackson.databind.deser.std.NumberDeserializers.NumberDeserializer import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.fasterxml.jackson.databind.node.ValueNode import com.fasterxml.jackson.module.scala.JsonScalaEnumeration -import com.twitter.util.{Time, WrappedValue} +import com.twitter.util.Time +import com.twitter.util.WrappedValue import com.twitter.util.jackson.caseclass.SerdeLogging import com.twitter.util.validation.MethodValidation -import com.twitter.util.validation.constraints.{OneOf, UUID} +import com.twitter.util.validation.constraints.OneOf +import com.twitter.util.validation.constraints.UUID import com.twitter.util.validation.engine.MethodValidationResult import com.twitter.{util => ctu} import jakarta.validation.Payload import jakarta.validation.constraints._ import java.time.format.DateTimeFormatter import java.time.temporal.Temporal -import java.time.{LocalDate, LocalDateTime} +import java.time.LocalDate +import java.time.LocalDateTime import scala.math.BigDecimal.RoundingMode @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") @@ -829,10 +831,10 @@ case class ItemSomeViews( case class ItemNoDefaultForView(@JsonView(Array(classOf[Views.Public])) name: String) -@JsonNaming(classOf[PropertyNamingStrategy.KebabCaseStrategy]) +@JsonNaming(classOf[PropertyNamingStrategies.KebabCaseStrategy]) case class CaseClassWithKebabCase(pleaseUseKebabCase: Boolean) -@JsonNaming(classOf[PropertyNamingStrategy.KebabCaseStrategy]) +@JsonNaming(classOf[PropertyNamingStrategies.KebabCaseStrategy]) trait KebabCaseMixin case class CaseClassShouldUseKebabCaseFromMixin(willThisGetTheRightCasing: Boolean) From 439cc8333e1597a215f00f1e204f5b6efc64f84e Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 25 Feb 2022 14:23:01 +0000 Subject: [PATCH 631/796] Twitter-oss: Prepare OSS libraries for release 22.2.0 Problem We want to release the next versions of our Twitter OSS libraries 22.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-11693 Differential Revision: https://phabricator.twitter.biz/D840164 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 941547aaca..2511e0c7bd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.2.0 +------ + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index a1bba1be72..f388fa2766 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -40,7 +40,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "22.1.0" +val utilCore = "com.twitter" %% "util-core" % "22.2.0" ``` # Units diff --git a/build.sbt b/build.sbt index d3f82e8c85..bab7ca861b 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.2.0-SNAPSHOT" +val releaseVersion = "22.2.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.1" From 7873d943a4e87950476841784a1cb84cb78036a6 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Fri, 25 Feb 2022 16:40:28 +0000 Subject: [PATCH 632/796] Twitter-oss: update OSS libraries post-release to 22.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.3.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. JIRA Issues: CSL-11693 Differential Revision: https://phabricator.twitter.biz/D840845 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f388fa2766..8b9be22496 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index bab7ca861b..331bbbeaab 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.2.0" +val releaseVersion = "22.3.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.1" From 450449920912e53a02306156d05a4e3c7fbc69f6 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 28 Feb 2022 20:06:25 +0000 Subject: [PATCH 633/796] util-core: Link promises in O(1) CASes instead of O(n) Problem When we link promises, we churn through the internal state updates proportional to the wait queue length of the promise being linked. Every state update is allocations + CAS. Solution Link promises in just one state update (one state object allocation + one CAS) per promise. Differential Revision: https://phabricator.twitter.biz/D832552 --- .../com/twitter/util/PromiseBenchmark.scala | 15 ++ .../main/scala/com/twitter/util/Promise.scala | 182 +++++++++--------- .../scala/com/twitter/util/PromiseTest.scala | 57 ++++++ 3 files changed, 166 insertions(+), 88 deletions(-) diff --git a/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala index 74779e5d88..4103cabfdb 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala @@ -4,12 +4,16 @@ import com.twitter.util.Promise.Detachable import org.openjdk.jmh.annotations._ @State(Scope.Benchmark) +@Warmup(iterations = 5) +@Measurement(iterations = 5) class PromiseBenchmark extends StdBenchAnnotations { private[this] val StringFuture = Future.value("hi") private[this] val Value = Return("okok") + private[this] val NoopCallback: Try[String] => Unit = _ => () + @Benchmark def attached(): Promise[String] = { Promise.attached(StringFuture) @@ -69,6 +73,17 @@ class PromiseBenchmark extends StdBenchAnnotations { state.i += 1 ret } + + @Benchmark + def become(state: PromiseBenchmark.PromiseState): Promise[String] = { + val p = new Promise[String](state.f) + p.respond(NoopCallback) + + val q = new Promise[String] + + q.become(p) // p.link(q) + p + } } object PromiseBenchmark { diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 8833d24223..5d73551b98 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -110,6 +110,12 @@ object Promise { private[util] object WaitQueue { + // Works the best when length(from) < length(to) + @tailrec + final def merge[A](from: WaitQueue[A], to: WaitQueue[A]): WaitQueue[A] = + if (from eq Empty) to + else merge(from.rest, WaitQueue(from.first, to)) + object Empty extends WaitQueue[Nothing] { final def first: K[Nothing] = throw new IllegalStateException("WaitQueue.Empty") @@ -429,14 +435,6 @@ object Promise { class Promise[A] extends Future[A] with Updatable[Try[A]] { import Promise._ - private final def continueAll(wq: WaitQueue[A]): Unit = { - var ks = wq - while (ks ne WaitQueue.Empty) { - continue(ks.first) - ks = ks.rest - } - } - /** * Note: exceptions in responds are monitored. That is, if the * computation `k` throws a raw (ie. not encoded in a Future) @@ -510,27 +508,37 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { * * @param f the new interrupt handler */ + final def setInterruptHandler(f: PartialFunction[Throwable, Unit]): Unit = + setInterruptHandler0(f, WaitQueue.empty) + @tailrec - final def setInterruptHandler(f: PartialFunction[Throwable, Unit]): Unit = state match { - case waitq: WaitQueue[A] => - if (!cas(waitq, new Interruptible(waitq, f, Local.save()))) - setInterruptHandler(f) + private final def setInterruptHandler0( + f: PartialFunction[Throwable, Unit], + wq: WaitQueue[A] + ): Unit = + state match { + case waitq: WaitQueue[A] => + if (!cas(waitq, new Interruptible(WaitQueue.merge(wq, waitq), f, Local.save()))) + setInterruptHandler0(f, wq) - case s: Interruptible[A] => - if (!cas(s, new Interruptible(s.waitq, f, Local.save()))) - setInterruptHandler(f) + case s: Interruptible[A] => + if (!cas(s, new Interruptible(WaitQueue.merge(wq, s.waitq), f, Local.save()))) + setInterruptHandler0(f, wq) - case s: Transforming[A] => - if (!cas(s, new Interruptible(s.waitq, f, Local.save()))) - setInterruptHandler(f) + case s: Transforming[A] => + if (!cas(s, new Interruptible(WaitQueue.merge(wq, s.waitq), f, Local.save()))) + setInterruptHandler0(f, wq) - case s: Interrupted[A] => - f.applyOrElse(s.signal, Promise.AlwaysUnit) + case s: Interrupted[A] => + if ((wq ne WaitQueue.Empty) && + !cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), s.signal))) + setInterruptHandler0(f, wq) + else f.applyOrElse(s.signal, Promise.AlwaysUnit) - case _: Try[A] /* Done */ => // ignore + case t: Try[A] /* Done */ => wq.runInScheduler(t) - case p: Promise[A] /* Linked */ => p.setInterruptHandler(f) - } + case p: Promise[A] /* Linked */ => p.setInterruptHandler0(f, wq) + } // Useful for debugging waitq. private[util] def waitqLength: Int = state match { @@ -550,45 +558,52 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { * * @param other the Future to which interrupts are forwarded. */ - @tailrec final def forwardInterruptsTo(other: Future[_]): Unit = { - // This reduces allocations in the common case. - if (other.isDefined) return - state match { - case waitq: WaitQueue[A] => - if (!cas(waitq, new Transforming(waitq, other))) - forwardInterruptsTo(other) + final def forwardInterruptsTo(other: Future[_]): Unit = + forwardInterruptsTo0(other, WaitQueue.empty) - case s: Interruptible[A] => - if (!cas(s, new Transforming(s.waitq, other))) - forwardInterruptsTo(other) - - case s: Transforming[A] => - if (!cas(s, new Transforming(s.waitq, other))) - forwardInterruptsTo(other) - - case s: Interrupted[_] => - other.raise(s.signal) - - case _: Try[A] /* Done */ => () // ignore - - case p: Promise[A] /* Linked */ => p.forwardInterruptsTo(other) - } + @tailrec private final def forwardInterruptsTo0(other: Future[_], wq: WaitQueue[A]): Unit = { + // This reduces allocations in the common case. + if (other.isDefined) continue(wq) + else + state match { + case waitq: WaitQueue[A] => + if (!cas(waitq, new Transforming(WaitQueue.merge(wq, waitq), other))) + forwardInterruptsTo0(other, wq) + + case s: Interruptible[A] => + if (!cas(s, new Transforming(WaitQueue.merge(wq, s.waitq), other))) + forwardInterruptsTo0(other, wq) + + case s: Transforming[A] => + if (!cas(s, new Transforming(WaitQueue.merge(wq, s.waitq), other))) + forwardInterruptsTo0(other, wq) + + case s: Interrupted[_] => + if ((wq ne WaitQueue.Empty) && + !cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), s.signal))) + forwardInterruptsTo0(other, wq) + else other.raise(s.signal) + + case t: Try[A] /* Done */ => wq.runInScheduler(t) + + case p: Promise[A] /* Linked */ => p.forwardInterruptsTo0(other, wq) + } } - final def raise(intr: Throwable): Unit = raiseImpl(intr) + final def raise(intr: Throwable): Unit = raise0(intr, WaitQueue.empty) // This is implemented as a private method on Promise so that the Scala 3 // compiler recognizes this as a tail-recursive implementation. `raise` is // a definition on `Future` instead of `Promise`, which the Scala 3 // compiler errors on with @tailrec annocation on the `Transforming` case, // where `s.other.raise` is invoked on a Future and not Promise. See CSL-11192. - @tailrec private def raiseImpl(intr: Throwable): Unit = state match { + @tailrec private final def raise0(intr: Throwable, wq: WaitQueue[A]): Unit = state match { case waitq: WaitQueue[A] => - if (!cas(waitq, new Interrupted(waitq, intr))) - raiseImpl(intr) + if (!cas(waitq, new Interrupted(WaitQueue.merge(wq, waitq), intr))) + raise0(intr, wq) case s: Interruptible[A] => - if (!cas(s, new Interrupted(s.waitq, intr))) raiseImpl(intr) + if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), intr))) raise0(intr, wq) else { val current = Local.save() if (current ne s.saved) @@ -598,18 +613,16 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { } case s: Transforming[A] => - if (!cas(s, new Interrupted(s.waitq, intr))) raiseImpl(intr) - else { - s.other.raise(intr) - } + if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), intr))) raise0(intr, wq) + else s.other.raise(intr) case s: Interrupted[A] => - if (!cas(s, new Interrupted(s.waitq, intr))) - raiseImpl(intr) + if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), intr))) + raise0(intr, wq) - case _: Try[A] /* Done */ => () // nothing to do, as its already satisfied. + case t: Try[A] /* Done */ => wq.runInScheduler(t) - case p: Promise[A] /* Linked */ => p.raiseImpl(intr) + case p: Promise[A] /* Linked */ => p.raise0(intr, wq) } @tailrec protected[Promise] final def detach(k: K[A]): Boolean = state match { @@ -662,7 +675,7 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { if (condition.await(timeout.inNanoseconds, java.util.concurrent.TimeUnit.NANOSECONDS)) this else throw new TimeoutException(timeout.toString) - case res: Try[A] /* Done */ => this + case _: Try[A] /* Done */ => this case p: Promise[A] /* Linked */ => p.ready(timeout); this } @@ -823,22 +836,24 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { } @tailrec - protected final def continue(k: K[A]): Unit = state match { - case waitq: WaitQueue[A] => - if (!cas(waitq, WaitQueue(k, waitq))) - continue(k) - case s: Interruptible[A] => - if (!cas(s, new Interruptible(WaitQueue(k, s.waitq), s.handler, s.saved))) - continue(k) - case s: Transforming[A] => - if (!cas(s, new Transforming(WaitQueue(k, s.waitq), s.other))) - continue(k) - case s: Interrupted[A] => - if (!cas(s, new Interrupted(WaitQueue(k, s.waitq), s.signal))) - continue(k) - case v: Try[A] /* Done */ => k.runInScheduler(v) - case p: Promise[A] /* Linked */ => p.continue(k) - } + protected final def continue(wq: WaitQueue[A]): Unit = + if (wq ne WaitQueue.Empty) + state match { + case waitq: WaitQueue[A] => + if (!cas(waitq, WaitQueue.merge(wq, waitq))) + continue(wq) + case s: Interruptible[A] => + if (!cas(s, new Interruptible(WaitQueue.merge(wq, s.waitq), s.handler, s.saved))) + continue(wq) + case s: Transforming[A] => + if (!cas(s, new Transforming(WaitQueue.merge(wq, s.waitq), s.other))) + continue(wq) + case s: Interrupted[A] => + if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), s.signal))) + continue(wq) + case v: Try[A] /* Done */ => wq.runInScheduler(v) + case p: Promise[A] /* Linked */ => p.continue(wq) + } /** * Should only be called when this Promise has already been fulfilled @@ -862,28 +877,19 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { state match { case waitq: WaitQueue[A] => if (!cas(waitq, target)) link(target) - else target.continueAll(waitq) + else target.continue(waitq) case s: Interruptible[A] => if (!cas(s, target)) link(target) - else { - target.continueAll(s.waitq) - target.setInterruptHandler(s.handler) - } + else target.setInterruptHandler0(s.handler, s.waitq) case s: Transforming[A] => if (!cas(s, target)) link(target) - else { - target.continueAll(s.waitq) - target.forwardInterruptsTo(s.other) - } + else target.forwardInterruptsTo0(s.other, s.waitq) case s: Interrupted[A] => if (!cas(s, target)) link(target) - else { - target.continueAll(s.waitq) - target.raise(s.signal) - } + else target.raise0(s.signal, s.waitq) case value: Try[A] /* Done */ => if (!target.updateIfEmpty(value) && value != Await.result(target)) { diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index 430399383a..88e2aeb213 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -3,10 +3,15 @@ package com.twitter.util import com.twitter.concurrent.Scheduler import com.twitter.concurrent.LocalScheduler import com.twitter.conversions.DurationOps._ +import java.util.concurrent.atomic.AtomicBoolean import org.scalatest.funsuite.AnyFunSuite class PromiseTest extends AnyFunSuite { + final class LinkablePromise[A] extends Promise[A] { + def link1(target: Promise[A]): Unit = link(target) + } + class TrackingScheduler extends LocalScheduler(false) { @volatile var submitted: Int = 0 override def submit(r: Runnable): Unit = { @@ -104,6 +109,58 @@ class PromiseTest extends AnyFunSuite { assert(ex.getMessage.contains(value)) } + test("interrupted.link(done) runs interrupted waitqueue") { + val q = new LinkablePromise[String] // interrupted with a waitqueue + val run = new AtomicBoolean(false) + q.respond(_ => run.set(true)) + q.raise(new Exception("boom")) + + val p = new Promise[String](Return("foo")) // satisfied + + q.link1(p) + assert(run.get()) + } + + test("transforming(never).link(done) runs transforming waitqueue") { + val q = new LinkablePromise[String] // interrupted with a waitqueue + val deadend = Future.never + val run = new AtomicBoolean(false) + q.respond(_ => run.set(true)) + q.forwardInterruptsTo(deadend) + + val p = new Promise[String](Return("foo")) // satisfied + + q.link1(p) + assert(run.get()) + } + + test("transforming(done).link(done) runs transforming waitqueue") { + val q = new LinkablePromise[String] // interrupted with a waitqueue + val deadend = new Promise[Unit] + val run = new AtomicBoolean(false) + q.respond(_ => run.set(true)) + q.forwardInterruptsTo(deadend) + + val p = new Promise[String](Return("foo")) // satisfied + deadend.setDone() + + q.link1(p) + assert(run.get()) + } + + test("interruptible.link(done) runs interruptible waitqueue") { + val q = new LinkablePromise[String] // interrupted with a waitqueue + val deadend = Future.Done + val run = new AtomicBoolean(false) + q.respond(_ => run.set(true)) + q.setInterruptHandler { case _ => () } + + val p = new Promise[String](Return("foo")) // satisfied + + q.link1(p) + assert(run.get()) + } + test("Updating a Promise more than once should fail") { val p = new Promise[Int]() val first = Return(1) From 19dce981e7f647e7c4a432fa9deed0828dc9c83d Mon Sep 17 00:00:00 2001 From: Lily He Date: Wed, 2 Mar 2022 17:38:44 +0000 Subject: [PATCH 634/796] finagle-stats-core: Export counters in Prometheus format Problem: We want to export Finagle counters in Prometheus format. This format allows Finagle to have rich (dimensional) metrics where metrics have labels attached to them. Solution: Encode counters name, value and their labels in the Prometheus format. Optionally write metadata of the type of metric and the units that the metric counts in. Write the metric to a StringBuilder. JIRA Issues: CSL-11562 Differential Revision: https://phabricator.twitter.biz/D839136 --- .../twitter/finagle/stats/MetricBuilder.scala | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 9623225ba8..ab1a7cb283 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -98,14 +98,32 @@ object MetricBuilder { * @note [[CounterishGaugeType]] will also create a [[Gauge]], but specifically one that * models a [[Counter]]. */ - sealed trait MetricType - case object CounterType extends MetricType - case object CounterishGaugeType extends MetricType - case object GaugeType extends MetricType - case object HistogramType extends MetricType + sealed trait MetricType { + def toPrometheusString: String + def toJsonString: String + } + case object CounterType extends MetricType { + def toJsonString: String = "counter" + def toPrometheusString: String = toJsonString + } + case object CounterishGaugeType extends MetricType { + def toJsonString: String = "counterish_gauge" + def toPrometheusString: String = "gauge" + } + case object GaugeType extends MetricType { + def toJsonString: String = "gauge" + def toPrometheusString: String = toJsonString + } + case object HistogramType extends MetricType { + def toJsonString: String = "histogram" + def toPrometheusString: String = toJsonString + } /** Counter type that will be always unlatched */ - case object UnlatchedCounter extends MetricType + case object UnlatchedCounter extends MetricType { + def toJsonString: String = "unlatched_counter" + def toPrometheusString: String = "counter" + } } sealed trait Metadata { From b000bcb546d5630beca8f1293dfcd71aee8129ad Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 3 Mar 2022 21:59:23 +0000 Subject: [PATCH 635/796] util-stats/finagle-stats-core: Start laying the groundwork for dimensional metrics Problem We want to start supporting the creation of dimensional metrics but we don't currently have a way to store labels and no way to thread them through to our metric exporters. Solution Add some types that will allow MetricsView to emit more data about what the metric was that this snapshot represents so our exporters can emit metrics however they like. Differential Revision: https://phabricator.twitter.biz/D844565 --- .../com/twitter/finagle/stats/MetricBuilder.scala | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index ab1a7cb283..6cfeaafdca 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -67,6 +67,7 @@ object MetricBuilder { verbosity: Verbosity = Verbosity.Default, sourceClass: Option[String] = None, name: Seq[String] = Seq.empty, + labels: Map[String, String] = Map.empty, relativeName: Seq[String] = Seq.empty, processPath: Option[String] = None, percentiles: IndexedSeq[Double] = IndexedSeq.empty, @@ -82,6 +83,7 @@ object MetricBuilder { verbosity, sourceClass, name, + labels, relativeName, processPath, percentiles, @@ -158,6 +160,7 @@ class MetricBuilder private ( val verbosity: Verbosity, val sourceClass: Option[String], val name: Seq[String], + val labels: Map[String, String], val relativeName: Seq[String], val processPath: Option[String], // Only persisted and relevant when building histograms. @@ -194,6 +197,7 @@ class MetricBuilder private ( verbosity = verbosity, sourceClass = sourceClass, name = name, + labels = labels, relativeName = relativeName, processPath = processPath, percentiles = percentiles, @@ -322,7 +326,7 @@ class MetricBuilder private ( override def equals(other: Any): Boolean = other match { case that: MetricBuilder => - (that.canEqual(this)) && + that.canEqual(this) && keyIndicator == that.keyIndicator && description == that.description && units == that.units && @@ -330,6 +334,7 @@ class MetricBuilder private ( verbosity == that.verbosity && sourceClass == that.sourceClass && name == that.name && + labels == that.labels && relativeName == that.relativeName && processPath == that.processPath && percentiles == that.percentiles && @@ -339,19 +344,20 @@ class MetricBuilder private ( override def hashCode(): Int = { val state = - Seq[AnyRef]( + Array[AnyRef]( description, units, role, verbosity, sourceClass, name, + labels, relativeName, processPath, percentiles, metricType) - val hashCodes = keyIndicator.hashCode() +: state.map(_.hashCode()) - hashCodes.foldLeft(0)((a, b) => 31 * a + b) + + state.foldLeft(keyIndicator.hashCode())((a, b) => 31 * a + b.hashCode()) } override def toString(): String = { From 98a5b1c8c01b0e47c933c22c9fe3b67e1852f75f Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 3 Mar 2022 23:22:44 +0000 Subject: [PATCH 636/796] util-core: More optimal Promise state match order Problem / Solution We can optimize the order in which we match against Promise state to ensure the most frequent state (as observed from instrumentation a production service) is matched earlier than the least frequent state. Differential Revision: https://phabricator.twitter.biz/D843165 --- .../main/scala/com/twitter/util/Promise.scala | 172 +++++++++--------- 1 file changed, 91 insertions(+), 81 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 5d73551b98..c97f0df42e 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -287,15 +287,6 @@ object Promise { } } - /* - * Implementation notes: - * - * While these various states for `Promises.state` should be nicely modeled - * as a sealed type, by omitting the wrappers around `Done` and `Linked` - * we are able to save significant number of allocations for slightly - * more unreadable code localized within `Promise`. - */ - /** * An unsatisfied [[Promise]] which has an interrupt handler attached to it. * `waitq` represents the continuations that should be run once it @@ -462,13 +453,25 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { promise } - // Note: this will always be one of: - // - WaitQueue (Waiting) - // - Interrupted - // - Interruptible - // - Transforming - // - Try[A] (Done) - // - Promise[A] + // Promise state encoding notes: + // + // There are six (6) possible promise states that we encode as raw objects (no state class + // wrappers) for the sake of reducing allocations. This is why, for example, state "Linked" + // is encoded as directly 'Promise[_]' and not as 'case class Linked(to: Promise[_])'. + // + // When we transition between states, we take into consideration the order in which we match + // against the possible state cases. To determine the most optimal match oder, we instrumented + // the Promise code to keep track of the histogram of the most frequent states. + // + // Here is the data we've collected from one of our production services: + // - Transforming: 73% + // - Waiting: 13% + // - Done: 8% + // - Interruptible: 4% + // - Linked: 1% + // - Interrupted: <1% + // + // We use this exact match order (from most frequent to less frequent) in the Promise code. @volatile private[this] var state: Any = WaitQueue.empty[A] private def theState(): Any = state @@ -489,12 +492,12 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { override def toString: String = { val theState = state match { - case waitq: Promise.WaitQueue[A] => s"Waiting($waitq)" - case s: Interruptible[A] => s"Interruptible(${s.waitq},${s.handler})" case s: Transforming[A] => s"Transforming(${s.waitq},${s.other})" - case s: Interrupted[A] => s"Interrupted(${s.waitq},${s.signal})" + case waitq: Promise.WaitQueue[A] => s"Waiting($waitq)" case res: Try[A] => s"Done($res)" + case s: Interruptible[A] => s"Interruptible(${s.waitq},${s.handler})" case p: Promise[A] => s"Linked(${p.toString})" + case s: Interrupted[A] => s"Interrupted(${s.waitq},${s.signal})" } s"Promise@$hashCode(state=$theState)" } @@ -517,37 +520,37 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { wq: WaitQueue[A] ): Unit = state match { + case s: Transforming[A] => + if (!cas(s, new Interruptible(WaitQueue.merge(wq, s.waitq), f, Local.save()))) + setInterruptHandler0(f, wq) + case waitq: WaitQueue[A] => if (!cas(waitq, new Interruptible(WaitQueue.merge(wq, waitq), f, Local.save()))) setInterruptHandler0(f, wq) + case t: Try[A] /* Done */ => wq.runInScheduler(t) + case s: Interruptible[A] => if (!cas(s, new Interruptible(WaitQueue.merge(wq, s.waitq), f, Local.save()))) setInterruptHandler0(f, wq) - case s: Transforming[A] => - if (!cas(s, new Interruptible(WaitQueue.merge(wq, s.waitq), f, Local.save()))) - setInterruptHandler0(f, wq) + case p: Promise[A] /* Linked */ => p.setInterruptHandler0(f, wq) case s: Interrupted[A] => if ((wq ne WaitQueue.Empty) && !cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), s.signal))) setInterruptHandler0(f, wq) else f.applyOrElse(s.signal, Promise.AlwaysUnit) - - case t: Try[A] /* Done */ => wq.runInScheduler(t) - - case p: Promise[A] /* Linked */ => p.setInterruptHandler0(f, wq) } // Useful for debugging waitq. private[util] def waitqLength: Int = state match { - case waitq: WaitQueue[A] => waitq.size - case s: Interruptible[A] => s.waitq.size case s: Transforming[A] => s.waitq.size - case s: Interrupted[A] => s.waitq.size + case waitq: WaitQueue[A] => waitq.size case _: Try[A] /* Done */ => 0 + case s: Interruptible[A] => s.waitq.size case _: Promise[A] /* Linked */ => 0 + case s: Interrupted[A] => s.waitq.size } /** @@ -566,27 +569,27 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { if (other.isDefined) continue(wq) else state match { + case s: Transforming[A] => + if (!cas(s, new Transforming(WaitQueue.merge(wq, s.waitq), other))) + forwardInterruptsTo0(other, wq) + case waitq: WaitQueue[A] => if (!cas(waitq, new Transforming(WaitQueue.merge(wq, waitq), other))) forwardInterruptsTo0(other, wq) + case t: Try[A] /* Done */ => wq.runInScheduler(t) + case s: Interruptible[A] => if (!cas(s, new Transforming(WaitQueue.merge(wq, s.waitq), other))) forwardInterruptsTo0(other, wq) - case s: Transforming[A] => - if (!cas(s, new Transforming(WaitQueue.merge(wq, s.waitq), other))) - forwardInterruptsTo0(other, wq) + case p: Promise[A] /* Linked */ => p.forwardInterruptsTo0(other, wq) case s: Interrupted[_] => if ((wq ne WaitQueue.Empty) && !cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), s.signal))) forwardInterruptsTo0(other, wq) else other.raise(s.signal) - - case t: Try[A] /* Done */ => wq.runInScheduler(t) - - case p: Promise[A] /* Linked */ => p.forwardInterruptsTo0(other, wq) } } @@ -598,10 +601,16 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { // compiler errors on with @tailrec annocation on the `Transforming` case, // where `s.other.raise` is invoked on a Future and not Promise. See CSL-11192. @tailrec private final def raise0(intr: Throwable, wq: WaitQueue[A]): Unit = state match { + case s: Transforming[A] => + if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), intr))) raise0(intr, wq) + else s.other.raise(intr) + case waitq: WaitQueue[A] => if (!cas(waitq, new Interrupted(WaitQueue.merge(wq, waitq), intr))) raise0(intr, wq) + case t: Try[A] /* Done */ => wq.runInScheduler(t) + case s: Interruptible[A] => if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), intr))) raise0(intr, wq) else { @@ -612,54 +621,48 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { finally Local.restore(current) } - case s: Transforming[A] => - if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), intr))) raise0(intr, wq) - else s.other.raise(intr) + case p: Promise[A] /* Linked */ => p.raise0(intr, wq) case s: Interrupted[A] => if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), intr))) raise0(intr, wq) - - case t: Try[A] /* Done */ => wq.runInScheduler(t) - - case p: Promise[A] /* Linked */ => p.raise0(intr, wq) } @tailrec protected[Promise] final def detach(k: K[A]): Boolean = state match { + case s: Transforming[A] => + if (!cas(s, new Transforming(s.waitq.remove(k), s.other))) + detach(k) + else + s.waitq.contains(k) + case waitq: WaitQueue[A] => if (!cas(waitq, waitq.remove(k))) detach(k) else waitq.contains(k) + case _: Try[A] /* Done */ => false + case s: Interruptible[A] => if (!cas(s, new Interruptible(s.waitq.remove(k), s.handler, s.saved))) detach(k) else s.waitq.contains(k) - case s: Transforming[A] => - if (!cas(s, new Transforming(s.waitq.remove(k), s.other))) - detach(k) - else - s.waitq.contains(k) + case p: Promise[A] /* Linked */ => p.detach(k) case s: Interrupted[A] => if (!cas(s, new Interrupted(s.waitq.remove(k), s.signal))) detach(k) else s.waitq.contains(k) - - case _: Try[A] /* Done */ => false - - case p: Promise[A] /* Linked */ => p.detach(k) } // Awaitable @throws(classOf[TimeoutException]) @throws(classOf[InterruptedException]) def ready(timeout: Duration)(implicit permit: Awaitable.CanAwait): this.type = state match { - case _: WaitQueue[A] | _: Interruptible[A] | _: Interrupted[A] | _: Transforming[A] => + case _: Transforming[A] | _: WaitQueue[A] | _: Interruptible[A] | _: Interrupted[A] => val condition = new ReleaseOnApplyCDL[A] respond(condition) @@ -693,10 +696,10 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { * Returns this promise's interrupt if it is interrupted. */ def isInterrupted: Option[Throwable] = state match { - case _: WaitQueue[A] | _: Interruptible[A] | _: Transforming[A] => None - case s: Interrupted[A] => Some(s.signal) + case _: Transforming[A] | _: WaitQueue[A] | _: Interruptible[A] => None case _: Try[A] /* Done */ => None case p: Promise[A] /* Linked */ => p.isInterrupted + case s: Interrupted[A] => Some(s.signal) } /** @@ -804,6 +807,13 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { */ @tailrec final def updateIfEmpty(result: Try[A]): Boolean = state match { + case s: Transforming[A] => + if (!cas(s, result)) updateIfEmpty(result) + else { + s.waitq.runInScheduler(result) + true + } + case waitq: WaitQueue[A] => if (!cas(waitq, result)) updateIfEmpty(result) else { @@ -811,48 +821,48 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { true } + case _: Try[A] /* Done */ => false + case s: Interruptible[A] => if (!cas(s, result)) updateIfEmpty(result) else { s.waitq.runInScheduler(result) true } - case s: Transforming[A] => - if (!cas(s, result)) updateIfEmpty(result) - else { - s.waitq.runInScheduler(result) - true - } + + case p: Promise[A] /* Linked */ => p.updateIfEmpty(result) + case s: Interrupted[A] => if (!cas(s, result)) updateIfEmpty(result) else { s.waitq.runInScheduler(result) true } - - case _: Try[A] /* Done */ => false - - case p: Promise[A] /* Linked */ => p.updateIfEmpty(result) } @tailrec protected final def continue(wq: WaitQueue[A]): Unit = if (wq ne WaitQueue.Empty) state match { + case s: Transforming[A] => + if (!cas(s, new Transforming(WaitQueue.merge(wq, s.waitq), s.other))) + continue(wq) + case waitq: WaitQueue[A] => if (!cas(waitq, WaitQueue.merge(wq, waitq))) continue(wq) + + case v: Try[A] /* Done */ => wq.runInScheduler(v) + case s: Interruptible[A] => if (!cas(s, new Interruptible(WaitQueue.merge(wq, s.waitq), s.handler, s.saved))) continue(wq) - case s: Transforming[A] => - if (!cas(s, new Transforming(WaitQueue.merge(wq, s.waitq), s.other))) - continue(wq) + + case p: Promise[A] /* Linked */ => p.continue(wq) + case s: Interrupted[A] => if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), s.signal))) continue(wq) - case v: Try[A] /* Done */ => wq.runInScheduler(v) - case p: Promise[A] /* Linked */ => p.continue(wq) } /** @@ -875,30 +885,30 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { if (this eq target) return state match { - case waitq: WaitQueue[A] => - if (!cas(waitq, target)) link(target) - else target.continue(waitq) - - case s: Interruptible[A] => - if (!cas(s, target)) link(target) - else target.setInterruptHandler0(s.handler, s.waitq) - case s: Transforming[A] => if (!cas(s, target)) link(target) else target.forwardInterruptsTo0(s.other, s.waitq) - case s: Interrupted[A] => - if (!cas(s, target)) link(target) - else target.raise0(s.signal, s.waitq) + case waitq: WaitQueue[A] => + if (!cas(waitq, target)) link(target) + else target.continue(waitq) case value: Try[A] /* Done */ => if (!target.updateIfEmpty(value) && value != Await.result(target)) { throw new IllegalArgumentException("Cannot link two Done Promises with differing values") } + case s: Interruptible[A] => + if (!cas(s, target)) link(target) + else target.setInterruptHandler0(s.handler, s.waitq) + case p: Promise[A] /* Linked */ => if (cas(p, target)) p.link(target) else link(target) + + case s: Interrupted[A] => + if (!cas(s, target)) link(target) + else target.raise0(s.signal, s.waitq) } } From 88184c21a38a86f946167d13c8aa71ed716a5193 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 7 Mar 2022 19:25:56 +0000 Subject: [PATCH 637/796] util-stats: Some improvements around MetricBuilder to make it more java friendly Problem The MetricBuilder api was designed around scala where the named and defaulted arguments in the `apply` constructor function are nice to use. However, those niceties don't work in java. Solution We already have a bunch of `.withX` methods that form a builder pattern on MetricBuilder. To make them useful for java folks we add a simpler constructor function. We also add some aliases for the `MetricType` family of types that are easier to get at from java. We also move the compile tests to a package that is outside of `c.t.finagle.stats` so we can be sure that the visibility of classes is usable from java code that may live in other packages. Differential Revision: https://phabricator.twitter.biz/D845297 --- util-stats/src/main/java/BUILD | 6 ++++ .../main/java/com/twitter/finagle/stats/BUILD | 13 +++++++++ .../twitter/finagle/stats/MetricTypes.java | 20 +++++++++++++ .../twitter/finagle/stats/MetricBuilder.scala | 15 ++++++++++ util-stats/src/test/java/BUILD | 1 + .../MetricBuilderCompilationTest.java | 28 +++++++++++++++---- .../StatsReceiverCompilationTest.java | 14 ++++++++-- 7 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 util-stats/src/main/java/BUILD create mode 100644 util-stats/src/main/java/com/twitter/finagle/stats/BUILD create mode 100644 util-stats/src/main/java/com/twitter/finagle/stats/MetricTypes.java rename util-stats/src/test/java/com/twitter/{finagle/stats => compiletest}/MetricBuilderCompilationTest.java (52%) rename util-stats/src/test/java/com/twitter/{finagle/stats => compiletest}/StatsReceiverCompilationTest.java (86%) diff --git a/util-stats/src/main/java/BUILD b/util-stats/src/main/java/BUILD new file mode 100644 index 0000000000..d046c97a6d --- /dev/null +++ b/util-stats/src/main/java/BUILD @@ -0,0 +1,6 @@ +target( + tags = ["bazel-compatible"], + dependencies = [ + "util/util-stats/src/main/java/com/twitter/finagle/stats", + ], +) diff --git a/util-stats/src/main/java/com/twitter/finagle/stats/BUILD b/util-stats/src/main/java/com/twitter/finagle/stats/BUILD new file mode 100644 index 0000000000..148f15ebad --- /dev/null +++ b/util-stats/src/main/java/com/twitter/finagle/stats/BUILD @@ -0,0 +1,13 @@ +java_library( + sources = ["*.java"], + compiler_option_sets = ["fatal_warnings"], + provides = artifact( + org = "com.twitter", + name = "util-stats-java", + repo = artifactory, + ), + tags = ["bazel-compatible"], + dependencies = [ + "util/util-stats", + ], +) diff --git a/util-stats/src/main/java/com/twitter/finagle/stats/MetricTypes.java b/util-stats/src/main/java/com/twitter/finagle/stats/MetricTypes.java new file mode 100644 index 0000000000..eed98aedb8 --- /dev/null +++ b/util-stats/src/main/java/com/twitter/finagle/stats/MetricTypes.java @@ -0,0 +1,20 @@ +package com.twitter.finagle.stats; + +/** + * Java APIs for {@link MetricBuilder.MetricType} + */ +public class MetricTypes { + private MetricTypes() { + throw new IllegalStateException(); + } + + public static final MetricBuilder.MetricType COUNTER_TYPE = MetricBuilder.CounterType$.MODULE$; + + public static final MetricBuilder.MetricType UNLATCHED_COUNTER_TYPE = MetricBuilder.UnlatchedCounter$.MODULE$; + + public static final MetricBuilder.MetricType COUNTERISH_GAUGE_TYPE = MetricBuilder.CounterishGaugeType$.MODULE$; + + public static final MetricBuilder.MetricType GAUGE_TYPE = MetricBuilder.GaugeType$.MODULE$; + + public static final MetricBuilder.MetricType HISTOGRAM_TYPE = MetricBuilder.HistogramType$.MODULE$; +} diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 6cfeaafdca..a8e6988fda 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -57,6 +57,8 @@ object MetricBuilder { * @param relativeName the relative metric name which will be appended to the scope of the StatsReceiver prior to long term storage * @param processPath a universal coordinate for the resource * @param percentiles used to indicate buckets for histograms, to be set by the StatsReceiver + * @param metricType the type of the metric being constructed + * @param isStandard whether the metric should the standard path separator or defer to the metrics store for the separator * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder */ def apply( @@ -93,6 +95,19 @@ object MetricBuilder { ) } + /** + * Construct a new `MetricBuilder` + * + * Simpler method for constructing a `MetricBuilder`. This API is a little more friendly for java + * users as they can bootstrap a `MetricBuilder` and then proceed to use with `.with` API's rather + * than being forced to use the mega apply method. + * + * @param metricType the type of the metric being constructed. + * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder. + */ + def newBuilder(metricType: MetricType, statsReceiver: StatsReceiver): MetricBuilder = + apply(metricType = metricType, statsReceiver = statsReceiver) + /** * Indicate the Metric type, [[CounterType]] will create a [[Counter]], * [[GaugeType]] will create a standard [[Gauge]], and [[HistogramType]] will create a [[Stat]]. diff --git a/util-stats/src/test/java/BUILD b/util-stats/src/test/java/BUILD index ecf328608b..3a247c28f1 100644 --- a/util-stats/src/test/java/BUILD +++ b/util-stats/src/test/java/BUILD @@ -10,6 +10,7 @@ junit_tests( "3rdparty/jvm/junit", "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", + "util/util-stats/src/main/java", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], ) diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java b/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java similarity index 52% rename from util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java rename to util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java index fe53eb8508..1ed057a60b 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/MetricBuilderCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java @@ -1,23 +1,39 @@ -package com.twitter.finagle.stats; +package com.twitter.compiletest; import org.junit.Test; import scala.Some; +import com.twitter.finagle.stats.Counter; +import com.twitter.finagle.stats.Gauge; +import com.twitter.finagle.stats.InMemoryStatsReceiver; +import com.twitter.finagle.stats.MetricBuilder; +import com.twitter.finagle.stats.MetricTypes; +import com.twitter.finagle.stats.Microseconds; +import com.twitter.finagle.stats.NoRoleSpecified; +import com.twitter.finagle.stats.Stat; +import com.twitter.finagle.stats.StatsReceiver; +import com.twitter.finagle.stats.Verbosity; + /** * Java compatibility test for {@link com.twitter.finagle.stats.MetricBuilder}. */ public final class MetricBuilderCompilationTest { + @Test public void testMetricBuilderConstruction() { StatsReceiver sr = new InMemoryStatsReceiver(); - MetricBuilder mb = sr.metricBuilder(MetricBuilder.CounterType$.MODULE$); + sr.metricBuilder(MetricTypes.COUNTER_TYPE); + sr.metricBuilder(MetricTypes.UNLATCHED_COUNTER_TYPE); + sr.metricBuilder(MetricTypes.COUNTERISH_GAUGE_TYPE); + sr.metricBuilder(MetricTypes.GAUGE_TYPE); + sr.metricBuilder(MetricTypes.HISTOGRAM_TYPE); } @Test public void testWithMethods() { StatsReceiver sr = new InMemoryStatsReceiver(); - MetricBuilder mb = sr.metricBuilder(MetricBuilder.CounterType$.MODULE$) + MetricBuilder mb = sr.metricBuilder(MetricTypes.COUNTER_TYPE) .withKeyIndicator(true) .withDescription("my cool metric") .withVerbosity(Verbosity.Debug()) @@ -33,9 +49,9 @@ public void testWithMethods() { @Test public void testConstructingMetrics() { StatsReceiver sr = new InMemoryStatsReceiver(); - MetricBuilder gmb = sr.metricBuilder(MetricBuilder.GaugeType$.MODULE$); - MetricBuilder smb = sr.metricBuilder(MetricBuilder.HistogramType$.MODULE$); - MetricBuilder cmb = sr.metricBuilder(MetricBuilder.CounterType$.MODULE$); + MetricBuilder gmb = sr.metricBuilder(MetricTypes.GAUGE_TYPE); + MetricBuilder smb = sr.metricBuilder(MetricTypes.HISTOGRAM_TYPE); + MetricBuilder cmb = sr.metricBuilder(MetricTypes.COUNTER_TYPE); Gauge g = gmb.gauge(() -> 3.0f, "my", "cool", "gauge"); Stat s = smb.histogram("my", "cool", "histogram"); diff --git a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java b/util-stats/src/test/java/com/twitter/compiletest/StatsReceiverCompilationTest.java similarity index 86% rename from util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java rename to util-stats/src/test/java/com/twitter/compiletest/StatsReceiverCompilationTest.java index b8ef5b3c3d..3e58e24632 100644 --- a/util-stats/src/test/java/com/twitter/finagle/stats/StatsReceiverCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/compiletest/StatsReceiverCompilationTest.java @@ -1,5 +1,14 @@ -package com.twitter.finagle.stats; - +package com.twitter.compiletest; + +import com.twitter.finagle.stats.AbstractStatsReceiver; +import com.twitter.finagle.stats.Counter; +import com.twitter.finagle.stats.Gauge; +import com.twitter.finagle.stats.InMemoryStatsReceiver; +import com.twitter.finagle.stats.JStats; +import com.twitter.finagle.stats.NullStatsReceiver; +import com.twitter.finagle.stats.Stat; +import com.twitter.finagle.stats.StatsReceiver; +import com.twitter.finagle.stats.Verbosity; import com.twitter.util.Future; import org.junit.Test; @@ -9,7 +18,6 @@ import java.util.concurrent.TimeUnit; import scala.Function0; -import scala.collection.immutable.Seq; /** * Java compatibility test for {@link com.twitter.finagle.stats.StatsReceiver}. From 968c30331a8a8b9e9ef50c821e6dc1622be34e45 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 7 Mar 2022 20:11:04 +0000 Subject: [PATCH 638/796] finagle-core: stop sending requests if they are already expired Problem If a request is configured with a negative timeout, or the remaining deadline is negative, we still send it to its receivers. Which causes duplicated timeout exceptions to receivers in the callgraph. Solution Stop sending requests if they are expired. Differential Revision: https://phabricator.twitter.biz/D845862 --- util-core/src/main/scala/com/twitter/util/Duration.scala | 2 ++ .../src/test/scala/com/twitter/util/DurationTest.scala | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index c52403e9ba..bcfcf4285b 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -318,6 +318,8 @@ sealed class Duration private[util] (protected val nanos: Long) def inNanoseconds: Long = nanos + def isNegative: Boolean = this.inNanoseconds < 0 + /** * Returns the length of the duration in the given TimeUnit. * diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 964703ebf7..55d7ee8302 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -24,6 +24,15 @@ trait DurationOps { val ops: Duration.type = Duration } class DurationTest extends AnyWordSpec with DurationOps with TimeLikeSpec[Duration] { "Duration" should { + "isNegative" in { + assert((-1.second).isNegative) + assert(!1.second.isNegative) + assert((-100.milliseconds).isNegative) + assert(!100.milliseconds.isNegative) + assert((-100.nanosecond).isNegative) + assert(!100.nanosecond.isNegative) + } + "*" in { assert(1.second * 2 == 2.seconds) assert(1.second * Long.MaxValue == Duration.Top) From a28aeb3ffe61a8b14a733d965e6e4be78240ccc4 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 8 Mar 2022 19:38:14 +0000 Subject: [PATCH 639/796] util-stats: Add builder pattern for labels Problem We have labels on the MetricBuilder but we can't add them outside the class itself outside of the mega apply method. Solution Make a builder method to swap out the labels. We also make the labels private[finagle] while the API is in flux so as to not close doors on ourselves. Differential Revision: https://phabricator.twitter.biz/D847815 --- .../scala/com/twitter/finagle/stats/MetricBuilder.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index a8e6988fda..2f364a0999 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -175,7 +175,7 @@ class MetricBuilder private ( val verbosity: Verbosity, val sourceClass: Option[String], val name: Seq[String], - val labels: Map[String, String], + private[finagle] val labels: Map[String, String], val relativeName: Seq[String], val processPath: Option[String], // Only persisted and relevant when building histograms. @@ -198,6 +198,7 @@ class MetricBuilder private ( verbosity: Verbosity = this.verbosity, sourceClass: Option[String] = this.sourceClass, name: Seq[String] = this.name, + labels: Map[String, String] = this.labels, relativeName: Seq[String] = this.relativeName, processPath: Option[String] = this.processPath, isStandard: Boolean = this.isStandard, @@ -251,6 +252,10 @@ class MetricBuilder private ( @varargs def withName(name: String*): MetricBuilder = this.copy(name = name) + // Temporarily private while API's are under development + private[finagle] def withLabels(labels: Map[String, String]): MetricBuilder = + this.copy(labels = labels) + @varargs def withRelativeName(relativeName: String*): MetricBuilder = if (this.relativeName == Seq.empty) this.copy(relativeName = relativeName) else this From b55defe74d195fb0781047efb431aa15c0cc4c1e Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Wed, 9 Mar 2022 06:18:59 +0000 Subject: [PATCH 640/796] Revert "finagle-core: stop sending requests if they are already expired" This reverts commit 505f0109eba8854d1b623d65e0d5fc1e18560d3b. Differential Revision: https://phabricator.twitter.biz/D848866 --- util-core/src/main/scala/com/twitter/util/Duration.scala | 2 -- .../src/test/scala/com/twitter/util/DurationTest.scala | 9 --------- 2 files changed, 11 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Duration.scala b/util-core/src/main/scala/com/twitter/util/Duration.scala index bcfcf4285b..c52403e9ba 100644 --- a/util-core/src/main/scala/com/twitter/util/Duration.scala +++ b/util-core/src/main/scala/com/twitter/util/Duration.scala @@ -318,8 +318,6 @@ sealed class Duration private[util] (protected val nanos: Long) def inNanoseconds: Long = nanos - def isNegative: Boolean = this.inNanoseconds < 0 - /** * Returns the length of the duration in the given TimeUnit. * diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 55d7ee8302..964703ebf7 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -24,15 +24,6 @@ trait DurationOps { val ops: Duration.type = Duration } class DurationTest extends AnyWordSpec with DurationOps with TimeLikeSpec[Duration] { "Duration" should { - "isNegative" in { - assert((-1.second).isNegative) - assert(!1.second.isNegative) - assert((-100.milliseconds).isNegative) - assert(!100.milliseconds.isNegative) - assert((-100.nanosecond).isNegative) - assert(!100.nanosecond.isNegative) - } - "*" in { assert(1.second * 2 == 2.seconds) assert(1.second * Long.MaxValue == Duration.Top) From 0ff46aea8d584d5851267932d8272e5494e8f22b Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 10 Mar 2022 01:26:00 +0000 Subject: [PATCH 641/796] util-stats: Lower the abstraction level of stats name translation Problem We want to be able to scope stats collectively and we normally do that with NameTranslatingStatsReceiver, but that abstraction only works on the hierarchical name and doesn't allow us to auto-magically scope things based on labels. Solution Introduce an abstraction just a bit lower, TranslatingStatsReceiver, which operates on the whole MetricBuilder. This abstraction makes the sky the limit for modifying metric definitions. Differential Revision: https://phabricator.twitter.biz/D848642 --- .../stats/NameTranslatingStatsReceiver.scala | 19 ++++---------- .../stats/TranslatingStatsReceiver.scala | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index 99a57d5cbe..6767076a25 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -2,32 +2,23 @@ package com.twitter.finagle.stats /** * A StatsReceiver receiver proxy that translates all counter, stat, and gauge - * names according to a `translate` function. + * names according to a `translate(name: Seq[String])` function. * * @param self The underlying StatsReceiver to which translated names are passed * * @param namespacePrefix the namespace used for translations */ abstract class NameTranslatingStatsReceiver( - protected val self: StatsReceiver, + self: StatsReceiver, namespacePrefix: String) - extends StatsReceiverProxy { + extends TranslatingStatsReceiver(self) { def this(self: StatsReceiver) = this(self, "") protected def translate(name: Seq[String]): Seq[String] - override def counter(metricBuilder: MetricBuilder): Counter = { - self.counter(metricBuilder.withName(translate(metricBuilder.name): _*)) - } - - override def stat(metricBuilder: MetricBuilder): Stat = { - self.stat(metricBuilder.withName(translate(metricBuilder.name): _*)) - } - - override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { - self.addGauge(metricBuilder.withName(translate(metricBuilder.name): _*))(f) - } + final protected def translate(builder: MetricBuilder): MetricBuilder = + builder.withName(translate(builder.name): _*) override def toString: String = s"$self/$namespacePrefix" } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala new file mode 100644 index 0000000000..d1957718f8 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala @@ -0,0 +1,26 @@ +package com.twitter.finagle.stats + +/** + * A StatsReceiver receiver proxy that translates all counter, stat, and gauge + * names according to a `translate` function. + * + * @param self The underlying StatsReceiver to which translated `MetricBuilder`s are passed + */ +abstract class TranslatingStatsReceiver( + protected val self: StatsReceiver) + extends StatsReceiverProxy { + + protected def translate(builder: MetricBuilder): MetricBuilder + + override def counter(metricBuilder: MetricBuilder): Counter = + self.counter(translate(metricBuilder)) + + override def stat(metricBuilder: MetricBuilder): Stat = + self.stat(translate(metricBuilder)) + + override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { + self.addGauge(translate(metricBuilder))(f) + } + + override def toString: String = s"Translating($self)" +} From 6e1f1b18e58612a690d443cb3701b055a8abcd53 Mon Sep 17 00:00:00 2001 From: Lily He Date: Thu, 10 Mar 2022 20:32:17 +0000 Subject: [PATCH 642/796] finagle-stats-core: Encode Prometheus summary Problem: We want to export Finagle histograms in Prometheus summary format. This format allows Finagle to have rich (dimensional) metrics where metrics have labels attached to them. Solution: Encode histogram name, value, quantiles, and their labels in the Prometheus summary format. Prometheus summaries show quantiles and the bounds for the quantiles. JIRA Issues: CSL-11564 Differential Revision: https://phabricator.twitter.biz/D848369 --- .../main/scala/com/twitter/finagle/stats/MetricBuilder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 2f364a0999..d8661a114e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -133,7 +133,7 @@ object MetricBuilder { } case object HistogramType extends MetricType { def toJsonString: String = "histogram" - def toPrometheusString: String = toJsonString + def toPrometheusString: String = "summary" } /** Counter type that will be always unlatched */ From 20a66daa41117e473b9d1cba2b2a459efca000af Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Fri, 11 Mar 2022 19:54:35 +0000 Subject: [PATCH 643/796] 3rdparty: jackson upgrade to 2.13.2 Problem We are on jackson 2.13.1 but we want to upgrade to a new version. Solution Upgrade to 2.13.2 JIRA Issues: CSL-11797 Differential Revision: https://phabricator.twitter.biz/D848592 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2511e0c7bd..04b1d31448 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,6 +26,8 @@ New Features Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ +* util: Bump version of Jackson to 2.13.2. ``PHAB_ID=D848592`` + * util-slf4j-api: Update the `Logger` API to include "call-by-name" method variations akin to the `Logging` trait. When creating a `Logger` from a Scala singleton object class, the resultant logger name will no longer include diff --git a/build.sbt b/build.sbt index 331bbbeaab..87b375a1d4 100644 --- a/build.sbt +++ b/build.sbt @@ -7,7 +7,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "22.3.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.13.1" +val jacksonVersion = "2.13.2" val json4sVersion = "4.0.3" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" From ff01d238d715d7a9f70919be9c885629dd8e9a67 Mon Sep 17 00:00:00 2001 From: Hamdi Allam Date: Wed, 23 Mar 2022 23:14:28 +0000 Subject: [PATCH 644/796] util/util-core: promise cpu usage resource tracing Problem: We want low-level resource information per request. Solution: In our Promise implementation, all continuations stem from the internal `Transformer` and `Monitored` api which are both continuations represented by `K`. We have a mutable Local that can keep track of resource usage as each continuation is run. The local is set in Finagle if the request is actively tracing. When the continuation is created, we check this local to determine if the continuation should be tracked Result: Read resource data from the local context when tracing a request in finagle. This includes: - Accumulated Cpu Time - # of executed continuations JIRA Issues: CSL-11501, CSL-11627 Differential Revision: https://phabricator.twitter.biz/D791323 --- .../com/twitter/util/PromiseBenchmark.scala | 67 +- .../scala/com/twitter/util/ConstFuture.scala | 30 +- .../scala/com/twitter/util/FuturePool.scala | 17 +- .../main/scala/com/twitter/util/Local.scala | 2160 +++++++++++++---- .../main/scala/com/twitter/util/Promise.scala | 21 +- .../com/twitter/util/ResourceTracker.scala | 114 + .../com/twitter/util/FuturePoolTest.scala | 22 + .../scala/com/twitter/util/PromiseTest.scala | 56 + 8 files changed, 2032 insertions(+), 455 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/util/ResourceTracker.scala diff --git a/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala index 4103cabfdb..4291556d7e 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/PromiseBenchmark.scala @@ -7,10 +7,21 @@ import org.openjdk.jmh.annotations._ @Warmup(iterations = 5) @Measurement(iterations = 5) class PromiseBenchmark extends StdBenchAnnotations { + import PromiseBenchmark._ - private[this] val StringFuture = Future.value("hi") + @Benchmark + def responder(state: ResponderState): Int = { + import state._ + initPromise.updateIfEmpty(UpdateValue) + Await.result(promise) + } - private[this] val Value = Return("okok") + @Benchmark + def responderWithResourceUsage(state: ResourceTrackedResponderState): Int = { + import state._ + initPromise.updateIfEmpty(UpdateValue) + Await.result(promise) + } private[this] val NoopCallback: Try[String] => Unit = _ => () @@ -20,13 +31,13 @@ class PromiseBenchmark extends StdBenchAnnotations { } @Benchmark - def detach(state: PromiseBenchmark.PromiseDetachState): Unit = { + def detach(state: PromiseDetachState): Unit = { import state._ promise.detach() } @Benchmark - def detachDetachableFuture(state: PromiseBenchmark.PromiseDetachState): Unit = { + def detachDetachableFuture(state: PromiseDetachState): Unit = { import state._ detachableFuture.detach() } @@ -44,22 +55,22 @@ class PromiseBenchmark extends StdBenchAnnotations { } @Benchmark - def interrupts1(state: PromiseBenchmark.InterruptsState): Promise[String] = { + def interrupts1(state: InterruptsState): Promise[String] = { Promise.interrupts(state.a) } @Benchmark - def interrupts2(state: PromiseBenchmark.InterruptsState): Promise[String] = { + def interrupts2(state: InterruptsState): Promise[String] = { Promise.interrupts(state.a, state.b) } @Benchmark - def interruptsN(state: PromiseBenchmark.InterruptsState): Promise[String] = { + def interruptsN(state: InterruptsState): Promise[String] = { Promise.interrupts(state.futures: _*) } @Benchmark - def promiseIsDefined(state: PromiseBenchmark.PromiseState): Boolean = { + def promiseIsDefined(state: PromiseState): Boolean = { if (state.i == state.len) state.i = 0 val ret = state.promises(state.i).isDefined state.i += 1 @@ -67,7 +78,7 @@ class PromiseBenchmark extends StdBenchAnnotations { } @Benchmark - def promisePoll(state: PromiseBenchmark.PromiseState): Option[Try[Unit]] = { + def promisePoll(state: PromiseState): Option[Try[Unit]] = { if (state.i == state.len) state.i = 0 val ret = state.promises(state.i).poll state.i += 1 @@ -87,6 +98,44 @@ class PromiseBenchmark extends StdBenchAnnotations { } object PromiseBenchmark { + final val StringFuture = Future.value("hi") + final val Value = Return("okok") + + @State(Scope.Thread) + class ResponderState { + val UpdateValue = Return(1) + + @Param(Array("1", "10", "100", "1000")) + var numK: Int = _ + + var initPromise: Promise[Int] = _ + var promise: Future[Int] = _ + + @Setup(Level.Invocation) + def prepare(): Unit = { + initPromise = new Promise[Int]() + promise = (0 to numK).foldLeft[Future[Int]](initPromise) { (p, i) => + p.map(num => num + i) + } + } + } + + @State(Scope.Thread) + class ResourceTrackedResponderState extends ResponderState { + val resourceTracker = new ResourceTracker() + + @Setup(Level.Invocation) + override def prepare(): Unit = { + ResourceTracker.set(resourceTracker) + super.prepare() + } + + @TearDown(Level.Invocation) + def tearDown(): Unit = { + ResourceTracker.clear() + } + } + @State(Scope.Thread) class PromiseDetachState { diff --git a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala index 8817d2a015..8eeebba0ce 100644 --- a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala @@ -17,14 +17,19 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { // https://twitter.github.io/util/guide/util-cookbook/futures.html#future-recursion // for details. The second is that this keeps the execution order consistent // with `Promise`. - def respond(k: Try[A] => Unit): Future[A] = { + def respond(f: Try[A] => Unit): Future[A] = { val saved = Local.save() Scheduler.submit(new Runnable { def run(): Unit = { val current = Local.save() - if (current ne saved) - Local.restore(saved) - try k(result) + if (current ne saved) Local.restore(saved) + + val tracker = saved.resourceTracker + val run = + if (tracker eq None) f + else ResourceTracker.wrapAndMeasureUsage(f, tracker.get) + + try run(result) catch Monitor.catcher finally Local.restore(current) } @@ -39,11 +44,14 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { def raise(interrupt: Throwable): Unit = () - protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = + // NOTE: since `f` is not run by the scheduler, the used cpu time will be tracked by + // the continuation that executes this transformation + protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { try Future.const(f(result)) catch { case scala.util.control.NonFatal(e) => Future.const(Throw(e)) } + } def transform[B](f: Try[A] => Future[B]): Future[B] = { val p = new Promise[B] @@ -52,10 +60,15 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { Scheduler.submit(new Runnable { def run(): Unit = { val current = Local.save() - if (current ne saved) - Local.restore(saved) + if (current ne saved) Local.restore(saved) + + val tracker = saved.resourceTracker + val run = + if (tracker eq None) f + else ResourceTracker.wrapAndMeasureUsage(f, tracker.get) + val computed = - try f(result) + try run(result) catch { case e: NonLocalReturnControl[_] => Future.exception(new FutureNonLocalReturnControl(e)) case scala.util.control.NonFatal(e) => Future.exception(e) @@ -63,6 +76,7 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { Monitor.handle(t) throw t } finally Local.restore(current) + p.become(computed) } }) diff --git a/util-core/src/main/scala/com/twitter/util/FuturePool.scala b/util-core/src/main/scala/com/twitter/util/FuturePool.scala index f3f9d20715..ad66a238e2 100644 --- a/util-core/src/main/scala/com/twitter/util/FuturePool.scala +++ b/util-core/src/main/scala/com/twitter/util/FuturePool.scala @@ -150,14 +150,17 @@ class ExecutorServiceFuturePool protected[this] ( } def apply[T](f: => T): Future[T] = { - if (forkingScheduler != null) { - forkingScheduler.fork(Future.value(f)) - } else { + val saved = Local.save() + val tracker = saved.resourceTracker + val applyF = + if (tracker eq None) () => f + else ResourceTracker.wrapAndMeasureUsage[T](f, tracker.get) + + if (forkingScheduler != null) forkingScheduler.fork(Future.value(applyF())) + else { val runOk = new AtomicBoolean(true) val p = new Promise[T] val task = new Runnable { - private[this] val saved = Local.save() - def run(): Unit = { // Make an effort to skip work in the case the promise // has been cancelled or already defined. @@ -165,9 +168,9 @@ class ExecutorServiceFuturePool protected[this] ( return val current = Local.save() - Local.restore(saved) + if (current ne saved) Local.restore(saved) - try p.updateIfEmpty(Try(f)) + try p.updateIfEmpty(Try(applyF())) catch { case nlrc: NonLocalReturnControl[_] => val fnlrc = new FutureNonLocalReturnControl(nlrc) diff --git a/util-core/src/main/scala/com/twitter/util/Local.scala b/util-core/src/main/scala/com/twitter/util/Local.scala index 8c144338c3..85e0ebcdc1 100644 --- a/util-core/src/main/scala/com/twitter/util/Local.scala +++ b/util-core/src/main/scala/com/twitter/util/Local.scala @@ -7,7 +7,11 @@ object Local { * when the number of elements is less than 16. * key - Local.Key, value - Option[_] */ - sealed abstract class Context private () { + sealed abstract class Context private ( + final private[util] val resourceTracker: Option[ResourceTracker]) { + private[util] def setResourceTracker(tracker: ResourceTracker): Context + private[util] def removeResourceTracker(): Context + private[util] def get(k: Key): Option[_] private[util] def remove(k: Key): Context private[util] def set(k: Key, v: Some[_]): Context @@ -18,15 +22,35 @@ object Local { /** * The empty Context */ - def empty: Context = EmptyContext + val empty: Context = EmptyContext + + private object EmptyContext extends Context(None) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context0(Some(tracker)) + def removeResourceTracker(): Context = this + + def get(k: Key): Option[_] = None + def remove(k: Key): Context = this + def set(k: Key, v: Some[_]): Context = new Context1(resourceTracker, k, v) + } + + private final class Context0(resourceTracker: Option[ResourceTracker]) + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context0(Some(tracker)) + def removeResourceTracker(): Context = EmptyContext - private object EmptyContext extends Context { def get(k: Key): Option[_] = None def remove(k: Key): Context = this - def set(k: Key, v: Some[_]): Context = new Context1(k, v) + def set(k: Key, v: Some[_]): Context = new Context1(resourceTracker, k, v) } - private final class Context1(k1: Key, v1: Some[_]) extends Context { + private final class Context1( + resourceTracker: Option[ResourceTracker], + k1: Key, + v1: Some[_]) + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = + new Context1(Some(tracker), k1, v1) + def removeResourceTracker(): Context = new Context1(None, k1, v1) def get(k: Key): Option[_] = if (k eq k1) v1 else None @@ -35,11 +59,20 @@ object Local { if (k eq k1) EmptyContext else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context1(k1, v) - else new Context2(k1, v1, k, v) + if (k eq k1) new Context1(resourceTracker, k1, v) + else new Context2(resourceTracker, k1, v1, k, v) } - private final class Context2(k1: Key, v1: Some[_], k2: Key, v2: Some[_]) extends Context { + private final class Context2( + resourceTracker: Option[ResourceTracker], + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_]) + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = + new Context2(Some(tracker), k1, v1, k2, v2) + def removeResourceTracker(): Context = new Context2(None, k1, v1, k2, v2) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -47,14 +80,14 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context1(k2, v2) - else if (k eq k2) new Context1(k1, v1) + if (k eq k1) new Context1(resourceTracker, k2, v2) + else if (k eq k2) new Context1(resourceTracker, k1, v1) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context2(k1, v, k2, v2) - else if (k eq k2) new Context2(k1, v1, k2, v) - else new Context3(k1, v1, k2, v2, k, v) + if (k eq k1) new Context2(resourceTracker, k1, v, k2, v2) + else if (k eq k2) new Context2(resourceTracker, k1, v1, k2, v) + else new Context3(resourceTracker, k1, v1, k2, v2, k, v) } // Script for generating the ContextN @@ -63,8 +96,12 @@ object Local { def mkContext(num: Int): String = { s"""private final class Context$num( + resourceTracker: Option[ResourceTracker], ${fieldList(num)} - ) extends Context { + ) extends Context(resourceTracker) { + ${mkSetResourceTracker(num)} + + ${mkRemoveResourceTracker(num)} ${mkGet(num)} @@ -82,6 +119,14 @@ object Local { }.mkString(", \n") } + def mkSetResourceTracker(num: Int): String = { + s"def setResourceTracker(tracker: ResourceTracker): Context = new Context$num(Some(tracker), ${fieldList(num)})" + } + + def mkRemoveResourceTracker(num: Int): String = { + s"def removeResourceTracker(): Context = new Context$num(None, ${fieldList(num)})" + } + def mkGet(num: Int): String = { val name = s"def get(k: Key): Option[_] = " val content = (1.to(num)).map { i => @@ -95,7 +140,7 @@ object Local { def newCtx(sum: Int, current: Int): String = { (1.to(current - 1) ++: (current + 1).to(sum)).map { i => s"k$i, v$i" - }.mkString("(", ", ", ")") + }.mkString("(resourceTracker, ", ", ", ")") } val content = (1.to(num)).map { i => s"k$i) new Context${num - 1}${newCtx(num, i)}" @@ -110,7 +155,7 @@ object Local { if (i == current && last) s"k, v" else if (i == current) s"k$i, v" else s"k$i, v$i" - }.mkString("(", ", ", ")") + }.mkString("(resourceTracker, ", ", ", ")") } val content = (1.to(num)).map { i => s"k$i) new Context$num${newCtx(num, i, false)}" @@ -119,12 +164,30 @@ object Local { } def main(args: Array[String]) { - print(2.to(15).map{ i => mkContext(i)}.mkString) + print(3.to(15).map{ i => mkContext(i)}.mkString) } } */ - private final class Context3(k1: Key, v1: Some[_], k2: Key, v2: Some[_], k3: Key, v3: Some[_]) - extends Context { + private final class Context3( + resourceTracker: Option[ResourceTracker], + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_]) + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context3( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_]) + + def removeResourceTracker(): Context = + new Context3(None, k1: Key, v1: Some[_], k2: Key, v2: Some[_], k3: Key, v3: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -133,19 +196,20 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context2(k2, v2, k3, v3) - else if (k eq k2) new Context2(k1, v1, k3, v3) - else if (k eq k3) new Context2(k1, v1, k2, v2) + if (k eq k1) new Context2(resourceTracker, k2, v2, k3, v3) + else if (k eq k2) new Context2(resourceTracker, k1, v1, k3, v3) + else if (k eq k3) new Context2(resourceTracker, k1, v1, k2, v2) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context3(k1, v, k2, v2, k3, v3) - else if (k eq k2) new Context3(k1, v1, k2, v, k3, v3) - else if (k eq k3) new Context3(k1, v1, k2, v2, k3, v) - else new Context4(k1, v1, k2, v2, k3, v3, k, v) + if (k eq k1) new Context3(resourceTracker, k1, v, k2, v2, k3, v3) + else if (k eq k2) new Context3(resourceTracker, k1, v1, k2, v, k3, v3) + else if (k eq k3) new Context3(resourceTracker, k1, v1, k2, v2, k3, v) + else new Context4(resourceTracker, k1, v1, k2, v2, k3, v3, k, v) } private final class Context4( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -154,7 +218,28 @@ object Local { v3: Some[_], k4: Key, v4: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context4( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_]) + + def removeResourceTracker(): Context = new Context4( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -164,21 +249,22 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context3(k2, v2, k3, v3, k4, v4) - else if (k eq k2) new Context3(k1, v1, k3, v3, k4, v4) - else if (k eq k3) new Context3(k1, v1, k2, v2, k4, v4) - else if (k eq k4) new Context3(k1, v1, k2, v2, k3, v3) + if (k eq k1) new Context3(resourceTracker, k2, v2, k3, v3, k4, v4) + else if (k eq k2) new Context3(resourceTracker, k1, v1, k3, v3, k4, v4) + else if (k eq k3) new Context3(resourceTracker, k1, v1, k2, v2, k4, v4) + else if (k eq k4) new Context3(resourceTracker, k1, v1, k2, v2, k3, v3) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context4(k1, v, k2, v2, k3, v3, k4, v4) - else if (k eq k2) new Context4(k1, v1, k2, v, k3, v3, k4, v4) - else if (k eq k3) new Context4(k1, v1, k2, v2, k3, v, k4, v4) - else if (k eq k4) new Context4(k1, v1, k2, v2, k3, v3, k4, v) - else new Context5(k1, v1, k2, v2, k3, v3, k4, v4, k, v) + if (k eq k1) new Context4(resourceTracker, k1, v, k2, v2, k3, v3, k4, v4) + else if (k eq k2) new Context4(resourceTracker, k1, v1, k2, v, k3, v3, k4, v4) + else if (k eq k3) new Context4(resourceTracker, k1, v1, k2, v2, k3, v, k4, v4) + else if (k eq k4) new Context4(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v) + else new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k, v) } private final class Context5( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -189,7 +275,32 @@ object Local { v4: Some[_], k5: Key, v5: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context5( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_]) + + def removeResourceTracker(): Context = new Context5( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -200,23 +311,24 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context4(k2, v2, k3, v3, k4, v4, k5, v5) - else if (k eq k2) new Context4(k1, v1, k3, v3, k4, v4, k5, v5) - else if (k eq k3) new Context4(k1, v1, k2, v2, k4, v4, k5, v5) - else if (k eq k4) new Context4(k1, v1, k2, v2, k3, v3, k5, v5) - else if (k eq k5) new Context4(k1, v1, k2, v2, k3, v3, k4, v4) + if (k eq k1) new Context4(resourceTracker, k2, v2, k3, v3, k4, v4, k5, v5) + else if (k eq k2) new Context4(resourceTracker, k1, v1, k3, v3, k4, v4, k5, v5) + else if (k eq k3) new Context4(resourceTracker, k1, v1, k2, v2, k4, v4, k5, v5) + else if (k eq k4) new Context4(resourceTracker, k1, v1, k2, v2, k3, v3, k5, v5) + else if (k eq k5) new Context4(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context5(k1, v, k2, v2, k3, v3, k4, v4, k5, v5) - else if (k eq k2) new Context5(k1, v1, k2, v, k3, v3, k4, v4, k5, v5) - else if (k eq k3) new Context5(k1, v1, k2, v2, k3, v, k4, v4, k5, v5) - else if (k eq k4) new Context5(k1, v1, k2, v2, k3, v3, k4, v, k5, v5) - else if (k eq k5) new Context5(k1, v1, k2, v2, k3, v3, k4, v4, k5, v) - else new Context6(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k, v) + if (k eq k1) new Context5(resourceTracker, k1, v, k2, v2, k3, v3, k4, v4, k5, v5) + else if (k eq k2) new Context5(resourceTracker, k1, v1, k2, v, k3, v3, k4, v4, k5, v5) + else if (k eq k3) new Context5(resourceTracker, k1, v1, k2, v2, k3, v, k4, v4, k5, v5) + else if (k eq k4) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v, k5, v5) + else if (k eq k5) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v) + else new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k, v) } private final class Context6( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -229,7 +341,36 @@ object Local { v5: Some[_], k6: Key, v6: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context6( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_]) + + def removeResourceTracker(): Context = new Context6( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -241,25 +382,31 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context5(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) - else if (k eq k2) new Context5(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6) - else if (k eq k3) new Context5(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6) - else if (k eq k4) new Context5(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6) - else if (k eq k5) new Context5(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6) - else if (k eq k6) new Context5(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5) + if (k eq k1) new Context5(resourceTracker, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) + else if (k eq k2) new Context5(resourceTracker, k1, v1, k3, v3, k4, v4, k5, v5, k6, v6) + else if (k eq k3) new Context5(resourceTracker, k1, v1, k2, v2, k4, v4, k5, v5, k6, v6) + else if (k eq k4) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k5, v5, k6, v6) + else if (k eq k5) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k6, v6) + else if (k eq k6) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context6(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) - else if (k eq k2) new Context6(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6) - else if (k eq k3) new Context6(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6) - else if (k eq k4) new Context6(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6) - else if (k eq k5) new Context6(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6) - else if (k eq k6) new Context6(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v) - else new Context7(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k, v) + if (k eq k1) new Context6(resourceTracker, k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) + else if (k eq k2) + new Context6(resourceTracker, k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6) + else if (k eq k3) + new Context6(resourceTracker, k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6) + else if (k eq k4) + new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6) + else if (k eq k5) + new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6) + else if (k eq k6) + new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v) + else new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k, v) } private final class Context7( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -274,7 +421,40 @@ object Local { v6: Some[_], k7: Key, v7: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context7( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_]) + + def removeResourceTracker(): Context = new Context7( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -287,27 +467,59 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context6(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) - else if (k eq k2) new Context6(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) - else if (k eq k3) new Context6(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7) - else if (k eq k4) new Context6(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7) - else if (k eq k5) new Context6(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7) - else if (k eq k6) new Context6(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7) - else if (k eq k7) new Context6(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) + if (k eq k1) new Context6(resourceTracker, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + else if (k eq k2) + new Context6(resourceTracker, k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + else if (k eq k3) + new Context6(resourceTracker, k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7) + else if (k eq k4) + new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7) + else if (k eq k5) + new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7) + else if (k eq k6) + new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7) + else if (k eq k7) + new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context7(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) - else if (k eq k2) new Context7(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) - else if (k eq k3) new Context7(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7) - else if (k eq k4) new Context7(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7) - else if (k eq k5) new Context7(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7) - else if (k eq k6) new Context7(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7) - else if (k eq k7) new Context7(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v) - else new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k, v) + if (k eq k1) + new Context7(resourceTracker, k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + else if (k eq k2) + new Context7(resourceTracker, k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + else if (k eq k3) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7) + else if (k eq k4) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7) + else if (k eq k5) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7) + else if (k eq k6) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7) + else if (k eq k7) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v) + else + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k, + v) } private final class Context8( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -324,7 +536,44 @@ object Local { v7: Some[_], k8: Key, v8: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context8( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_]) + + def removeResourceTracker(): Context = new Context8( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -338,36 +587,202 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context7(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) - else if (k eq k2) new Context7(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) - else if (k eq k3) new Context7(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) - else if (k eq k4) new Context7(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8) - else if (k eq k5) new Context7(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8) - else if (k eq k6) new Context7(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8) - else if (k eq k7) new Context7(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8) - else if (k eq k8) new Context7(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + if (k eq k1) + new Context7(resourceTracker, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + else if (k eq k2) + new Context7(resourceTracker, k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + else if (k eq k3) + new Context7(resourceTracker, k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + else if (k eq k4) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8) + else if (k eq k5) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8) + else if (k eq k6) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8) + else if (k eq k7) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8) + else if (k eq k8) + new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context8(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + if (k eq k1) + new Context8( + resourceTracker, + k1, + v, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k2) - new Context8(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + new Context8( + resourceTracker, + k1, + v1, + k2, + v, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k3) - new Context8(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k4) - new Context8(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k5) - new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k6) - new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v, + k7, + v7, + k8, + v8) else if (k eq k7) - new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v, + k8, + v8) else if (k eq k8) - new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v) - else new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k, v) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v) + else + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k, + v) } private final class Context9( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -386,7 +801,48 @@ object Local { v8: Some[_], k9: Key, v9: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context9( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_]) + + def removeResourceTracker(): Context = new Context9( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -401,46 +857,372 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context8(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + if (k eq k1) + new Context8( + resourceTracker, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k2) - new Context8(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + new Context8( + resourceTracker, + k1, + v1, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k3) - new Context8(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k4) - new Context8(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k5) - new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k6) - new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k7) - new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k8, + v8, + k9, + v9) else if (k eq k8) - new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k9, + v9) else if (k eq k9) - new Context8(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + new Context8( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8) else this def set(k: Key, v: Some[_]): Context = if (k eq k1) - new Context9(k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + new Context9( + resourceTracker, + k1, + v, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k2) - new Context9(k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + new Context9( + resourceTracker, + k1, + v1, + k2, + v, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k3) - new Context9(k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k4) - new Context9(k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k5) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7, k8, v8, k9, v9) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k6) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7, k8, v8, k9, v9) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v, + k7, + v7, + k8, + v8, + k9, + v9) else if (k eq k7) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v, k8, v8, k9, v9) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v, + k8, + v8, + k9, + v9) else if (k eq k8) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v, k9, v9) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v, + k9, + v9) else if (k eq k9) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v) else new Context10( + resourceTracker, k1, v1, k2, @@ -460,11 +1242,11 @@ object Local { k9, v9, k, - v - ) + v) } private final class Context10( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -485,7 +1267,52 @@ object Local { v9: Some[_], k10: Key, v10: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context10( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_]) + + def removeResourceTracker(): Context = new Context10( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -502,30 +1329,221 @@ object Local { def remove(k: Key): Context = if (k eq k1) - new Context9(k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + new Context9( + resourceTracker, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10) else if (k eq k2) - new Context9(k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + new Context9( + resourceTracker, + k1, + v1, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10) else if (k eq k3) - new Context9(k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10) else if (k eq k4) - new Context9(k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10) else if (k eq k5) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8, k9, v9, k10, v10) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10) else if (k eq k6) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8, k9, v9, k10, v10) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k7, + v7, + k8, + v8, + k9, + v9, + k10, + v10) else if (k eq k7) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8, k9, v9, k10, v10) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k8, + v8, + k9, + v9, + k10, + v10) else if (k eq k8) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k9, v9, k10, v10) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k9, + v9, + k10, + v10) else if (k eq k9) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k10, v10) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k10, + v10) else if (k eq k10) - new Context9(k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8, k9, v9) + new Context9( + resourceTracker, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8, + k9, + v9) else this def set(k: Key, v: Some[_]): Context = if (k eq k1) new Context10( + resourceTracker, k1, v, k2, @@ -545,10 +1563,10 @@ object Local { k9, v9, k10, - v10 - ) + v10) else if (k eq k2) new Context10( + resourceTracker, k1, v1, k2, @@ -568,10 +1586,10 @@ object Local { k9, v9, k10, - v10 - ) + v10) else if (k eq k3) new Context10( + resourceTracker, k1, v1, k2, @@ -591,10 +1609,10 @@ object Local { k9, v9, k10, - v10 - ) + v10) else if (k eq k4) new Context10( + resourceTracker, k1, v1, k2, @@ -614,10 +1632,10 @@ object Local { k9, v9, k10, - v10 - ) + v10) else if (k eq k5) new Context10( + resourceTracker, k1, v1, k2, @@ -637,10 +1655,10 @@ object Local { k9, v9, k10, - v10 - ) + v10) else if (k eq k6) new Context10( + resourceTracker, k1, v1, k2, @@ -660,10 +1678,10 @@ object Local { k9, v9, k10, - v10 - ) + v10) else if (k eq k7) new Context10( + resourceTracker, k1, v1, k2, @@ -683,10 +1701,10 @@ object Local { k9, v9, k10, - v10 - ) + v10) else if (k eq k8) new Context10( + resourceTracker, k1, v1, k2, @@ -706,10 +1724,10 @@ object Local { k9, v9, k10, - v10 - ) + v10) else if (k eq k9) new Context10( + resourceTracker, k1, v1, k2, @@ -729,10 +1747,10 @@ object Local { k9, v, k10, - v10 - ) + v10) else if (k eq k10) new Context10( + resourceTracker, k1, v1, k2, @@ -752,10 +1770,10 @@ object Local { k9, v9, k10, - v - ) + v) else new Context11( + resourceTracker, k1, v1, k2, @@ -777,11 +1795,11 @@ object Local { k10, v10, k, - v - ) + v) } private final class Context11( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -804,7 +1822,56 @@ object Local { v10: Some[_], k11: Key, v11: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context11( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_]) + + def removeResourceTracker(): Context = new Context11( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -823,6 +1890,7 @@ object Local { def remove(k: Key): Context = if (k eq k1) new Context10( + resourceTracker, k2, v2, k3, @@ -842,10 +1910,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k2) new Context10( + resourceTracker, k1, v1, k3, @@ -865,10 +1933,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k3) new Context10( + resourceTracker, k1, v1, k2, @@ -888,10 +1956,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k4) new Context10( + resourceTracker, k1, v1, k2, @@ -911,10 +1979,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k5) new Context10( + resourceTracker, k1, v1, k2, @@ -934,10 +2002,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k6) new Context10( + resourceTracker, k1, v1, k2, @@ -957,10 +2025,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k7) new Context10( + resourceTracker, k1, v1, k2, @@ -980,10 +2048,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k8) new Context10( + resourceTracker, k1, v1, k2, @@ -1003,10 +2071,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k9) new Context10( + resourceTracker, k1, v1, k2, @@ -1026,10 +2094,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k10) new Context10( + resourceTracker, k1, v1, k2, @@ -1049,10 +2117,10 @@ object Local { k9, v9, k11, - v11 - ) + v11) else if (k eq k11) new Context10( + resourceTracker, k1, v1, k2, @@ -1072,13 +2140,13 @@ object Local { k9, v9, k10, - v10 - ) + v10) else this def set(k: Key, v: Some[_]): Context = if (k eq k1) new Context11( + resourceTracker, k1, v, k2, @@ -1100,10 +2168,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k2) new Context11( + resourceTracker, k1, v1, k2, @@ -1125,10 +2193,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k3) new Context11( + resourceTracker, k1, v1, k2, @@ -1150,10 +2218,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k4) new Context11( + resourceTracker, k1, v1, k2, @@ -1175,10 +2243,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k5) new Context11( + resourceTracker, k1, v1, k2, @@ -1200,10 +2268,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k6) new Context11( + resourceTracker, k1, v1, k2, @@ -1225,10 +2293,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k7) new Context11( + resourceTracker, k1, v1, k2, @@ -1250,10 +2318,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k8) new Context11( + resourceTracker, k1, v1, k2, @@ -1275,10 +2343,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k9) new Context11( + resourceTracker, k1, v1, k2, @@ -1300,10 +2368,10 @@ object Local { k10, v10, k11, - v11 - ) + v11) else if (k eq k10) new Context11( + resourceTracker, k1, v1, k2, @@ -1325,10 +2393,10 @@ object Local { k10, v, k11, - v11 - ) + v11) else if (k eq k11) new Context11( + resourceTracker, k1, v1, k2, @@ -1350,10 +2418,10 @@ object Local { k10, v10, k11, - v - ) + v) else new Context12( + resourceTracker, k1, v1, k2, @@ -1377,11 +2445,11 @@ object Local { k11, v11, k, - v - ) + v) } private final class Context12( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -1406,7 +2474,60 @@ object Local { v11: Some[_], k12: Key, v12: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context12( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_]) + + def removeResourceTracker(): Context = new Context12( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -1426,6 +2547,7 @@ object Local { def remove(k: Key): Context = if (k eq k1) new Context11( + resourceTracker, k2, v2, k3, @@ -1447,10 +2569,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k2) new Context11( + resourceTracker, k1, v1, k3, @@ -1472,10 +2594,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k3) new Context11( + resourceTracker, k1, v1, k2, @@ -1497,10 +2619,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k4) new Context11( + resourceTracker, k1, v1, k2, @@ -1522,10 +2644,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k5) new Context11( + resourceTracker, k1, v1, k2, @@ -1547,10 +2669,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k6) new Context11( + resourceTracker, k1, v1, k2, @@ -1572,10 +2694,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k7) new Context11( + resourceTracker, k1, v1, k2, @@ -1597,10 +2719,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k8) new Context11( + resourceTracker, k1, v1, k2, @@ -1622,10 +2744,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k9) new Context11( + resourceTracker, k1, v1, k2, @@ -1647,10 +2769,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k10) new Context11( + resourceTracker, k1, v1, k2, @@ -1672,10 +2794,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k11) new Context11( + resourceTracker, k1, v1, k2, @@ -1697,10 +2819,10 @@ object Local { k10, v10, k12, - v12 - ) + v12) else if (k eq k12) new Context11( + resourceTracker, k1, v1, k2, @@ -1722,13 +2844,13 @@ object Local { k10, v10, k11, - v11 - ) + v11) else this def set(k: Key, v: Some[_]): Context = if (k eq k1) new Context12( + resourceTracker, k1, v, k2, @@ -1752,10 +2874,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k2) new Context12( + resourceTracker, k1, v1, k2, @@ -1779,10 +2901,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k3) new Context12( + resourceTracker, k1, v1, k2, @@ -1806,10 +2928,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k4) new Context12( + resourceTracker, k1, v1, k2, @@ -1833,10 +2955,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k5) new Context12( + resourceTracker, k1, v1, k2, @@ -1860,10 +2982,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k6) new Context12( + resourceTracker, k1, v1, k2, @@ -1887,10 +3009,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k7) new Context12( + resourceTracker, k1, v1, k2, @@ -1914,10 +3036,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k8) new Context12( + resourceTracker, k1, v1, k2, @@ -1941,10 +3063,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k9) new Context12( + resourceTracker, k1, v1, k2, @@ -1968,10 +3090,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k10) new Context12( + resourceTracker, k1, v1, k2, @@ -1995,10 +3117,10 @@ object Local { k11, v11, k12, - v12 - ) + v12) else if (k eq k11) new Context12( + resourceTracker, k1, v1, k2, @@ -2022,10 +3144,10 @@ object Local { k11, v, k12, - v12 - ) + v12) else if (k eq k12) new Context12( + resourceTracker, k1, v1, k2, @@ -2049,10 +3171,10 @@ object Local { k11, v11, k12, - v - ) + v) else new Context13( + resourceTracker, k1, v1, k2, @@ -2078,11 +3200,11 @@ object Local { k12, v12, k, - v - ) + v) } private final class Context13( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -2109,7 +3231,64 @@ object Local { v12: Some[_], k13: Key, v13: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context13( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_]) + + def removeResourceTracker(): Context = new Context13( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -2130,6 +3309,7 @@ object Local { def remove(k: Key): Context = if (k eq k1) new Context12( + resourceTracker, k2, v2, k3, @@ -2153,10 +3333,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k2) new Context12( + resourceTracker, k1, v1, k3, @@ -2180,10 +3360,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k3) new Context12( + resourceTracker, k1, v1, k2, @@ -2207,10 +3387,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k4) new Context12( + resourceTracker, k1, v1, k2, @@ -2234,10 +3414,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k5) new Context12( + resourceTracker, k1, v1, k2, @@ -2261,10 +3441,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k6) new Context12( + resourceTracker, k1, v1, k2, @@ -2288,10 +3468,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k7) new Context12( + resourceTracker, k1, v1, k2, @@ -2315,10 +3495,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k8) new Context12( + resourceTracker, k1, v1, k2, @@ -2342,10 +3522,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k9) new Context12( + resourceTracker, k1, v1, k2, @@ -2369,10 +3549,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k10) new Context12( + resourceTracker, k1, v1, k2, @@ -2396,10 +3576,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k11) new Context12( + resourceTracker, k1, v1, k2, @@ -2423,10 +3603,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k12) new Context12( + resourceTracker, k1, v1, k2, @@ -2450,10 +3630,10 @@ object Local { k11, v11, k13, - v13 - ) + v13) else if (k eq k13) new Context12( + resourceTracker, k1, v1, k2, @@ -2477,13 +3657,13 @@ object Local { k11, v11, k12, - v12 - ) + v12) else this def set(k: Key, v: Some[_]): Context = if (k eq k1) new Context13( + resourceTracker, k1, v, k2, @@ -2509,10 +3689,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k2) new Context13( + resourceTracker, k1, v1, k2, @@ -2538,10 +3718,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k3) new Context13( + resourceTracker, k1, v1, k2, @@ -2567,10 +3747,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k4) new Context13( + resourceTracker, k1, v1, k2, @@ -2596,10 +3776,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k5) new Context13( + resourceTracker, k1, v1, k2, @@ -2625,10 +3805,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k6) new Context13( + resourceTracker, k1, v1, k2, @@ -2654,10 +3834,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k7) new Context13( + resourceTracker, k1, v1, k2, @@ -2683,10 +3863,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k8) new Context13( + resourceTracker, k1, v1, k2, @@ -2712,10 +3892,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k9) new Context13( + resourceTracker, k1, v1, k2, @@ -2741,10 +3921,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k10) new Context13( + resourceTracker, k1, v1, k2, @@ -2770,10 +3950,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k11) new Context13( + resourceTracker, k1, v1, k2, @@ -2799,10 +3979,10 @@ object Local { k12, v12, k13, - v13 - ) + v13) else if (k eq k12) new Context13( + resourceTracker, k1, v1, k2, @@ -2828,10 +4008,10 @@ object Local { k12, v, k13, - v13 - ) + v13) else if (k eq k13) new Context13( + resourceTracker, k1, v1, k2, @@ -2857,10 +4037,10 @@ object Local { k12, v12, k13, - v - ) + v) else new Context14( + resourceTracker, k1, v1, k2, @@ -2888,11 +4068,11 @@ object Local { k13, v13, k, - v - ) + v) } private final class Context14( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -2921,7 +4101,68 @@ object Local { v13: Some[_], k14: Key, v14: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context14( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_], + k14: Key, + v14: Some[_]) + + def removeResourceTracker(): Context = new Context14( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_], + k14: Key, + v14: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -2943,6 +4184,7 @@ object Local { def remove(k: Key): Context = if (k eq k1) new Context13( + resourceTracker, k2, v2, k3, @@ -2968,10 +4210,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k2) new Context13( + resourceTracker, k1, v1, k3, @@ -2997,10 +4239,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k3) new Context13( + resourceTracker, k1, v1, k2, @@ -3026,10 +4268,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k4) new Context13( + resourceTracker, k1, v1, k2, @@ -3055,10 +4297,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k5) new Context13( + resourceTracker, k1, v1, k2, @@ -3084,10 +4326,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k6) new Context13( + resourceTracker, k1, v1, k2, @@ -3113,10 +4355,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k7) new Context13( + resourceTracker, k1, v1, k2, @@ -3142,10 +4384,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k8) new Context13( + resourceTracker, k1, v1, k2, @@ -3171,10 +4413,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k9) new Context13( + resourceTracker, k1, v1, k2, @@ -3200,10 +4442,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k10) new Context13( + resourceTracker, k1, v1, k2, @@ -3229,10 +4471,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k11) new Context13( + resourceTracker, k1, v1, k2, @@ -3258,10 +4500,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k12) new Context13( + resourceTracker, k1, v1, k2, @@ -3287,10 +4529,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k13) new Context13( + resourceTracker, k1, v1, k2, @@ -3316,10 +4558,10 @@ object Local { k12, v12, k14, - v14 - ) + v14) else if (k eq k14) new Context13( + resourceTracker, k1, v1, k2, @@ -3345,13 +4587,13 @@ object Local { k12, v12, k13, - v13 - ) + v13) else this def set(k: Key, v: Some[_]): Context = if (k eq k1) new Context14( + resourceTracker, k1, v, k2, @@ -3379,10 +4621,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k2) new Context14( + resourceTracker, k1, v1, k2, @@ -3410,10 +4652,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k3) new Context14( + resourceTracker, k1, v1, k2, @@ -3441,10 +4683,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k4) new Context14( + resourceTracker, k1, v1, k2, @@ -3472,10 +4714,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k5) new Context14( + resourceTracker, k1, v1, k2, @@ -3503,10 +4745,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k6) new Context14( + resourceTracker, k1, v1, k2, @@ -3534,10 +4776,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k7) new Context14( + resourceTracker, k1, v1, k2, @@ -3565,10 +4807,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k8) new Context14( + resourceTracker, k1, v1, k2, @@ -3596,10 +4838,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k9) new Context14( + resourceTracker, k1, v1, k2, @@ -3627,10 +4869,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k10) new Context14( + resourceTracker, k1, v1, k2, @@ -3658,10 +4900,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k11) new Context14( + resourceTracker, k1, v1, k2, @@ -3689,10 +4931,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k12) new Context14( + resourceTracker, k1, v1, k2, @@ -3720,10 +4962,10 @@ object Local { k13, v13, k14, - v14 - ) + v14) else if (k eq k13) new Context14( + resourceTracker, k1, v1, k2, @@ -3751,10 +4993,10 @@ object Local { k13, v, k14, - v14 - ) + v14) else if (k eq k14) new Context14( + resourceTracker, k1, v1, k2, @@ -3782,10 +5024,10 @@ object Local { k13, v13, k14, - v - ) + v) else new Context15( + resourceTracker, k1, v1, k2, @@ -3815,11 +5057,11 @@ object Local { k14, v14, k, - v - ) + v) } private final class Context15( + resourceTracker: Option[ResourceTracker], k1: Key, v1: Some[_], k2: Key, @@ -3850,7 +5092,72 @@ object Local { v14: Some[_], k15: Key, v15: Some[_]) - extends Context { + extends Context(resourceTracker) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context15( + Some(tracker), + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_], + k14: Key, + v14: Some[_], + k15: Key, + v15: Some[_]) + + def removeResourceTracker(): Context = new Context15( + None, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_], + k14: Key, + v14: Some[_], + k15: Key, + v15: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -3873,6 +5180,7 @@ object Local { def remove(k: Key): Context = if (k eq k1) new Context14( + resourceTracker, k2, v2, k3, @@ -3900,10 +5208,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k2) new Context14( + resourceTracker, k1, v1, k3, @@ -3931,10 +5239,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k3) new Context14( + resourceTracker, k1, v1, k2, @@ -3962,10 +5270,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k4) new Context14( + resourceTracker, k1, v1, k2, @@ -3993,10 +5301,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k5) new Context14( + resourceTracker, k1, v1, k2, @@ -4024,10 +5332,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k6) new Context14( + resourceTracker, k1, v1, k2, @@ -4055,10 +5363,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k7) new Context14( + resourceTracker, k1, v1, k2, @@ -4086,10 +5394,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k8) new Context14( + resourceTracker, k1, v1, k2, @@ -4117,10 +5425,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k9) new Context14( + resourceTracker, k1, v1, k2, @@ -4148,10 +5456,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k10) new Context14( + resourceTracker, k1, v1, k2, @@ -4179,10 +5487,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k11) new Context14( + resourceTracker, k1, v1, k2, @@ -4210,10 +5518,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k12) new Context14( + resourceTracker, k1, v1, k2, @@ -4241,10 +5549,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k13) new Context14( + resourceTracker, k1, v1, k2, @@ -4272,10 +5580,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k14) new Context14( + resourceTracker, k1, v1, k2, @@ -4303,10 +5611,10 @@ object Local { k13, v13, k15, - v15 - ) + v15) else if (k eq k15) new Context14( + resourceTracker, k1, v1, k2, @@ -4334,13 +5642,13 @@ object Local { k13, v13, k14, - v14 - ) + v14) else this def set(k: Key, v: Some[_]): Context = if (k eq k1) new Context15( + resourceTracker, k1, v, k2, @@ -4370,10 +5678,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k2) new Context15( + resourceTracker, k1, v1, k2, @@ -4403,10 +5711,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k3) new Context15( + resourceTracker, k1, v1, k2, @@ -4436,10 +5744,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k4) new Context15( + resourceTracker, k1, v1, k2, @@ -4469,10 +5777,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k5) new Context15( + resourceTracker, k1, v1, k2, @@ -4502,10 +5810,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k6) new Context15( + resourceTracker, k1, v1, k2, @@ -4535,10 +5843,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k7) new Context15( + resourceTracker, k1, v1, k2, @@ -4568,10 +5876,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k8) new Context15( + resourceTracker, k1, v1, k2, @@ -4601,10 +5909,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k9) new Context15( + resourceTracker, k1, v1, k2, @@ -4634,10 +5942,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k10) new Context15( + resourceTracker, k1, v1, k2, @@ -4667,10 +5975,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k11) new Context15( + resourceTracker, k1, v1, k2, @@ -4700,10 +6008,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k12) new Context15( + resourceTracker, k1, v1, k2, @@ -4733,10 +6041,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k13) new Context15( + resourceTracker, k1, v1, k2, @@ -4766,10 +6074,10 @@ object Local { k14, v14, k15, - v15 - ) + v15) else if (k eq k14) new Context15( + resourceTracker, k1, v1, k2, @@ -4799,10 +6107,10 @@ object Local { k14, v, k15, - v15 - ) + v15) else if (k eq k15) new Context15( + resourceTracker, k1, v1, k2, @@ -4832,12 +6140,16 @@ object Local { k14, v14, k15, - v - ) + v) else new ContextN(k, v, this) } - private final class ContextN(kN: Key, vN: Some[_], rest: Context) extends Context { + private final class ContextN(kN: Key, vN: Some[_], rest: Context) + extends Context(rest.resourceTracker) { + + def setResourceTracker(tracker: ResourceTracker): Context = + new ContextN(kN, vN, rest.setResourceTracker(tracker)) + def removeResourceTracker(): Context = new ContextN(kN, vN, rest.removeResourceTracker()) def get(k: Key): Option[_] = if (k eq kN) vN diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index c97f0df42e..1d6692caa4 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -218,6 +218,7 @@ object Promise { * provided result */ private final class Monitored[A](saved: Local.Context, k: Try[A] => Unit) extends K[A] { + def apply(result: Try[A]): Unit = { val current = Local.save() if (current ne saved) @@ -425,6 +426,7 @@ object Promise { */ class Promise[A] extends Future[A] with Updatable[Try[A]] { import Promise._ + import ResourceTracker.wrapAndMeasureUsage /** * Note: exceptions in responds are monitored. That is, if the @@ -433,23 +435,28 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { * [[Monitor]] for details. */ final def respond(k: Try[A] => Unit): Future[A] = { - continue(new Monitored(Local.save(), k)) + val saved = Local.save() + val tracker = saved.resourceTracker + if (tracker eq None) continue(new Monitored(saved, k)) + else continue(new Monitored(saved, wrapAndMeasureUsage(k, tracker.get))) this } final def transform[B](f: Try[A] => Future[B]): Future[B] = { val promise = interrupts[B](this) - - continue(new FutureTransformer(Local.save(), f, promise)) - + val saved = Local.save() + val tracker = saved.resourceTracker + if (tracker eq None) continue(new FutureTransformer(saved, f, promise)) + else continue(new FutureTransformer(saved, wrapAndMeasureUsage(f, tracker.get), promise)) promise } final protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { val promise = interrupts[B](this) - - continue(new TryTransformer(Local.save(), f, promise)) - + val saved = Local.save() + val tracker = saved.resourceTracker + if (tracker eq None) continue(new TryTransformer(saved, f, promise)) + else continue(new TryTransformer(saved, wrapAndMeasureUsage(f, tracker.get), promise)) promise } diff --git a/util-core/src/main/scala/com/twitter/util/ResourceTracker.scala b/util-core/src/main/scala/com/twitter/util/ResourceTracker.scala new file mode 100644 index 0000000000..0c38e290d4 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/ResourceTracker.scala @@ -0,0 +1,114 @@ +package com.twitter.util + +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.LongAdder + +private[twitter] object ResourceTracker { + private[this] final val threadMxBean = java.lang.management.ManagementFactory.getThreadMXBean() + + /** + * An indicator if the running JDK supports querying thread statistics. + */ + final val threadCpuTimeSupported = threadMxBean.isCurrentThreadCpuTimeSupported + + /** + * Retreive the current thread's cpu time only if [[threadCpuTimeSupported]] is true. + * Otherwise, -1 is returned. Results may vary depending on the JDK's management implementation + */ + final def currentThreadCpuTime: Long = { + if (threadCpuTimeSupported) threadMxBean.getCurrentThreadCpuTime + else -1 + } + + /** + * Wrap the invocation of `f` with low-level resource usage that's pushed to the `tracker`. + */ + private[util] def wrapAndMeasureUsage[A, B]( + f: A => B, + tracker: ResourceTracker + ): A => B = { (result) => + { + tracker.incContinuations() + val cpuStart = threadMxBean.getCurrentThreadCpuTime + try f(result) + finally { + val cpuTime = threadMxBean.getCurrentThreadCpuTime - cpuStart + tracker.addCpuTime(cpuTime) + } + } + } + + /** + * Wrap the invocation of `f` with low-level resource usage that's pushed to the `tracker`. + */ + private[util] def wrapAndMeasureUsage[T]( + f: => T, + tracker: ResourceTracker + ): () => T = { () => + { + tracker.incContinuations() + val cpuStart = threadMxBean.getCurrentThreadCpuTime + try f + finally { + val cpuTime = threadMxBean.getCurrentThreadCpuTime - cpuStart + tracker.addCpuTime(cpuTime) + } + } + } + + // exposed for testing + private[util] def let[T](tracker: ResourceTracker)(f: => T): T = { + val oldCtx = Local.save() + val newCtx = oldCtx.setResourceTracker(tracker) + Local.restore(newCtx) + try f + finally Local.restore(oldCtx) + } + + // exposed for testing + private[util] def set(tracker: ResourceTracker): Unit = { + val oldCtx = Local.save() + val newCtx = oldCtx.setResourceTracker(tracker) + Local.restore(newCtx) + } + + // exposed for testing + private[util] def clear(): Unit = { + val oldCtx = Local.save() + val newCtx = oldCtx.removeResourceTracker() + if (newCtx ne oldCtx) Local.restore(newCtx) + } + + /** + * Track the execution of completed [[Future]]s within `f`. + */ + def apply[A](f: ResourceTracker => A) = { + val accumulator = new ResourceTracker() + let(accumulator)(f(accumulator)) + } +} + +/** + * A class used as the value for the local [[ResourceTracker#resourceTracker]] that's updated + * with low level resource usage as [[Promise]] continuations are executed. + * + * @note some measurements such as the cpu time relies on the JDK's implementation of the + * management interfaces. This means that the results may vary between JDK implementations. + */ +private[twitter] class ResourceTracker { + private[this] val cpuTime: LongAdder = new LongAdder() + private[this] val continuations: AtomicInteger = new AtomicInteger() + + private[util] def addCpuTime(time: Long): Unit = cpuTime.add(time) + private[util] def incContinuations(): Unit = continuations.getAndIncrement() + + /** + * Retrieve the total accumulated CPU time + */ + def totalCpuTime(): Long = cpuTime.sum() + + /** + * Retrieve the total number of executed continuations + */ + def numContinuations(): Int = continuations.get() +} diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index 722c08b8c0..d1faf6abf8 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -17,6 +17,28 @@ class FuturePoolTest extends AnyFunSuite with Eventually { implicit override val patienceConfig: PatienceConfig = PatienceConfig(timeout = scaled(Span(15, Seconds)), interval = scaled(Span(5, Millis))) + class MockTracker(var updates: Long = 0) extends ResourceTracker { + override def addCpuTime(time: Long): Unit = { + updates = updates + 1 + } + } + + test("ResourceTracker, when set -- FuturePool tracks usage on execution") { + if (ResourceTracker.threadCpuTimeSupported) { + val executor = Executors.newFixedThreadPool(1) + val pool = FuturePool(executor) + + val mockTracker = new MockTracker + val source = new Promise[Int] + val result = ResourceTracker.let(mockTracker) { + pool(Await.result(source)) // simulate blocking call + } + + source.setValue(1); Await.result(result) + assert(mockTracker.updates == 1) + } + } + test("FuturePool should dispatch to another thread") { val executor = Executors.newFixedThreadPool(1) val pool = FuturePool(executor) diff --git a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala index 88e2aeb213..7eb2e68b43 100644 --- a/util-core/src/test/scala/com/twitter/util/PromiseTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PromiseTest.scala @@ -20,6 +20,62 @@ class PromiseTest extends AnyFunSuite { } } + class MockTracker(var updates: Long = 0) extends ResourceTracker { + override def addCpuTime(time: Long): Unit = { + updates = updates + 1 + } + } + + test("ResourceTracker when set, updates per completed continuation") { + if (ResourceTracker.threadCpuTimeSupported) { + val mockTracker = new MockTracker() + val p1 = new Promise[Unit]() + val p = ResourceTracker.let(mockTracker) { + // 2 continuations + p1.map(_ => ()).respond(_ => ()) + + // 1 continuation + // + // **Note**: Because [[ConstFuture]] implements `transformTry` synchronously, + // the cpuTime is not updated since on `Future.Unit#map`, instead accounted for + // by the saved context of the parent continuation, `flatMap`. + p1.flatMap(_ => Future.Unit.map(_ => ())) + } + p1.setDone() + assert(mockTracker.updates == 3) + } + } + + test("ResourceTracker updates in the presence of exceptions") { + if (ResourceTracker.threadCpuTimeSupported) { + val mockTracker = new MockTracker() + val p = ResourceTracker.let(mockTracker) { + Future.Unit.transform(_ => throw new Exception("boom again")) + } + assert(mockTracker.updates == 1) + } + } + + test("ResourceTracker handles nested promise") { + if (ResourceTracker.threadCpuTimeSupported) { + val mockTracker = new MockTracker() + val p1 = new Promise[Unit]() + val p2 = new Promise[Unit]() + + val p = ResourceTracker.let(mockTracker) { + p1.flatMap { _ => + p2.map(_ => ()) + } + } + + p1.setDone() + assert(mockTracker.updates == 1) + + p2.setDone(); Await.result(p) + assert(mockTracker.updates == 2) + } + } + test("Promise.detach should not detach other attached promises") { val p = new Promise[Unit] val attached1 = Promise.attached(p) From 137667a64c13391390fb5e796641490829bdde8a Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 24 Mar 2022 18:48:52 +0000 Subject: [PATCH 645/796] util-stats: Introduce DefaultExpression Problem/Solution Introduce a utility to provide common default expressions. Refactor `MethodBuilderRegistry` depend on the new APIs. This new utility can be shared among customers when they want to build expressions with different `Expression.apply` APIs. JIRA Issues: CSL-11826 Differential Revision: https://phabricator.twitter.biz/D855070 --- .../finagle/stats/exp/DefaultExpression.scala | 81 +++++++++++++++++++ .../stats/exp/DefaultExpressionTest.scala | 39 +++++++++ 2 files changed, 120 insertions(+) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/exp/DefaultExpression.scala create mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/exp/DefaultExpressionTest.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/DefaultExpression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/DefaultExpression.scala new file mode 100644 index 0000000000..69c321d336 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/DefaultExpression.scala @@ -0,0 +1,81 @@ +package com.twitter.finagle.stats.exp + +import com.twitter.finagle.stats.exp.ExpressionNames._ +import com.twitter.finagle.stats.Milliseconds +import com.twitter.finagle.stats.Percentage +import com.twitter.finagle.stats.Requests + +/** + * A utility object to create common [[ExpressionSchema]]s with user defined + * [[Expression]]s. + */ +private[twitter] object DefaultExpression { + + /** + * Create an [[ExpressionSchema]] to calculate success rate with a user + * defined {@code success} [[Expression]] and a {@code failure} + * [[Expression]]. + * + * @param success an [[Expression]] to measure the success metric. + * @param failure an [[Expression]] to measure the failure metric. + * @return an [[ExpressionSchema]] that calculates success rate from success + * and failure metric. The default unit is [[Percentage]], the + * default bounds is [[MonotoneThresholds]] with operator as + * [[GreaterThan]], bad threshold as 99.5, good threshold as 99.97. + * + */ + def successRate(success: Expression, failure: Expression): ExpressionSchema = + ExpressionSchema( + successRateName, + Expression(100).multiply(success.divide(success.plus(failure)))) + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.97)) + .withUnit(Percentage) + .withDescription(s"Default $successRateName expression") + + /** + * Create an [[ExpressionSchema]] that calculates throughput with a user + * defined {@code throughput} [[Expression]]. + * + * @param throughput an [[Expression]] to measure the throughput metric + * @return an [[ExpressionSchema]] that calculates the throughput. The + * default unit is [[Requests]]. + */ + def throughput(throughput: Expression): ExpressionSchema = + requestUnitExpression(throughputName, throughput) + + /** + * Create an [[ExpressionSchema]] that calculates failures with a user + * defined {@code failures} [[Expression]]. + * + * @param failures an [[Expression]] to measure the failures metric + * @return an [[ExpressionSchema]] that calculates the failures. The + * default unit is [[Requests]]. + */ + def failures(failures: Expression): ExpressionSchema = + requestUnitExpression(failuresName, failures) + + /** + * Create an [[ExpressionSchema]] that calculates p99 latencies with a user + * defined {@code latencyP99} [[Expression]]. + * + * @param latencyP99 an [[Expression]] to measure the p99 latencies metric. + * @return an [[ExpressionSchema]] that calculates the p99 latencies. The + * default unit is [[Milliseconds]]. + */ + def latency99(latencyP99: Expression): ExpressionSchema = + ExpressionSchema(latencyName, latencyP99) + .withUnit(Milliseconds) + .withDescription("Default P99 latency expression") + + private[this] def requestUnitExpression(name: String, expression: Expression) = + ExpressionSchema(name, expression) + .withUnit(Requests) + .withDescription(s"Default $name expression") +} + +private[twitter] object ExpressionNames { + val successRateName = "success_rate" + val throughputName = "throughput" + val latencyName = "latency" + val failuresName = "failures" +} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/DefaultExpressionTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/DefaultExpressionTest.scala new file mode 100644 index 0000000000..a7142ec4e9 --- /dev/null +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/DefaultExpressionTest.scala @@ -0,0 +1,39 @@ +package com.twitter.finagle.stats.exp + +import com.twitter.finagle.stats.MetricUnit +import com.twitter.finagle.stats.Milliseconds +import com.twitter.finagle.stats.Percentage +import com.twitter.finagle.stats.Requests +import org.scalatest.funsuite.AnyFunSuite + +class DefaultExpressionTest extends AnyFunSuite { + test("success rate expression") { + val successRate = DefaultExpression.successRate(Expression(97), Expression(3)) + assertExpression(successRate, ExpressionNames.successRateName, Percentage) + assert(successRate.bounds == MonotoneThresholds(GreaterThan, 99.5, 99.97)) + } + + test("throughput") { + val throughput = DefaultExpression.throughput(Expression(7777)) + assertExpression(throughput, ExpressionNames.throughputName, Requests) + } + + test("failures") { + val failures = DefaultExpression.failures(Expression(0)) + assertExpression(failures, ExpressionNames.failuresName, Requests) + } + + test("latency p99") { + val latencyP99 = DefaultExpression.latency99(Expression(1000)) + assertExpression(latencyP99, ExpressionNames.latencyName, Milliseconds) + } + + private[this] def assertExpression( + expression: ExpressionSchema, + name: String, + unit: MetricUnit + ): Unit = { + assert(expression.name == name) + assert(expression.unit == unit) + } +} From f0697231b4240eeb434d0b34431d49f9c1c15289 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Fri, 25 Mar 2022 19:25:12 +0000 Subject: [PATCH 646/796] util-stats: more support for creating a Stat expression **Problem/Solution** Move `HistogramComponent` to its own class **Problem/Solution** Provide default percentiles in `HistogramComponent` **Problem** Percentile is not a component of `HistogramComponent`, to create a Stat expression, users need to pass either a `HistogramComponent` or a percentile as `Double`. **Solution** Introduce `Percentile` as a new `HistogramComponent` and refactor the API to create a Stat expression to take a `HistogramComponent`. This makes it easier for us to introduce a new API to create a list of Stat expressions with different `HistogramComponent` (will be done in a separate phab). JIRA Issues: CSL-11828 Differential Revision: https://phabricator.twitter.biz/D855103 --- .../finagle/stats/exp/Expression.scala | 18 ++++----------- .../finagle/stats/exp/ExpressionSchema.scala | 23 ++++++++++--------- .../stats/exp/HistogramComponent.scala | 14 +++++++++++ .../stats/InMemoryStatsReceiverTest.scala | 5 ++-- .../stats/exp/ExpressionSchemaTest.scala | 17 +++++++++----- 5 files changed, 44 insertions(+), 33 deletions(-) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index 04771beb1b..935fac5869 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -5,18 +5,9 @@ import com.twitter.finagle.stats.Metadata import com.twitter.finagle.stats.MetricBuilder import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.StatsReceiver -import com.twitter.finagle.stats.exp.Expression.HistogramComponent import scala.annotation.varargs private[twitter] object Expression { - - sealed trait HistogramComponent - case object Min extends HistogramComponent - case object Max extends HistogramComponent - case object Avg extends HistogramComponent - case object Sum extends HistogramComponent - case object Count extends HistogramComponent - private[exp] def getStatsReceivers(expr: Expression): Set[StatsReceiver] = expr match { case FunctionExpression(_, exprs) => exprs.foldLeft(Set.empty[StatsReceiver]) { @@ -37,12 +28,11 @@ private[twitter] object Expression { /** * Create a histogram expression - * @param component the histogram component either a [[HistogramComponent]] for Left - * or a percentile in Double for Right. + * @param component a [[HistogramComponent]] */ def apply( metadata: Metadata, - component: Either[HistogramComponent, Double] + component: HistogramComponent ): Expression = { metadata.toMetricBuilder match { case Some(metricBuilder) => @@ -124,11 +114,11 @@ case class MetricExpression private[exp] (metricBuilder: MetricBuilder, showRoll /** * Represent a histogram expression with specified component, for example the average, or a percentile - * @param component either a [[HistogramComponent]] or a percentile in Double + * @param component a [[HistogramComponent]] */ case class HistogramExpression private[exp] ( metricBuilder: MetricBuilder, - component: Either[HistogramComponent, Double]) + component: HistogramComponent) extends Expression /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index f13ceb3561..7bd44b50ed 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -4,11 +4,12 @@ import com.twitter.finagle.stats.HistogramFormatter import com.twitter.finagle.stats.MetricUnit import com.twitter.finagle.stats.SourceRole import com.twitter.finagle.stats.Unspecified -import com.twitter.finagle.stats.exp.Expression.Avg -import com.twitter.finagle.stats.exp.Expression.Count -import com.twitter.finagle.stats.exp.Expression.Max -import com.twitter.finagle.stats.exp.Expression.Min -import com.twitter.finagle.stats.exp.Expression.Sum +import com.twitter.finagle.stats.exp.HistogramComponent.Avg +import com.twitter.finagle.stats.exp.HistogramComponent.Count +import com.twitter.finagle.stats.exp.HistogramComponent.Max +import com.twitter.finagle.stats.exp.HistogramComponent.Min +import com.twitter.finagle.stats.exp.HistogramComponent.Percentile +import com.twitter.finagle.stats.exp.HistogramComponent.Sum import com.twitter.util.Try /** @@ -116,12 +117,12 @@ private[twitter] object ExpressionSchema { private def histogramLabel(histoExpr: HistogramExpression): Map[String, String] = { val value = histoExpr.component match { - case Right(percentile) => HistogramFormatter.labelPercentile(percentile) - case Left(Min) => HistogramFormatter.labelMin - case Left(Max) => HistogramFormatter.labelMax - case Left(Avg) => HistogramFormatter.labelAverage - case Left(Sum) => HistogramFormatter.labelSum - case Left(Count) => HistogramFormatter.labelCount + case Percentile(percentile) => HistogramFormatter.labelPercentile(percentile) + case Min => HistogramFormatter.labelMin + case Max => HistogramFormatter.labelMax + case Avg => HistogramFormatter.labelAverage + case Sum => HistogramFormatter.labelSum + case Count => HistogramFormatter.labelCount } Map("bucket" -> value) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala new file mode 100644 index 0000000000..d9693b4c59 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala @@ -0,0 +1,14 @@ +package com.twitter.finagle.stats.exp + +private[twitter] object HistogramComponent { + case object Min extends HistogramComponent + case object Max extends HistogramComponent + case object Avg extends HistogramComponent + case object Sum extends HistogramComponent + case object Count extends HistogramComponent + case class Percentile(percentile: Double) extends HistogramComponent + val defaultPercentiles: Seq[HistogramComponent] = + Seq(Percentile(50), Percentile(99), Percentile(999), Percentile(9999)) +} + +private[twitter] sealed trait HistogramComponent diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 312f386622..6b93ddd48d 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -6,6 +6,7 @@ import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.exp.Expression import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.finagle.stats.exp.ExpressionSchemaKey +import com.twitter.finagle.stats.exp.HistogramComponent import java.io.ByteArrayOutputStream import java.io.PrintStream import java.nio.charset.StandardCharsets @@ -382,7 +383,7 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra val expression = ExpressionSchema( "test_expression", - Expression(aCounter.metadata).plus(Expression(bHisto.metadata, Left(Expression.Min)) + Expression(aCounter.metadata).plus(Expression(bHisto.metadata, HistogramComponent.Min) .plus(Expression(cGauge.metadata))) ).build() @@ -397,7 +398,7 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra val expected_expression = ExpressionSchema( "test_expression", Expression(aaSchema).plus( - Expression(bbSchema, Left(Expression.Min)).plus(Expression(ccSchema)))) + Expression(bbSchema, HistogramComponent.Min).plus(Expression(ccSchema)))) assert(sr.expressions .get(ExpressionSchemaKey("test_expression", Map(), Nil)).get.expr == expected_expression.expr) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index e230064109..67433452f4 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -52,9 +52,10 @@ class ExpressionSchemaTest extends AnyFunSuite { .withUnit(Percentage) .withDescription("The success rate of the slow query") - val latency = ExpressionSchema("latency", Expression(latencyMb, Right(0.99))) - .withUnit(Milliseconds) - .withDescription("The latency of the slow query") + val latency = + ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Percentile(0.99))) + .withUnit(Milliseconds) + .withDescription("The latency of the slow query") successRate.build() latency.build() @@ -102,10 +103,14 @@ class ExpressionSchemaTest extends AnyFunSuite { test("histogram expressions come with default labels") { new Ctx { - val latencyP90 = ExpressionSchema("latency", Expression(latencyMb, Right(0.9))).build() - val latencyP99 = ExpressionSchema("latency", Expression(latencyMb, Right(0.99))).build() + val latencyP90 = + ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Percentile(0.9))) + .build() + val latencyP99 = + ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Percentile(0.99))) + .build() val latencyAvg = - ExpressionSchema("latency", Expression(latencyMb, Left(Expression.Avg))).build() + ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Avg)).build() assert(sr.expressions.contains(nameToKey("latency", Map("bucket" -> "p90")))) assert(sr.expressions.contains(nameToKey("latency", Map("bucket" -> "p99")))) From a7958355a4eb6f54229f56fc84bb1fa9546abcd3 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 28 Mar 2022 21:13:34 +0000 Subject: [PATCH 647/796] util-stats: Deprecate methods on the MetricBuilder API Problem We use MetricBuilder as or source-of-truth representation of a metrics identity. However, it also has a `StatsReceiver` as a field that kind of messes up this clean representation. Solution Deprecate the `statsReceiver` field and remove the metric instantiation methods. Differential Revision: https://phabricator.twitter.biz/D862244 --- CHANGELOG.rst | 7 +++++++ .../scala/com/twitter/finagle/stats/MetricBuilder.scala | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 04b1d31448..88903c6cc7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -42,6 +42,13 @@ Breaking API Changes * util-core: The `c.t.util.Responder` trait has been removed. ``PHAB_ID=D824830`` +Deprecations +~~~~~~~~~~~~ + +* util-stats: Deprecated methods on `MetricBuilder` for directly instantiating metrics so that we can + eventually remove the `statsReceiever` field from `MetricBuilder` and let it just be the source of + truth for defining a metric. ``PHAB_ID=D862244`` + 22.1.0 ------ diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index d8661a114e..851b8cdf98 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -182,6 +182,7 @@ class MetricBuilder private ( val percentiles: IndexedSeq[Double], val isStandard: Boolean, val metricType: MetricType, + @deprecated("Being removed to make MetricBuilder strictly a metric definition.", "2022-03-28") val statsReceiver: StatsReceiver) extends Metadata { @@ -306,6 +307,7 @@ class MetricBuilder private ( * Produce a counter as described by the builder inside the underlying StatsReceiver. * @return the counter created. */ + @deprecated("Use the StatsReceiver interface for instantiation.", "2022-03-28") @varargs final def counter(name: String*): Counter = { this.statsReceiver.validateMetricType(this, CounterType) @@ -316,6 +318,7 @@ class MetricBuilder private ( * Produce a gauge as described by the builder inside the underlying StatsReceiver. * @return the gauge created. */ + @deprecated("Use the StatsReceiver interface for instantiation.", "2022-03-28") final def gauge(name: String*)(f: => Float): Gauge = { this.statsReceiver.validateMetricType(this, GaugeType) this.statsReceiver.addGauge(this.withName(name: _*))(f) @@ -326,6 +329,7 @@ class MetricBuilder private ( * This API is for Java compatibility * @return the gauge created. */ + @deprecated("Use the StatsReceiver interface for instantiation.", "2022-03-28") @varargs final def gauge(f: Supplier[Float], name: String*): Gauge = { this.statsReceiver.validateMetricType(this, GaugeType) @@ -336,6 +340,7 @@ class MetricBuilder private ( * Produce a histogram as described by the builder inside the underlying StatsReceiver. * @return the histogram created. */ + @deprecated("Use the StatsReceiver interface for instantiation.", "2022-03-28") @varargs final def histogram(name: String*): Stat = { this.statsReceiver.validateMetricType(this, HistogramType) From c213dc6002f153fed91ea642e40b460897f2ce43 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 29 Mar 2022 18:34:34 +0000 Subject: [PATCH 648/796] Twitter-oss: Prepare OSS libraries for release 22.3.0 Problem We want to release the next versions of our Twitter OSS libraries 22.3.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D863328 --- CHANGELOG.rst | 17 ++++++++++------- README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 88903c6cc7..e03be71315 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,16 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.3.0 +------ + +Deprecations +~~~~~~~~~~~~ + +* util-stats: Deprecated methods on `MetricBuilder` for directly instantiating metrics so that we can + eventually remove the `statsReceiever` field from `MetricBuilder` and let it just be the source of + truth for defining a metric. ``PHAB_ID=D862244`` + 22.2.0 ------ @@ -42,13 +52,6 @@ Breaking API Changes * util-core: The `c.t.util.Responder` trait has been removed. ``PHAB_ID=D824830`` -Deprecations -~~~~~~~~~~~~ - -* util-stats: Deprecated methods on `MetricBuilder` for directly instantiating metrics so that we can - eventually remove the `statsReceiever` field from `MetricBuilder` and let it just be the source of - truth for defining a metric. ``PHAB_ID=D862244`` - 22.1.0 ------ diff --git a/README.md b/README.md index 8b9be22496..2d77bfa40e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -40,7 +40,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "22.2.0" +val utilCore = "com.twitter" %% "util-core" % "22.3.0" ``` # Units diff --git a/build.sbt b/build.sbt index 87b375a1d4..e31377b18b 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.3.0-SNAPSHOT" +val releaseVersion = "22.3.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.2" From 20e5f809abdb45471f5046ee151b8a62fdfd3b01 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Tue, 29 Mar 2022 19:23:38 +0000 Subject: [PATCH 649/796] Twitter-oss: update OSS libraries post-release to 22.4.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D863501 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2d77bfa40e..0342ca6d74 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index e31377b18b..c782e02490 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.3.0" +val releaseVersion = "22.4.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.2" From 3685d7251f0fc43a6f5746ee6e562c5c91ed34a3 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 29 Mar 2022 22:58:23 +0000 Subject: [PATCH 650/796] util-stats: Remove metric instantiation methods from `MetricBuilder` Problem We have a StatsReceiver that lives on MetricBuilder. It doesn't need to be there. It's currently used mostly for supporting expressions and their convenient `.build()` method which hunts for a stats receiver in the MetricSchema to register the expression with. Solution In order to unwind this we first need to remove the metric creation methods from MetricBuilder. In the future we'll make expressions register themselves with a global instance of a StatsReceiver or even a new interface that feels more purpose driven. Differential Revision: https://phabricator.twitter.biz/D859036 --- CHANGELOG.rst | 6 +++ .../main/scala/com/twitter/jvm/JvmStats.scala | 52 ++++++++++++------- .../twitter/finagle/stats/MetricBuilder.scala | 45 ---------------- .../MetricBuilderCompilationTest.java | 15 ------ .../finagle/stats/MetricBuilderTest.scala | 16 ------ 5 files changed, 38 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e03be71315..b155cc937a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-stats: The metric instantiation methods have been removed from `MetricBuilder`. Use the methods on + `StatsReceiver` to instantiate metrics instead. ``PHAB_ID=D859036`` + 22.3.0 ------ diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 7cd025aad1..84ebafd291 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -80,7 +80,8 @@ object JvmStats { case compilation => val compilationStats = stats.scope("compilation") gauges.add( - compilationStats.metricBuilder(GaugeType).withCounterishGauge.gauge("time_msec") { + compilationStats.addGauge( + compilationStats.metricBuilder(GaugeType).withCounterishGauge.withName("time_msec")) { compilation.getTotalCompilationTime().toFloat }) } @@ -88,11 +89,13 @@ object JvmStats { val classes = ManagementFactory.getClassLoadingMXBean() val classLoadingStats = stats.scope("classes") gauges.add( - classLoadingStats.metricBuilder(GaugeType).withCounterishGauge.gauge("total_loaded") { + classLoadingStats.addGauge( + classLoadingStats.metricBuilder(GaugeType).withCounterishGauge.withName("total_loaded")) { classes.getTotalLoadedClassCount().toFloat }) gauges.add( - classLoadingStats.metricBuilder(GaugeType).withCounterishGauge.gauge("total_unloaded") { + classLoadingStats.addGauge( + classLoadingStats.metricBuilder(GaugeType).withCounterishGauge.withName("total_unloaded")) { classes.getUnloadedClassCount().toFloat }) gauges.add( @@ -142,15 +145,20 @@ object JvmStats { } val spStats = stats.scope("safepoint") - gauges.add(spStats.metricBuilder(GaugeType).withCounterishGauge.gauge("sync_time_millis") { - jvm.safepoint.syncTimeMillis.toFloat - }) - gauges.add(spStats.metricBuilder(GaugeType).withCounterishGauge.gauge("total_time_millis") { - jvm.safepoint.totalTimeMillis.toFloat - }) - gauges.add(spStats.metricBuilder(GaugeType).withCounterishGauge.gauge("count") { - jvm.safepoint.count.toFloat - }) + gauges.add( + spStats.addGauge( + spStats.metricBuilder(GaugeType).withCounterishGauge.withName("sync_time_millis")) { + jvm.safepoint.syncTimeMillis.toFloat + }) + gauges.add( + spStats.addGauge( + spStats.metricBuilder(GaugeType).withCounterishGauge.withName("total_time_millis")) { + jvm.safepoint.totalTimeMillis.toFloat + }) + gauges.add( + spStats.addGauge(spStats.metricBuilder(GaugeType).withCounterishGauge.withName("count")) { + jvm.safepoint.count.toFloat + }) ManagementFactory.getPlatformMXBeans(classOf[BufferPoolMXBean]) match { case null => @@ -169,10 +177,12 @@ object JvmStats { gcPool.foreach { gc => val name = gc.getName.regexSub("""[^\w]""".r) { m => "_" } val poolCycles = - gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "cycles") { + gcStats.addGauge( + gcStats.metricBuilder(GaugeType).withCounterishGauge.withName(name, "cycles")) { gc.getCollectionCount.toFloat } - val poolMsec = gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge(name, "msec") { + val poolMsec = gcStats.addGauge( + gcStats.metricBuilder(GaugeType).withCounterishGauge.withName(name, "msec")) { gc.getCollectionTime.toFloat } @@ -195,12 +205,14 @@ object JvmStats { } // note, these could be -1 if the collector doesn't have support for it. - val cycles = gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge("cycles") { - gcPool.map(_.getCollectionCount).filter(_ > 0).sum.toFloat - } - val msec = gcStats.metricBuilder(GaugeType).withCounterishGauge.gauge("msec") { - gcPool.map(_.getCollectionTime).filter(_ > 0).sum.toFloat - } + val cycles = + gcStats.addGauge(gcStats.metricBuilder(GaugeType).withCounterishGauge.withName("cycles")) { + gcPool.map(_.getCollectionCount).filter(_ > 0).sum.toFloat + } + val msec = + gcStats.addGauge(gcStats.metricBuilder(GaugeType).withCounterishGauge.withName("msec")) { + gcPool.map(_.getCollectionTime).filter(_ > 0).sum.toFloat + } gauges.add(cycles) gauges.add(msec) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 851b8cdf98..79890d78f5 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -6,7 +6,6 @@ import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.MetricBuilder.MetricType import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter -import java.util.function.Supplier import scala.annotation.varargs /** @@ -303,50 +302,6 @@ class MetricBuilder private ( private[MetricBuilder] final def counterishGaugeSchema: MetricBuilder = this.copy(metricType = CounterishGaugeType) - /** - * Produce a counter as described by the builder inside the underlying StatsReceiver. - * @return the counter created. - */ - @deprecated("Use the StatsReceiver interface for instantiation.", "2022-03-28") - @varargs - final def counter(name: String*): Counter = { - this.statsReceiver.validateMetricType(this, CounterType) - this.statsReceiver.counter(this.withName(name: _*)) - } - - /** - * Produce a gauge as described by the builder inside the underlying StatsReceiver. - * @return the gauge created. - */ - @deprecated("Use the StatsReceiver interface for instantiation.", "2022-03-28") - final def gauge(name: String*)(f: => Float): Gauge = { - this.statsReceiver.validateMetricType(this, GaugeType) - this.statsReceiver.addGauge(this.withName(name: _*))(f) - } - - /** - * Produce a gauge as described by the builder inside the underlying StatsReceiver. - * This API is for Java compatibility - * @return the gauge created. - */ - @deprecated("Use the StatsReceiver interface for instantiation.", "2022-03-28") - @varargs - final def gauge(f: Supplier[Float], name: String*): Gauge = { - this.statsReceiver.validateMetricType(this, GaugeType) - this.statsReceiver.addGauge(this.withName(name: _*))(f.get()) - } - - /** - * Produce a histogram as described by the builder inside the underlying StatsReceiver. - * @return the histogram created. - */ - @deprecated("Use the StatsReceiver interface for instantiation.", "2022-03-28") - @varargs - final def histogram(name: String*): Stat = { - this.statsReceiver.validateMetricType(this, HistogramType) - this.statsReceiver.stat(this.withName(name: _*)) - } - def canEqual(other: Any): Boolean = other.isInstanceOf[MetricBuilder] override def equals(other: Any): Boolean = other match { diff --git a/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java b/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java index 1ed057a60b..9148e9feff 100644 --- a/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java @@ -4,14 +4,11 @@ import scala.Some; -import com.twitter.finagle.stats.Counter; -import com.twitter.finagle.stats.Gauge; import com.twitter.finagle.stats.InMemoryStatsReceiver; import com.twitter.finagle.stats.MetricBuilder; import com.twitter.finagle.stats.MetricTypes; import com.twitter.finagle.stats.Microseconds; import com.twitter.finagle.stats.NoRoleSpecified; -import com.twitter.finagle.stats.Stat; import com.twitter.finagle.stats.StatsReceiver; import com.twitter.finagle.stats.Verbosity; @@ -45,16 +42,4 @@ public void testWithMethods() { .withRelativeName("cool", "name") .withPercentiles(0.99, 0.88, 0.77); } - - @Test - public void testConstructingMetrics() { - StatsReceiver sr = new InMemoryStatsReceiver(); - MetricBuilder gmb = sr.metricBuilder(MetricTypes.GAUGE_TYPE); - MetricBuilder smb = sr.metricBuilder(MetricTypes.HISTOGRAM_TYPE); - MetricBuilder cmb = sr.metricBuilder(MetricTypes.COUNTER_TYPE); - - Gauge g = gmb.gauge(() -> 3.0f, "my", "cool", "gauge"); - Stat s = smb.histogram("my", "cool", "histogram"); - Counter c = cmb.counter("my", "cool", "counter"); - } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala index 7d433e8957..4c29c73e87 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala @@ -1,8 +1,6 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.MetricBuilder.CounterType -import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType -import com.twitter.finagle.stats.MetricBuilder.GaugeType import org.scalatest.funsuite.AnyFunSuite class MetricBuilderTest extends AnyFunSuite { @@ -18,18 +16,4 @@ class MetricBuilderTest extends AnyFunSuite { metadataScopeSeparator.setSeparator("_aTerriblyLongStringForSomeReason_") assert(mb.toString.contains("foo_aTerriblyLongStringForSomeReason_bar")) } - - test("the .gauge() api builds CounterishGauge vs Gauge where appropriate") { - val mb = MetricBuilder( - name = Seq("c_gauge"), - metricType = GaugeType, - statsReceiver = new InMemoryStatsReceiver - ).withCounterishGauge - - assert(mb.metricType == CounterishGaugeType) - - val g = mb.gauge("c_gauge") { 1.0.toFloat } - - assert(g.metadata.toMetricBuilder.get.metricType == CounterishGaugeType) - } } From 835b3da5ee04e7aed53db249d46a2bc4a37bd229 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 30 Mar 2022 16:42:27 +0000 Subject: [PATCH 651/796] util-stats: Add ability to tell MetricBuilder that the dimensional name is not fully defined Problem We have a lot of metrics already defined in hierarchical form and we need to add labels to them before they're meaningful in dimensional form. Solution Add a new field to MetricBuilder that speaks to the validity of the label state. This will also be helpful in the future for figuring out what metrics are not fully labeled. Differential Revision: https://phabricator.twitter.biz/D852745 --- .../finagle/stats/InMemoryStatsReceiver.scala | 24 ++- .../twitter/finagle/stats/MetricBuilder.scala | 83 +++++++-- .../stats/NameTranslatingStatsReceiver.scala | 8 +- .../twitter/finagle/stats/StatsReceiver.scala | 111 +++++++++--- .../stats/InMemoryStatsReceiverTest.scala | 164 +++++++++++++++++- .../finagle/stats/StatsReceiverTest.scala | 164 +++++++++++++++++- 6 files changed, 486 insertions(+), 68 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index ebe74d4246..b827566ae2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -83,8 +83,16 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { val expressions: mutable.Map[ExpressionSchemaKey, ExpressionSchema] = new ConcurrentHashMap[ExpressionSchemaKey, ExpressionSchema]().asScala - override def counter(name: String*): ReadableCounter = - counter(this.metricBuilder(CounterType).withName(name: _*)) + override def counter(name: String*): ReadableCounter = { + val metricBuilder = this + .metricBuilder(CounterType) + .withName(name: _*) + + counter( + if (name.length > 1) metricBuilder.withHierarchicalOnly + else metricBuilder.withDimensionalSupport + ) + } /** * Creates a [[ReadableCounter]] of the given `name`. @@ -117,8 +125,16 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } } - override def stat(name: String*): ReadableStat = - stat(this.metricBuilder(HistogramType).withName(name: _*)) + override def stat(name: String*): ReadableStat = { + val metricBuilder = this + .metricBuilder(HistogramType) + .withName(name: _*) + + stat( + if (name.length > 1) metricBuilder.withHierarchicalOnly + else metricBuilder.withDimensionalSupport + ) + } /** * Creates a [[ReadableStat]] of the given `name`. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 79890d78f5..d35f49dec0 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -4,6 +4,7 @@ import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.MetricBuilder.Identity import com.twitter.finagle.stats.MetricBuilder.MetricType import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter import scala.annotation.varargs @@ -68,7 +69,6 @@ object MetricBuilder { verbosity: Verbosity = Verbosity.Default, sourceClass: Option[String] = None, name: Seq[String] = Seq.empty, - labels: Map[String, String] = Map.empty, relativeName: Seq[String] = Seq.empty, processPath: Option[String] = None, percentiles: IndexedSeq[Double] = IndexedSeq.empty, @@ -83,8 +83,9 @@ object MetricBuilder { role, verbosity, sourceClass, - name, - labels, + // For now we're not allowing construction with labels. + // They need to be added later via `.withLabels`. + Identity.Hierarchical(name, Map.empty), relativeName, processPath, percentiles, @@ -140,6 +141,36 @@ object MetricBuilder { def toJsonString: String = "unlatched_counter" def toPrometheusString: String = "counter" } + + /** Identity information about this metric */ + sealed abstract class Identity private () { + def hierarchicalName: Seq[String] + def labels: Map[String, String] + } + + /** + * Experimental way to represent both hierarchical and dimensional identity information. + * + * @note this type is expected to undergo some churn as dimensional metric support matures. + */ + object Identity { + + /** Hierarchical only [[Identity]] representation */ + final case class Hierarchical(hierarchicalName: Seq[String], labels: Map[String, String]) + extends Identity { + override def toString: String = { + hierarchicalName.mkString("Hierarchical(", metadataScopeSeparator(), s", $labels)") + } + } + + /** Full [[Identity]] representation including a [[Hierarchical]] name and dimensional labels */ + final case class Full(hierarchicalName: Seq[String], labels: Map[String, String]) + extends Identity { + override def toString: String = { + hierarchicalName.mkString("Full(", metadataScopeSeparator(), s", $labels)") + } + } + } } sealed trait Metadata { @@ -173,8 +204,7 @@ class MetricBuilder private ( val role: SourceRole, val verbosity: Verbosity, val sourceClass: Option[String], - val name: Seq[String], - private[finagle] val labels: Map[String, String], + val identity: Identity, val relativeName: Seq[String], val processPath: Option[String], // Only persisted and relevant when building histograms. @@ -197,8 +227,7 @@ class MetricBuilder private ( role: SourceRole = this.role, verbosity: Verbosity = this.verbosity, sourceClass: Option[String] = this.sourceClass, - name: Seq[String] = this.name, - labels: Map[String, String] = this.labels, + identity: Identity = this.identity, relativeName: Seq[String] = this.relativeName, processPath: Option[String] = this.processPath, isStandard: Boolean = this.isStandard, @@ -212,8 +241,7 @@ class MetricBuilder private ( role = role, verbosity = verbosity, sourceClass = sourceClass, - name = name, - labels = labels, + identity = identity, relativeName = relativeName, processPath = processPath, percentiles = percentiles, @@ -250,11 +278,31 @@ class MetricBuilder private ( def withRole(role: SourceRole): MetricBuilder = this.copy(role = role) @varargs - def withName(name: String*): MetricBuilder = this.copy(name = name) + def withName(name: String*): MetricBuilder = { + val nextIdentity = this.identity match { + case Identity.Hierarchical(_, labels) => Identity.Hierarchical(name, labels) + case Identity.Full(_, labels) => Identity.Full(name, labels) + } + copy(identity = nextIdentity) + } + + def name: Seq[String] = identity.hierarchicalName + + // Private for now + private[twitter] def withLabels(labels: Map[String, String]): MetricBuilder = identity match { + case Identity.Hierarchical(name, _) => copy(identity = Identity.Hierarchical(name, labels)) + case Identity.Full(name, _) => copy(identity = Identity.Full(name, labels)) + } - // Temporarily private while API's are under development - private[finagle] def withLabels(labels: Map[String, String]): MetricBuilder = - this.copy(labels = labels) + private[twitter] def withHierarchicalOnly: MetricBuilder = identity match { + case Identity.Hierarchical(_, _) => this + case Identity.Full(name, labels) => this.copy(identity = Identity.Hierarchical(name, labels)) + } + + private[twitter] def withDimensionalSupport: MetricBuilder = identity match { + case Identity.Hierarchical(name, labels) => this.copy(identity = Identity.Full(name, labels)) + case Identity.Full(_, _) => this + } @varargs def withRelativeName(relativeName: String*): MetricBuilder = @@ -313,8 +361,7 @@ class MetricBuilder private ( role == that.role && verbosity == that.verbosity && sourceClass == that.sourceClass && - name == that.name && - labels == that.labels && + identity == that.identity && relativeName == that.relativeName && processPath == that.processPath && percentiles == that.percentiles && @@ -330,8 +377,7 @@ class MetricBuilder private ( role, verbosity, sourceClass, - name, - labels, + identity, relativeName, processPath, percentiles, @@ -341,7 +387,6 @@ class MetricBuilder private ( } override def toString(): String = { - val nameString = name.mkString(metadataScopeSeparator()) - s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $nameString, $relativeName, $processPath, $percentiles, $metricType, $statsReceiver)" + s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $identity, $relativeName, $processPath, $percentiles, $metricType, $statsReceiver)" } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index 6767076a25..6264e02cb5 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -5,7 +5,6 @@ package com.twitter.finagle.stats * names according to a `translate(name: Seq[String])` function. * * @param self The underlying StatsReceiver to which translated names are passed - * * @param namespacePrefix the namespace used for translations */ abstract class NameTranslatingStatsReceiver( @@ -17,8 +16,11 @@ abstract class NameTranslatingStatsReceiver( protected def translate(name: Seq[String]): Seq[String] - final protected def translate(builder: MetricBuilder): MetricBuilder = - builder.withName(translate(builder.name): _*) + final protected def translate(builder: MetricBuilder): MetricBuilder = { + // We need to make sure we've marked the dimensional metrics as 'poisoned' + // since we didn't give it a label. + builder.withName(translate(builder.name): _*).withHierarchicalOnly + } override def toString: String = s"$self/$namespacePrefix" } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index fb2f5e6556..9b9da471fd 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -129,27 +129,41 @@ trait StatsReceiver { * Get a [[Counter counter]] with the given `description` and `name`. */ @varargs - def counter(description: Some[String], name: String*): Counter = + final def counter(description: Some[String], name: String*): Counter = counter(description.get, Verbosity.Default, name: _*) /** * Get a [[Counter counter]] with the given `name`. */ @varargs - def counter(verbosity: Verbosity, name: String*): Counter = - counter(this.metricBuilder(CounterType).withVerbosity(verbosity).withName(name: _*)) + final def counter(verbosity: Verbosity, name: String*): Counter = { + val builder = this + .metricBuilder(CounterType) + .withVerbosity(verbosity) + .withName(name: _*) + + counter( + if (name.length > 1) builder.withHierarchicalOnly + else builder.withDimensionalSupport + ) + } /** * Get a [[Counter counter]] with the given `description` and `name`. */ @varargs - def counter(description: String, verbosity: Verbosity, name: String*): Counter = + final def counter(description: String, verbosity: Verbosity, name: String*): Counter = { + val builder = this + .metricBuilder(CounterType) + .withVerbosity(verbosity) + .withName(name: _*) + .withDescription(description) + counter( - this - .metricBuilder(CounterType) - .withVerbosity(verbosity) - .withName(name: _*) - .withDescription(description)) + if (name.length > 1) builder.withHierarchicalOnly + else builder.withDimensionalSupport + ) + } /** * Get a [[Counter counter]] with the given schema. @@ -166,27 +180,41 @@ trait StatsReceiver { * Get a [[Stat stat]] with the given `description` and `name`. */ @varargs - def stat(description: Some[String], name: String*): Stat = + final def stat(description: Some[String], name: String*): Stat = stat(description.get, Verbosity.Default, name: _*) /** * Get a [[Stat stat]] with the given name. */ @varargs - def stat(verbosity: Verbosity, name: String*): Stat = - stat(this.metricBuilder(HistogramType).withVerbosity(verbosity).withName(name: _*)) + final def stat(verbosity: Verbosity, name: String*): Stat = { + val builder = this + .metricBuilder(HistogramType) + .withVerbosity(verbosity) + .withName(name: _*) + + stat( + if (name.length > 1) builder.withHierarchicalOnly + else builder.withDimensionalSupport + ) + } /** * Get a [[Stat stat]] with the given `description` and `name`. */ @varargs - def stat(description: String, verbosity: Verbosity, name: String*): Stat = + final def stat(description: String, verbosity: Verbosity, name: String*): Stat = { + val builder = this + .metricBuilder(HistogramType) + .withVerbosity(verbosity) + .withName(name: _*) + .withDescription(description) + stat( - this - .metricBuilder(HistogramType) - .withVerbosity(verbosity) - .withName(name: _*) - .withDescription(description)) + if (name.length > 1) builder.withHierarchicalOnly + else builder.withDimensionalSupport + ) + } /** * Get a [[Stat stat]] with the given schema. @@ -266,17 +294,39 @@ trait StatsReceiver { * * @see [[https://docs.oracle.com/javase/7/docs/api/java/lang/ref/WeakReference.html java.lang.ref.WeakReference]] */ - def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = - addGauge(this.metricBuilder(GaugeType).withVerbosity(verbosity).withName(name: _*))(f) + final def addGauge(verbosity: Verbosity, name: String*)(f: => Float): Gauge = { + val builder = this + .metricBuilder(GaugeType) + .withVerbosity(verbosity) + .withName(name: _*) + + addGauge( + if (name.length > 1) builder.withHierarchicalOnly + else builder.withDimensionalSupport + )(f) + } /** * Add the function `f` as a [[Gauge gauge]] with the given name and description. */ - def addGauge(description: String, verbosity: Verbosity, name: String*)(f: => Float): Gauge = + final def addGauge( + description: String, + verbosity: Verbosity, + name: String* + )( + f: => Float + ): Gauge = { + val builder = this + .metricBuilder(GaugeType) + .withVerbosity(verbosity) + .withName(name: _*) + .withDescription(description) + addGauge( - this - .metricBuilder(GaugeType).withVerbosity(verbosity).withName(name: _*).withDescription( - description))(f) + if (name.length > 1) builder.withHierarchicalOnly + else builder.withDimensionalSupport + )(f) + } /** * Just like $AddGaugeScaladocLink but optimized for better Java experience. @@ -288,8 +338,17 @@ trait StatsReceiver { * Just like $AddVerboseGaugeScaladocLink but optimized for better Java experience. */ @varargs - def addGauge(f: Supplier[Float], verbosity: Verbosity, name: String*): Gauge = - addGauge(this.metricBuilder(GaugeType).withVerbosity(verbosity).withName(name: _*))(f.get()) + final def addGauge(f: Supplier[Float], verbosity: Verbosity, name: String*): Gauge = { + val builder = this + .metricBuilder(GaugeType) + .withVerbosity(verbosity) + .withName(name: _*) + + addGauge( + if (name.length > 1) builder.withHierarchicalOnly + else builder.withDimensionalSupport + )(f.get()) + } /** * Add the function `f` as a [[Gauge gauge]] with the given name. diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 6b93ddd48d..52f5afd86d 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -3,6 +3,7 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.MetricBuilder.Identity import com.twitter.finagle.stats.exp.Expression import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.finagle.stats.exp.ExpressionSchemaKey @@ -349,8 +350,7 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra val stats = new InMemoryStatsReceiver() stats.addGauge("coolGauge") { 3 } stats.counter("sweet", "counter") - val radSr = stats.scope("rad") - radSr.stat("histo") + stats.scope("rad").stat("histo") val baos = new ByteArrayOutputStream() val ps = new PrintStream(baos, true, "utf-8") @@ -362,13 +362,13 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(parts.length == 3) assert( parts( - 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, coolGauge, List(), None, Vector(), GaugeType, InMemoryStatsReceiver)") + 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Full(coolGauge, Map()), List(), None, Vector(), GaugeType, InMemoryStatsReceiver)") assert( parts( - 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, rad/histo, List(), None, Vector(), HistogramType, InMemoryStatsReceiver/rad)") + 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Hierarchical(rad/histo, Map()), List(), None, Vector(), HistogramType, InMemoryStatsReceiver/rad)") assert( parts( - 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, sweet/counter, List(), None, Vector(), CounterType, InMemoryStatsReceiver)") + 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Hierarchical(sweet/counter, Map()), List(), None, Vector(), CounterType, InMemoryStatsReceiver)") } finally { ps.close() } @@ -389,11 +389,20 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra // what we expected as hydrated metric builders val aaSchema = - MetricBuilder(name = Seq("test", "a"), metricType = CounterType, statsReceiver = sr) + MetricBuilder( + name = Seq("test", "a"), + metricType = CounterType, + statsReceiver = sr).withHierarchicalOnly val bbSchema = - MetricBuilder(name = Seq("test", "b"), metricType = HistogramType, statsReceiver = sr) + MetricBuilder( + name = Seq("test", "b"), + metricType = HistogramType, + statsReceiver = sr).withHierarchicalOnly val ccSchema = - MetricBuilder(name = Seq("test", "c"), metricType = GaugeType, statsReceiver = sr) + MetricBuilder( + name = Seq("test", "c"), + metricType = GaugeType, + statsReceiver = sr).withHierarchicalOnly val expected_expression = ExpressionSchema( "test_expression", @@ -429,4 +438,143 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(sr.getAllExpressionsWithLabel("nonexistent key", "nonexistent value").isEmpty) } + private class SupportsDimensionalStatsReceiver extends InMemoryStatsReceiver { + var supportsDimensional: java.lang.Boolean = null + + private def hasDimensions(mb: MetricBuilder): Boolean = mb.identity match { + case Identity.Hierarchical(_, _) => false + case Identity.Full(_, _) => true + } + + def reset(): Unit = { + supportsDimensional = null + } + + override def counter(metricBuilder: MetricBuilder): ReadableCounter = { + val c = super.counter(metricBuilder) + supportsDimensional = hasDimensions(c.metadata.toMetricBuilder.get) + c + } + + override def stat(metricBuilder: MetricBuilder): ReadableStat = { + val s = super.stat(metricBuilder) + supportsDimensional = hasDimensions(s.metadata.toMetricBuilder.get) + s + } + + override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { + val g = super.addGauge(metricBuilder)(f) + supportsDimensional = hasDimensions(g.metadata.toMetricBuilder.get) + g + } + } + + test("StatsReceiver.scope.counter: dimensional metrics are disabled without a label") { + val receiver = new SupportsDimensionalStatsReceiver + val scoped = receiver.scope("foo") + + scoped.counter("bar") + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.counter: varargs metrics names longer than 1 disable dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.counter("foo", "bar") + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.counter(Some("description"), "foo", "bar") + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.counter("description", Verbosity.Default, "foo", "bar") + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.counter: varargs metrics names of 1 allow dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.counter("bar") + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.counter(Some("description"), "bar") + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.counter("description", Verbosity.Default, "bar") + assert(receiver.supportsDimensional) + } + + test("StatsReceiver.scope.addGauge: dimensional metrics are disabled without a label") { + val receiver = new SupportsDimensionalStatsReceiver + val scoped = receiver.scope("foo") + + scoped.addGauge("bar") { 1.0f } + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.addGauge: varargs metrics names longer than 1 disable dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.addGauge("foo", "bar") { 1.0f } + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.addGauge(Some("description"), "foo", "bar") { 1.0f } + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.addGauge("description", Verbosity.Default, "foo", "bar") { 1.0f } + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.addGauge: varargs metrics names of 1 allow dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.addGauge("bar") { 1.0f } + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.addGauge(Some("description"), "bar") { 1.0f } + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.addGauge("description", Verbosity.Default, "bar") { 1.0f } + assert(receiver.supportsDimensional) + } + + test("StatsReceiver.scope.stat: dimensional metrics are disabled without a label") { + val receiver = new SupportsDimensionalStatsReceiver + val scoped = receiver.scope("foo") + + scoped.stat("bar") + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.stat: varargs metrics names longer than 1 disable dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.stat("foo", "bar") + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.stat(Some("description"), "foo", "bar") + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.stat("description", Verbosity.Default, "foo", "bar") + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.stat: varargs metrics names of 1 allow dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.stat("bar") + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.stat(Some("description"), "bar") + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.stat("description", Verbosity.Default, "bar") + assert(receiver.supportsDimensional) + } + } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index c9144415a7..8423acc7d6 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -1,20 +1,21 @@ package com.twitter.finagle.stats import com.twitter.conversions.DurationOps._ -import com.twitter.finagle.stats.MetricBuilder.{ - CounterType, - CounterishGaugeType, - GaugeType, - HistogramType, - UnlatchedCounter -} -import com.twitter.util.{Await, Future} +import com.twitter.finagle.stats.MetricBuilder.CounterType +import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType +import com.twitter.finagle.stats.MetricBuilder.GaugeType +import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.MetricBuilder.Identity +import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter +import com.twitter.util.Await +import com.twitter.util.Future import java.util.concurrent.TimeUnit import org.mockito.Mockito._ import org.scalatest.funsuite.AnyFunSuite import scala.collection.mutable.ArrayBuffer class StatsReceiverTest extends AnyFunSuite { + test("RollupStatsReceiver counter/stats") { val mem = new InMemoryStatsReceiver val receiver = new RollupStatsReceiver(mem) @@ -98,6 +99,153 @@ class StatsReceiverTest extends AnyFunSuite { verify(receiver, times(3)).stat("2", "chainz") } + private class SupportsDimensionalStatsReceiver extends StatsReceiver { + var supportsDimensional: java.lang.Boolean = null + + private def hasDimensions(mb: MetricBuilder): Boolean = mb.identity match { + case Identity.Hierarchical(_, _) => false + case Identity.Full(_, _) => true + } + + def reset(): Unit = { + supportsDimensional = null + } + + def repr: AnyRef = this + + def counter(metricBuilder: MetricBuilder): Counter = { + supportsDimensional = hasDimensions(metricBuilder) + new Counter { + def incr(delta: Long): Unit = () + def metadata: Metadata = metricBuilder + } + } + + def stat(metricBuilder: MetricBuilder): Stat = { + supportsDimensional = hasDimensions(metricBuilder) + new Stat { + def add(value: Float): Unit = () + def metadata: Metadata = metricBuilder + } + } + + def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { + supportsDimensional = hasDimensions(metricBuilder) + new Gauge { + def remove(): Unit = () + def metadata: Metadata = metricBuilder + } + } + } + + test("StatsReceiver.scope.counter: dimensional metrics are disabled without a label") { + val receiver = new SupportsDimensionalStatsReceiver + val scoped = receiver.scope("foo") + + scoped.counter("bar") + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.counter: varargs metrics names longer than 1 disable dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.counter("foo", "bar") + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.counter(Some("description"), "foo", "bar") + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.counter("description", Verbosity.Default, "foo", "bar") + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.counter: varargs metrics names of 1 allow dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.counter("bar") + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.counter(Some("description"), "bar") + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.counter("description", Verbosity.Default, "bar") + assert(receiver.supportsDimensional) + } + + test("StatsReceiver.scope.addGauge: dimensional metrics are disabled without a label") { + val receiver = new SupportsDimensionalStatsReceiver + val scoped = receiver.scope("foo") + + scoped.addGauge("bar") { 1.0f } + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.addGauge: varargs metrics names longer than 1 disable dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.addGauge("foo", "bar") { 1.0f } + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.addGauge(Some("description"), "foo", "bar") { 1.0f } + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.addGauge("description", Verbosity.Default, "foo", "bar") { 1.0f } + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.addGauge: varargs metrics names of 1 allow dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.addGauge("bar") { 1.0f } + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.addGauge(Some("description"), "bar") { 1.0f } + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.addGauge("description", Verbosity.Default, "bar") { 1.0f } + assert(receiver.supportsDimensional) + } + + test("StatsReceiver.scope.stat: dimensional metrics are disabled without a label") { + val receiver = new SupportsDimensionalStatsReceiver + val scoped = receiver.scope("foo") + + scoped.stat("bar") + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.stat: varargs metrics names longer than 1 disable dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.stat("foo", "bar") + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.stat(Some("description"), "foo", "bar") + assert(!receiver.supportsDimensional) + receiver.reset() + + receiver.stat("description", Verbosity.Default, "foo", "bar") + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.stat: varargs metrics names of 1 allow dimensional metrics") { + val receiver = new SupportsDimensionalStatsReceiver + receiver.stat("bar") + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.stat(Some("description"), "bar") + assert(receiver.supportsDimensional) + receiver.reset() + + receiver.stat("description", Verbosity.Default, "bar") + assert(receiver.supportsDimensional) + } + test("StatsReceiver.scope: prefix stats by a scope string") { val receiver = new InMemoryStatsReceiver val scoped = receiver.scope("foo") From 517deedfa3b6cd9e1da0dcc474887cd6b78b89d9 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 30 Mar 2022 18:35:27 +0000 Subject: [PATCH 652/796] finagle-stats, util-stats: Some PROJECT file updates Problem finagle-stats doesn't have a PROJECT file which makes reviews defer to the finagle level PROJECT file for reviewers. This seems like an oversight. Solution Add a PROJECT file. While we're here, add Lily as an owner since she's been doing a lot of work in this area. Differential Revision: https://phabricator.twitter.biz/D864474 --- util-stats/PROJECT | 1 + 1 file changed, 1 insertion(+) diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 51de3ca05d..584008ecfa 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -3,6 +3,7 @@ owners: - banderson - beisenman + - lhe - mnakamura - roanta - vkostyukov From a82980a483d8ebbeaae36d7763d1e00d4ea695de Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Fri, 1 Apr 2022 00:17:21 +0000 Subject: [PATCH 653/796] util-stats: introduce APIs to create Expressions with StatsReceiver Problem When customers use `StringExpression` to create an `Expression` with a list of metric names, they might need to get the scope of the `StatsReceiver` they are using, and remove/append metric names to it in order to match the counter or stats namespace. Solution Introduce APIs to create `Expression`s with a `StatsReceiver`. JIRA Issues: CSL-11827 Differential Revision: https://phabricator.twitter.biz/D861551 --- .../finagle/stats/exp/Expression.scala | 65 +++++++++++++++++++ .../stats/exp/HistogramComponent.scala | 3 +- .../stats/exp/ExpressionSchemaTest.scala | 47 ++++++++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index 935fac5869..a9a7927b4c 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -73,6 +73,71 @@ private[twitter] object Expression { * @param exprs a sequence of metrics in strings */ def apply(exprs: Seq[String], isCounter: Boolean): Expression = StringExpression(exprs, isCounter) + + /** + * Create an expression from creating a [[com.twitter.finagle.stats.Counter]] + * with a given [[StatsReceiver]]. + * + * @param statsReceiver The [[StatsReceiver]] used by the caller to record metric. + * @param showRollup When true, create an expression with all tail metrics. + * @param name The name to create the [[com.twitter.finagle.stats.Counter]]. + * @return An optional [[Expression]] carrying the counter metadata. + * Return `None` if the metric of name {@code name} is not found + * in the {@code statsReceiver}. + */ + @varargs + def counter( + statsReceiver: StatsReceiver, + showRollup: Boolean, + name: String* + ): Option[Expression] = + Some(this(statsReceiver.counter(name: _*).metadata, showRollup)) + + /** + * Create an expression from creating a [[com.twitter.finagle.stats.Stat]] with + * a given [[StatsReceiver]]. + * + * @param statsReceiver The [[StatsReceiver]] used by the caller to record metric. + * @param histogramComponent A [[HistogramComponent]] used to create a histogram + * expression. + * @param name The name to create the [[com.twitter.finagle.stats.Stat]]. + * @return An optional [[Expression]] carrying the stat metadata. + * Return `None` if the metric of name {@code name} is not + * found in the {@code statsReceiver}. + */ + @varargs + def stat( + statsReceiver: StatsReceiver, + histogramComponent: HistogramComponent, + name: String* + ): Option[Expression] = + Some(this(statsReceiver.stat(name: _*).metadata, histogramComponent)) + + /** + * Create a list of expressions from creating a [[com.twitter.finagle.stats.Stat]] + * for each [[HistogramComponent]] with a given [[StatsReceiver]]. + * + * @param statsReceiver The [[StatsReceiver]] used by the caller to record + * metric. + * @param histogramComponents A list of [[HistogramComponent]] used to create a + * histogram expression. + * @param name The name to create the + * [[com.twitter.finagle.stats.Stat]]. + * @return A sequence of [[Expression]] carrying the stat metadata. + * The order of the returned sequence is in correspondence + * with the oder of the {@code histogramComponents}. Return + * an empty sequence if the metric of name {@code name} is + * not found in the {@code statsReceiver}. + */ + @varargs + def stats( + statsReceiver: StatsReceiver, + histogramComponents: Seq[HistogramComponent], + name: String* + ): Seq[Expression] = { + val statMetadata = statsReceiver.stat(name: _*).metadata + histogramComponents.map(this(statMetadata, _)) + } } /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala index d9693b4c59..954addb69e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala @@ -7,7 +7,8 @@ private[twitter] object HistogramComponent { case object Sum extends HistogramComponent case object Count extends HistogramComponent case class Percentile(percentile: Double) extends HistogramComponent - val defaultPercentiles: Seq[HistogramComponent] = + + val DefaultPercentiles: Seq[HistogramComponent] = Seq(Percentile(50), Percentile(99), Percentile(999), Percentile(9999)) } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 67433452f4..4ceec0eccb 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -4,6 +4,7 @@ import com.twitter.conversions.DurationOps._ import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats._ +import com.twitter.finagle.stats.exp.HistogramComponent.Percentile import com.twitter.util.Stopwatch import com.twitter.util.Time import com.twitter.util.TimeControl @@ -190,4 +191,50 @@ class ExpressionSchemaTest extends AnyFunSuite { Seq()))) LoadedStatsReceiver.self = loadedSr } + + test("use Expression.counter() to create a success rate ExpressionSchema") { + val sr = new InMemoryStatsReceiver + // create a success rate ExpressionSchema + // it is safe to call `.get` since we don't return `None` for now + ExpressionSchema( + "success_rate", + Expression(100).multiply( + Expression + .counter(sr, showRollup = false, "clnt", "aclient", "success").get.divide(Expression + .counter(sr, showRollup = false, "clnt", "aclient", "success").get.plus( + Expression.counter(sr, showRollup = true, "clnt", "aclient", "failures").get))) + ).withRole(Client).withServiceName("aclient").build() + + assert(sr.expressions.values.size == 1) + assert( + sr.expressions.contains( + ExpressionSchemaKey( + "success_rate", + Map("role" -> "Client", "service_name" -> "aclient"), + Seq()))) + } + + test( + "use Expression.stats() to create a list of ExpressionSchema with default histogram percentiles") { + val sr = new InMemoryStatsReceiver + // create a list histogram ExpressionSchema with default percentiles + val expressions: Seq[Expression] = + Expression.stats(sr, HistogramComponent.DefaultPercentiles, "clnt", "aclient", "latencies") + expressions.map { expression => + ExpressionSchema("latencies", expression).withRole(Client).withServiceName("aclient").build() + } + + assert(sr.expressions.values.size == expressions.size) + for (Percentile(p) <- HistogramComponent.DefaultPercentiles) { + assert( + sr.expressions.contains( + ExpressionSchemaKey( + "latencies", + Map( + "bucket" -> HistogramFormatter.labelPercentile(p), + "role" -> "Client", + "service_name" -> "aclient"), + Seq()))) + } + } } From 67b70557cfba18becbac3269bb64854ea142ca0a Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 7 Apr 2022 04:29:16 +0000 Subject: [PATCH 654/796] util-stats: Add a TranslatingStatsReceiver that will add a label and also prepend the label value as a scope Problem We want to be able to define metrics that can be identified by both a hierarchical name and a dimensional name, but are otherwise the same metric. However, we normally use "scoped" StatsReceivers which are fundamentally hierarchical only. Solution Add a new way of scoping a StatsReceiver that will allow us to add both the hierarchical prefix and a label at the same time. This will let us progressively migrate our stats by changing the scoping of StatsReceivers. JIRA Issues: CSL-11565 Differential Revision: https://phabricator.twitter.biz/D864997 --- .../twitter/finagle/stats/MetricBuilder.scala | 2 ++ .../stats/ScopeTranslatingStatsReceiver.scala | 34 ++++++++++++++++++ .../twitter/finagle/stats/StatsReceiver.scala | 15 ++++++++ .../finagle/stats/StatsReceiverTest.scala | 35 +++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/ScopeTranslatingStatsReceiver.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index d35f49dec0..2ca5c617ef 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -286,6 +286,8 @@ class MetricBuilder private ( copy(identity = nextIdentity) } + private[finagle] def withIdentity(identity: Identity): MetricBuilder = copy(identity = identity) + def name: Seq[String] = identity.hierarchicalName // Private for now diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/ScopeTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/ScopeTranslatingStatsReceiver.scala new file mode 100644 index 0000000000..dfc7963fab --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/ScopeTranslatingStatsReceiver.scala @@ -0,0 +1,34 @@ +package com.twitter.finagle.stats + +import com.twitter.finagle.stats.MetricBuilder.Identity + +/** + * A [[TranslatingStatsReceiver]] for working with both dimensional and hierarchical metrics. + * + * Translates the [[MetricBuilder]] to prepend the label value as a scope in addition to adding + * it to the labels map. + */ +private class ScopeTranslatingStatsReceiver( + sr: StatsReceiver, + labelName: String, + labelValue: String) + extends TranslatingStatsReceiver(sr) { + + require(labelName.nonEmpty) + require(labelValue.nonEmpty) + + private[this] val labelPair = labelName -> labelValue + + protected def translate(builder: MetricBuilder): MetricBuilder = + builder.withIdentity(newIdentity(builder.identity)) + + private[this] def newIdentity(identity: Identity): Identity = identity match { + case Identity.Full(name, labels) => + Identity.Full(labelValue +: name, labels + labelPair) + case Identity.Hierarchical(name, labels) => + Identity.Hierarchical(labelValue +: name, labels + labelPair) + } + + // We preserve this because unfortunately it is sometimes parsed + override def toString: String = s"$self/$labelValue" +} diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 9b9da471fd..7e791a84bf 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -404,6 +404,21 @@ trait StatsReceiver { } } + /** + * Add a label and prepend the label value to the names of the returned [[StatsReceiver]]. + * + * For example: + * + * {{{ + * statsReceiver.scopeAndLabel("client_name", "backend1").counter("requests") + * }}} + * + * will generate a [[Counter]] with both hierarchical name `backend1/requests` and dimensional name + * `requests {client_name="backend1"}` + */ + def scopeAndLabel(labelName: String, labelValue: String): StatsReceiver = + new ScopeTranslatingStatsReceiver(this, labelName, labelValue) + /** * Prepend `namespace` and `namespaces` to the names of the returned [[StatsReceiver]]. * diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 8423acc7d6..f40dccdc64 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -146,6 +146,41 @@ class StatsReceiverTest extends AnyFunSuite { assert(!receiver.supportsDimensional) } + test("StatsReceiver.scopeAndLabel: a non-empty label name and label value are required") { + val sr = new InMemoryStatsReceiver + intercept[IllegalArgumentException] { + sr.scopeAndLabel("", "cool") + } + + intercept[IllegalArgumentException] { + sr.scopeAndLabel("cool", "") + } + } + + test("StatsReceiver.scopeAndLabel.counter: dimensions are supported") { + val receiver = new SupportsDimensionalStatsReceiver + val scoped = receiver.scopeAndLabel("scope", "foo") + + scoped.counter("bar") + assert(receiver.supportsDimensional) + } + + test("StatsReceiver.scopeAndLabel.scope.counter: dimensional support isn't magically added") { + val receiver = new SupportsDimensionalStatsReceiver + val scoped = receiver.scopeAndLabel("scope", "foo").scope("no_dimensions") + + scoped.counter("bar") + assert(!receiver.supportsDimensional) + } + + test("StatsReceiver.scopeAndLabel: adds both a scope and the labels") { + val sr = new InMemoryStatsReceiver + val c = sr.scopeAndLabel("scope", "good").counter("stuff") + val metricBuilder = c.metadata.toMetricBuilder.get + assert(metricBuilder.name == Seq("good", "stuff")) + assert(metricBuilder.identity.labels == Map("scope" -> "good")) + } + test("StatsReceiver.counter: varargs metrics names longer than 1 disable dimensional metrics") { val receiver = new SupportsDimensionalStatsReceiver receiver.counter("foo", "bar") From 6f35fb12f2be78d184004d306f8733d8249dbbec Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Wed, 13 Apr 2022 19:02:09 +0000 Subject: [PATCH 655/796] Twitter-oss: Prepare OSS libraries for release 22.4.0 Problem We want to release the next versions of our Twitter OSS libraries 22.4.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D874434 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b155cc937a..e220163575 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.4.0 +------ + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 0342ca6d74..fc69bcfe5f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -40,7 +40,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "22.3.0" +val utilCore = "com.twitter" %% "util-core" % "22.4.0" ``` # Units diff --git a/build.sbt b/build.sbt index c782e02490..c2a59661f5 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.4.0-SNAPSHOT" +val releaseVersion = "22.4.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.2" From 90f7a6c3a2191b50a95b18cac60c8d03ba55e81c Mon Sep 17 00:00:00 2001 From: Tony Printezis Date: Thu, 14 Apr 2022 19:34:47 +0000 Subject: [PATCH 656/796] VM-5239: ObjectSizeCalculator will return wrong values with JDK 11 (and fail with JDK 17) JIRA Issues: VM-5239 Differential Revision: https://phabricator.twitter.biz/D866113 --- .../twitter/util/ObjectSizeCalculator.java | 94 +++++++++---------- 1 file changed, 46 insertions(+), 48 deletions(-) diff --git a/util-core/src/test/java/com/twitter/util/ObjectSizeCalculator.java b/util-core/src/test/java/com/twitter/util/ObjectSizeCalculator.java index 86efbb72d1..79fe00b97f 100644 --- a/util-core/src/test/java/com/twitter/util/ObjectSizeCalculator.java +++ b/util-core/src/test/java/com/twitter/util/ObjectSizeCalculator.java @@ -1,7 +1,8 @@ package com.twitter.util; +import com.sun.management.HotSpotDiagnosticMXBean; +import com.sun.management.VMOption; import java.lang.management.ManagementFactory; -import java.lang.management.MemoryPoolMXBean; import java.lang.reflect.Array; import java.lang.reflect.Field; import java.lang.reflect.Modifier; @@ -357,54 +358,51 @@ static MemoryLayoutSpecification getEffectiveMemoryLayoutSpecification() { dataModel + "' of sun.arch.data.model system property"); } - final String strVmVersion = System.getProperty("java.vm.version"); - final int vmVersion = Integer.parseInt(strVmVersion.substring(0, - strVmVersion.indexOf('.'))); - if (vmVersion >= 17) { - long maxMemory = 0; - for (MemoryPoolMXBean mp : ManagementFactory.getMemoryPoolMXBeans()) { - maxMemory += mp.getUsage().getMax(); - } - if (maxMemory < 30L * 1024 * 1024 * 1024) { - // HotSpot 17.0 and above use compressed OOPs below 30GB of RAM total - // for all memory pools (yes, including code cache). - return new MemoryLayoutSpecification() { - @Override public int getArrayHeaderSize() { - return 16; - } - @Override public int getObjectHeaderSize() { - return 12; - } - @Override public int getObjectPadding() { - return 8; - } - @Override public int getReferenceSize() { - return 4; - } - @Override public int getSuperclassFieldPadding() { - return 4; - } - }; - } + final HotSpotDiagnosticMXBean diagBean = ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class); + if (diagBean == null) { + throw new UnsupportedOperationException("ObjectSizeCalculator only supported on HotSpot VM"); } + final VMOption useCoopsOption = diagBean.getVMOption("UseCompressedOops"); + final boolean useCoops = Boolean.parseBoolean(useCoopsOption.getValue()); - // In other cases, it's a 64-bit uncompressed OOPs object model - return new MemoryLayoutSpecification() { - @Override public int getArrayHeaderSize() { - return 24; - } - @Override public int getObjectHeaderSize() { - return 16; - } - @Override public int getObjectPadding() { - return 8; - } - @Override public int getReferenceSize() { - return 8; - } - @Override public int getSuperclassFieldPadding() { - return 8; - } - }; + if (useCoops) { + // Returning the 64-bit compressed OOPs object model + return new MemoryLayoutSpecification() { + @Override public int getArrayHeaderSize() { + return 16; + } + @Override public int getObjectHeaderSize() { + return 12; + } + @Override public int getObjectPadding() { + return 8; + } + @Override public int getReferenceSize() { + return 4; + } + @Override public int getSuperclassFieldPadding() { + return 4; + } + }; + } else { + // Returning the 64-bit non-compressed OOPs object model + return new MemoryLayoutSpecification() { + @Override public int getArrayHeaderSize() { + return 24; + } + @Override public int getObjectHeaderSize() { + return 16; + } + @Override public int getObjectPadding() { + return 8; + } + @Override public int getReferenceSize() { + return 8; + } + @Override public int getSuperclassFieldPadding() { + return 8; + } + }; + } } } From 199167e9bd3c777330094eb44d7c256669e4f08d Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Mon, 18 Apr 2022 22:17:04 +0000 Subject: [PATCH 657/796] Twitter-oss: update OSS libraries post-release to 22.5.0 Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.5.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version Differential Revision: https://phabricator.twitter.biz/D876500 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fc69bcfe5f..f897f3d888 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index c2a59661f5..c77ad996d1 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.4.0" +val releaseVersion = "22.5.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.2" From f4b6cfef1e7bd21f0c40960a5167f84e86672e41 Mon Sep 17 00:00:00 2001 From: Tony Printezis Date: Thu, 21 Apr 2022 20:17:16 +0000 Subject: [PATCH 658/796] VM-5272: NullSslSession.scala with JDK 17: method getPeerCertificateChain needs 'override' modifier JIRA Issues: VM-5272 Differential Revision: https://phabricator.twitter.biz/D868165 --- .../scala/com/twitter/util/security/NullSslSession.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util-security/src/main/scala/com/twitter/util/security/NullSslSession.scala b/util-security/src/main/scala/com/twitter/util/security/NullSslSession.scala index 3152c63017..945a98acdf 100644 --- a/util-security/src/main/scala/com/twitter/util/security/NullSslSession.scala +++ b/util-security/src/main/scala/com/twitter/util/security/NullSslSession.scala @@ -2,7 +2,8 @@ package com.twitter.util.security import java.security.Principal import java.security.cert.Certificate -import javax.net.ssl.{SSLSession, SSLSessionContext} +import javax.net.ssl.SSLSession +import javax.net.ssl.SSLSessionContext import javax.security.cert.X509Certificate /** @@ -18,7 +19,7 @@ object NullSslSession extends SSLSession { def getLocalCertificates: Array[Certificate] = Array.empty def getLocalPrincipal: Principal = NullPrincipal def getPacketBufferSize: Int = 0 - def getPeerCertificateChain: Array[X509Certificate] = Array.empty + override def getPeerCertificateChain: Array[X509Certificate] = Array.empty def getPeerCertificates: Array[Certificate] = Array.empty def getPeerHost: String = "" def getPeerPort: Int = 0 From 021710628ab3541d07812b376864b57380db3532 Mon Sep 17 00:00:00 2001 From: Lily He Date: Tue, 3 May 2022 17:22:41 +0000 Subject: [PATCH 659/796] csl: Add fileOwner for CHANGELOG.rst Problem/Solution: Add the whole team as the file owner for the CHANGELOG.rst file for: finagle, finatra, scrooge, twitter-server, util Differential Revision: https://phabricator.twitter.biz/D886420 --- PROJECT.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/PROJECT.yml b/PROJECT.yml index ec21b4ac20..eff27342aa 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -8,3 +8,7 @@ owners: - yufang watchers: - csl-team:ldap + +fileOwners: + CHANGELOG.rst: + - csl-team:ldap From ce78eeb4eaed4408f1e0eff8de605a1927335395 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Fri, 6 May 2022 20:39:45 +0000 Subject: [PATCH 660/796] util-validator: move the java constraints and error message resources to a top level package under util Problem/Solution The util validator java packages and the error message resource file are needed as a dependencies of Scrooge (will come in a separate change). In order to not break sbt, we need to move the java package and the resource file into its top level directory. Differential Revision: https://phabricator.twitter.biz/D888534 --- build.sbt | 27 ++++++++++++++++++- .../test/scala/com/twitter/util/jackson/BUILD | 2 +- util-validator-constraints/PROJECT | 12 +++++++++ .../java/com/twitter/util/validation/BUILD | 0 .../util/validation/MethodValidation.java | 0 .../twitter/util/validation/constraints/BUILD | 0 .../validation/constraints/CountryCode.java | 0 .../util/validation/constraints/OneOf.java | 0 .../util/validation/constraints/UUID.java | 0 .../src/main/resources/BUILD | 6 +++++ .../resources/ValidationMessages.properties | 0 util-validator/src/main/resources/BUILD | 1 - .../scala/com/twitter/util/validation/BUILD | 11 ++++---- .../com/twitter/util/validation/engine/BUILD | 2 +- .../util/validation/internal/engine/BUILD | 2 +- .../internal/metadata/descriptor/BUILD | 2 +- .../util/validation/internal/validators/BUILD | 2 +- .../internal/validators/notempty/BUILD | 2 +- .../validation/internal/validators/size/BUILD | 2 +- 19 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 util-validator-constraints/PROJECT rename {util-validator => util-validator-constraints}/src/main/java/com/twitter/util/validation/BUILD (100%) rename {util-validator => util-validator-constraints}/src/main/java/com/twitter/util/validation/MethodValidation.java (100%) rename {util-validator => util-validator-constraints}/src/main/java/com/twitter/util/validation/constraints/BUILD (100%) rename {util-validator => util-validator-constraints}/src/main/java/com/twitter/util/validation/constraints/CountryCode.java (100%) rename {util-validator => util-validator-constraints}/src/main/java/com/twitter/util/validation/constraints/OneOf.java (100%) rename {util-validator => util-validator-constraints}/src/main/java/com/twitter/util/validation/constraints/UUID.java (100%) create mode 100644 util-validator-constraints/src/main/resources/BUILD rename {util-validator => util-validator-constraints}/src/main/resources/ValidationMessages.properties (100%) diff --git a/build.sbt b/build.sbt index c77ad996d1..a6d9927d29 100644 --- a/build.sbt +++ b/build.sbt @@ -216,6 +216,19 @@ val sharedScala3EnabledSettings = Seq(libraryDependencies ++= scala3Dependencies) ++ Seq(excludeDependencies ++= Seq("org.scala-lang.modules" % "scala-collection-compat_3")) +// settings for projects that are cross compiled with scala 2.10 +val settingsCrossCompiledWithTwoTen = + baseSettings ++ + Seq( + crossScalaVersions := Seq("2.10.7") ++ _crossScalaVersions, + scalaVersion := _scalaVersion, + javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint:unchecked"), + doc / javacOptions := Seq("-source", "1.8"), + libraryDependencies ++= Seq( + "org.scalacheck" %% "scalacheck" % "1.14.3" % "test" + ) + ) + lazy val noPublishSettings = Seq( publish / skip := true ) @@ -285,6 +298,7 @@ lazy val util = Project( utilThrift, utilTunable, utilValidator, + utilValidatorConstraint, utilZk, utilZkTest ) @@ -730,7 +744,18 @@ lazy val utilValidator = Project( "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", "org.slf4j" % "slf4j-simple" % slf4jVersion % "test" ) - ).dependsOn(utilCore, utilReflect, utilSlf4jApi) + ).dependsOn(utilCore, utilReflect, utilSlf4jApi, utilValidatorConstraint) + +lazy val utilValidatorConstraint = Project( + id = "util-validator-constraints", + base = file("util-validator-constraints") +).settings( + settingsCrossCompiledWithTwoTen +).settings( + libraryDependencies ++= Seq( + "jakarta.validation" % "jakarta.validation-api" % "3.0.0" + ) +) lazy val utilZk = Project( id = "util-zk", diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD index 8bc8aedc45..19697e3e3c 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD @@ -38,7 +38,7 @@ junit_tests( "util/util-jackson/src/test/resources", "util/util-mock/src/main/scala/com/twitter/util/mock", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation", "util/util-validator/src/main/scala/com/twitter/util/validation", ], ) diff --git a/util-validator-constraints/PROJECT b/util-validator-constraints/PROJECT new file mode 100644 index 0000000000..2541e2a188 --- /dev/null +++ b/util-validator-constraints/PROJECT @@ -0,0 +1,12 @@ +# See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path + +owners: + - banderson + - ibennett + - jyan + - mnakamura + - roanta + - vkostyukov + - yufang +watchers: + - csl-team@twitter.com \ No newline at end of file diff --git a/util-validator/src/main/java/com/twitter/util/validation/BUILD b/util-validator-constraints/src/main/java/com/twitter/util/validation/BUILD similarity index 100% rename from util-validator/src/main/java/com/twitter/util/validation/BUILD rename to util-validator-constraints/src/main/java/com/twitter/util/validation/BUILD diff --git a/util-validator/src/main/java/com/twitter/util/validation/MethodValidation.java b/util-validator-constraints/src/main/java/com/twitter/util/validation/MethodValidation.java similarity index 100% rename from util-validator/src/main/java/com/twitter/util/validation/MethodValidation.java rename to util-validator-constraints/src/main/java/com/twitter/util/validation/MethodValidation.java diff --git a/util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD b/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/BUILD similarity index 100% rename from util-validator/src/main/java/com/twitter/util/validation/constraints/BUILD rename to util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/BUILD diff --git a/util-validator/src/main/java/com/twitter/util/validation/constraints/CountryCode.java b/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/CountryCode.java similarity index 100% rename from util-validator/src/main/java/com/twitter/util/validation/constraints/CountryCode.java rename to util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/CountryCode.java diff --git a/util-validator/src/main/java/com/twitter/util/validation/constraints/OneOf.java b/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/OneOf.java similarity index 100% rename from util-validator/src/main/java/com/twitter/util/validation/constraints/OneOf.java rename to util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/OneOf.java diff --git a/util-validator/src/main/java/com/twitter/util/validation/constraints/UUID.java b/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/UUID.java similarity index 100% rename from util-validator/src/main/java/com/twitter/util/validation/constraints/UUID.java rename to util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/UUID.java diff --git a/util-validator-constraints/src/main/resources/BUILD b/util-validator-constraints/src/main/resources/BUILD new file mode 100644 index 0000000000..ad57fee95f --- /dev/null +++ b/util-validator-constraints/src/main/resources/BUILD @@ -0,0 +1,6 @@ +resources( + sources = [ + "**/*.properties", + ], + tags = ["bazel-compatible"], +) diff --git a/util-validator/src/main/resources/ValidationMessages.properties b/util-validator-constraints/src/main/resources/ValidationMessages.properties similarity index 100% rename from util-validator/src/main/resources/ValidationMessages.properties rename to util-validator-constraints/src/main/resources/ValidationMessages.properties diff --git a/util-validator/src/main/resources/BUILD b/util-validator/src/main/resources/BUILD index 5eb72c8354..a59b2ee358 100644 --- a/util-validator/src/main/resources/BUILD +++ b/util-validator/src/main/resources/BUILD @@ -1,6 +1,5 @@ resources( sources = [ - "**/*.properties", "**/*.xml", ], tags = ["bazel-compatible"], diff --git a/util-validator/src/main/scala/com/twitter/util/validation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/BUILD index f5233ce28c..6c6b8fa8ff 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/BUILD @@ -16,9 +16,10 @@ scala_library( "util/util-core:scala", "util/util-reflect/src/main/scala/com/twitter/util/reflect", "util/util-slf4j-api/src/main/scala", - "util/util-validator/src/main/java/com/twitter/util/validation", - "util/util-validator/src/main/java/com/twitter/util/validation/constraints", - "util/util-validator/src/main/resources", # loads custom constraint mappings and validation messages + "util/util-validator-constraints/src/main/java/com/twitter/util/validation", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints", + "util/util-validator-constraints/src/main/resources", # loads validation messages + "util/util-validator/src/main/resources", # loads custom constraint mappings "util/util-validator/src/main/scala/com/twitter/util/validation/cfg", "util/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation", "util/util-validator/src/main/scala/com/twitter/util/validation/conversions", @@ -38,8 +39,8 @@ scala_library( exports = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/json4s:json4s-core", - "util/util-validator/src/main/java/com/twitter/util/validation", - "util/util-validator/src/main/java/com/twitter/util/validation/constraints", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints", "util/util-validator/src/main/scala/com/twitter/util/validation/cfg", "util/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation", "util/util-validator/src/main/scala/com/twitter/util/validation/conversions", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD index 5b7dbf9347..22ed6fbfc3 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD @@ -13,7 +13,7 @@ scala_library( "3rdparty/jvm/org/hibernate/validator:hibernate-validator", "util/util-core:scala", "util/util-slf4j-api/src/main/scala", - "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation", "util/util-validator/src/main/scala/com/twitter/util/validation/internal/engine", "util/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor", ], diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD index 7095a3213b..1fd3148c58 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD @@ -12,7 +12,7 @@ scala_library( "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", "3rdparty/jvm/org/scala-lang/modules:scala-collection-compat", - "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation", "util/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor", "util/util-validator/src/main/scala/org/hibernate/validator/internal/engine", ], diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD index da48717d39..ec8430abd1 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD @@ -11,7 +11,7 @@ scala_library( dependencies = [ "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", - "util/util-validator/src/main/java/com/twitter/util/validation", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation", "util/util-validator/src/main/scala/org/hibernate/validator/internal/engine", ], ) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD index b4f340e2e6..988d90853f 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD @@ -13,7 +13,7 @@ scala_library( "3rdparty/jvm/org/hibernate/validator:hibernate-validator", "util/util-core:scala", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", - "util/util-validator/src/main/java/com/twitter/util/validation/constraints", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints", "util/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation", ], ) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD index e690dffe82..14a934a33a 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD @@ -12,6 +12,6 @@ scala_library( "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", "util/util-core:scala", - "util/util-validator/src/main/java/com/twitter/util/validation/constraints", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints", ], ) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD index 3dc3438a47..109f82ffa8 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD @@ -12,6 +12,6 @@ scala_library( "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/hibernate/validator:hibernate-validator", "util/util-core:scala", - "util/util-validator/src/main/java/com/twitter/util/validation/constraints", + "util/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints", ], ) From ca9c1c847f4524efe838cedd4050647e1a2173f1 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 17 May 2022 22:09:40 +0000 Subject: [PATCH 661/796] util-stats: More reshaping of dimensional metrics support Problem Recent plans have made scopes more valuable for us and less things will likely be changed from scopes to labels. This is great news as that means most scopes are still valid. However, we need to account for that in our model which doesn't allow scopes for dimensional metrics. Solution Simply allow it. We change the identity to be the union of the hierarchical and dimensional names plus the labels and whether the metric is valid dimensionally. JIRA Issues: CSL-11965 Differential Revision: https://phabricator.twitter.biz/D891342 --- .../finagle/stats/InMemoryStatsReceiver.scala | 10 +- .../twitter/finagle/stats/MetricBuilder.scala | 63 +++--- .../stats/NameTranslatingStatsReceiver.scala | 56 ++++- .../stats/ScopeTranslatingStatsReceiver.scala | 34 --- .../twitter/finagle/stats/StatsReceiver.scala | 86 +++---- .../stats/TranslatingStatsReceiver.scala | 33 +++ .../stats/InMemoryStatsReceiverTest.scala | 162 +------------- .../finagle/stats/StatsReceiverTest.scala | 210 ++++-------------- 8 files changed, 206 insertions(+), 448 deletions(-) delete mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/ScopeTranslatingStatsReceiver.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index b827566ae2..ef68bb7544 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -88,10 +88,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { .metricBuilder(CounterType) .withName(name: _*) - counter( - if (name.length > 1) metricBuilder.withHierarchicalOnly - else metricBuilder.withDimensionalSupport - ) + counter(metricBuilder) } /** @@ -130,10 +127,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { .metricBuilder(HistogramType) .withName(name: _*) - stat( - if (name.length > 1) metricBuilder.withHierarchicalOnly - else metricBuilder.withDimensionalSupport - ) + stat(metricBuilder) } /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 2ca5c617ef..28a8a6671e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -44,6 +44,8 @@ object metadataScopeSeparator { object MetricBuilder { + val DimensionalNameScopeSeparator: String = "_" + /** * Construct a MethodBuilder. * @@ -83,9 +85,9 @@ object MetricBuilder { role, verbosity, sourceClass, - // For now we're not allowing construction with labels. + // For now we're not allowing construction w"ith labels. // They need to be added later via `.withLabels`. - Identity.Hierarchical(name, Map.empty), + Identity(name, name, Map.empty, false), relativeName, processPath, percentiles, @@ -142,33 +144,20 @@ object MetricBuilder { def toPrometheusString: String = "counter" } - /** Identity information about this metric */ - sealed abstract class Identity private () { - def hierarchicalName: Seq[String] - def labels: Map[String, String] - } - /** * Experimental way to represent both hierarchical and dimensional identity information. * * @note this type is expected to undergo some churn as dimensional metric support matures. */ - object Identity { - - /** Hierarchical only [[Identity]] representation */ - final case class Hierarchical(hierarchicalName: Seq[String], labels: Map[String, String]) - extends Identity { - override def toString: String = { - hierarchicalName.mkString("Hierarchical(", metadataScopeSeparator(), s", $labels)") - } - } - - /** Full [[Identity]] representation including a [[Hierarchical]] name and dimensional labels */ - final case class Full(hierarchicalName: Seq[String], labels: Map[String, String]) - extends Identity { - override def toString: String = { - hierarchicalName.mkString("Full(", metadataScopeSeparator(), s", $labels)") - } + final case class Identity private[stats] ( + hierarchicalName: Seq[String], + dimensionalName: Seq[String], + labels: Map[String, String], + hierarchicalOnly: Boolean) { + override def toString: String = { + val hname = hierarchicalName.mkString(metadataScopeSeparator()) + val dname = dimensionalName.mkString(DimensionalNameScopeSeparator) + s"Identity($hname, $dname, $labels, $hierarchicalOnly)" } } } @@ -279,31 +268,29 @@ class MetricBuilder private ( @varargs def withName(name: String*): MetricBuilder = { - val nextIdentity = this.identity match { - case Identity.Hierarchical(_, labels) => Identity.Hierarchical(name, labels) - case Identity.Full(_, labels) => Identity.Full(name, labels) - } - copy(identity = nextIdentity) + copy(identity = identity.copy(hierarchicalName = name, dimensionalName = name)) } private[finagle] def withIdentity(identity: Identity): MetricBuilder = copy(identity = identity) + /** + * The hierarchical name of the metric + */ def name: Seq[String] = identity.hierarchicalName // Private for now - private[twitter] def withLabels(labels: Map[String, String]): MetricBuilder = identity match { - case Identity.Hierarchical(name, _) => copy(identity = Identity.Hierarchical(name, labels)) - case Identity.Full(name, _) => copy(identity = Identity.Full(name, labels)) + private[twitter] def withLabels(labels: Map[String, String]): MetricBuilder = { + copy(identity = identity.copy(labels = labels)) } - private[twitter] def withHierarchicalOnly: MetricBuilder = identity match { - case Identity.Hierarchical(_, _) => this - case Identity.Full(name, labels) => this.copy(identity = Identity.Hierarchical(name, labels)) + private[twitter] def withHierarchicalOnly: MetricBuilder = { + if (identity.hierarchicalOnly) this + else copy(identity = identity.copy(hierarchicalOnly = true)) } - private[twitter] def withDimensionalSupport: MetricBuilder = identity match { - case Identity.Hierarchical(name, labels) => this.copy(identity = Identity.Full(name, labels)) - case Identity.Full(_, _) => this + private[twitter] def withDimensionalSupport: MetricBuilder = { + if (identity.hierarchicalOnly) copy(identity = identity.copy(hierarchicalOnly = false)) + else this } @varargs diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index 6264e02cb5..d98f03680a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -6,21 +6,69 @@ package com.twitter.finagle.stats * * @param self The underlying StatsReceiver to which translated names are passed * @param namespacePrefix the namespace used for translations + * @param mode determines which names to translate: hierarchical, dimensional, or both. */ abstract class NameTranslatingStatsReceiver( self: StatsReceiver, - namespacePrefix: String) + namespacePrefix: String, + mode: NameTranslatingStatsReceiver.Mode) extends TranslatingStatsReceiver(self) { + import NameTranslatingStatsReceiver._ + + def this(self: StatsReceiver, namespacePrefix: String) = + this(self, namespacePrefix, NameTranslatingStatsReceiver.FullTranslation) + def this(self: StatsReceiver) = this(self, "") protected def translate(name: Seq[String]): Seq[String] final protected def translate(builder: MetricBuilder): MetricBuilder = { - // We need to make sure we've marked the dimensional metrics as 'poisoned' - // since we didn't give it a label. - builder.withName(translate(builder.name): _*).withHierarchicalOnly + val nextIdentity = mode match { + case FullTranslation => + builder.identity.copy( + hierarchicalName = translate(builder.identity.hierarchicalName), + dimensionalName = translate(builder.identity.dimensionalName)) + + case HierarchicalOnly => + builder.identity.copy(hierarchicalName = translate(builder.identity.hierarchicalName)) + + case DimensionalOnly => + builder.identity.copy(dimensionalName = translate(builder.identity.dimensionalName)) + } + + builder.withIdentity(nextIdentity) } override def toString: String = s"$self/$namespacePrefix" } + +private object NameTranslatingStatsReceiver { + + /** The mode with which to translate stats, either hierarchical, dimensional, or both. */ + sealed trait Mode + + /** Translate metric names for both hierarchical and dimensional representations */ + case object FullTranslation extends Mode + + /** Translate metric names only for hierarchical representations */ + case object HierarchicalOnly extends Mode + + /** Translate metric names only for dimensional representations */ + case object DimensionalOnly extends Mode + + /** + * Translate the scope of a [[StatsReceiver]] + * + * @param parent the [[StatsReceiver]] to delegate the translated name to. + * @param namespace the segment to prepend to the name using the appropriate scope delimiter. + * @param mode select which representations to translate. + * */ + final class ScopeTranslatingStatsReceiver( + parent: StatsReceiver, + namespace: String, + mode: NameTranslatingStatsReceiver.Mode) + extends NameTranslatingStatsReceiver(parent, namespace, mode) { + protected def translate(name: Seq[String]): Seq[String] = namespace +: name + } +} diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/ScopeTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/ScopeTranslatingStatsReceiver.scala deleted file mode 100644 index dfc7963fab..0000000000 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/ScopeTranslatingStatsReceiver.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.twitter.finagle.stats - -import com.twitter.finagle.stats.MetricBuilder.Identity - -/** - * A [[TranslatingStatsReceiver]] for working with both dimensional and hierarchical metrics. - * - * Translates the [[MetricBuilder]] to prepend the label value as a scope in addition to adding - * it to the labels map. - */ -private class ScopeTranslatingStatsReceiver( - sr: StatsReceiver, - labelName: String, - labelValue: String) - extends TranslatingStatsReceiver(sr) { - - require(labelName.nonEmpty) - require(labelValue.nonEmpty) - - private[this] val labelPair = labelName -> labelValue - - protected def translate(builder: MetricBuilder): MetricBuilder = - builder.withIdentity(newIdentity(builder.identity)) - - private[this] def newIdentity(identity: Identity): Identity = identity match { - case Identity.Full(name, labels) => - Identity.Full(labelValue +: name, labels + labelPair) - case Identity.Hierarchical(name, labels) => - Identity.Hierarchical(labelValue +: name, labels + labelPair) - } - - // We preserve this because unfortunately it is sometimes parsed - override def toString: String = s"$self/$labelValue" -} diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 7e791a84bf..9c3dafc968 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -6,6 +6,8 @@ import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.MetricBuilder.MetricType import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter +import com.twitter.finagle.stats.NameTranslatingStatsReceiver.ScopeTranslatingStatsReceiver +import com.twitter.finagle.stats.TranslatingStatsReceiver.LabelTranslatingStatsReceiver import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.util.Return import com.twitter.util.Try @@ -142,10 +144,7 @@ trait StatsReceiver { .withVerbosity(verbosity) .withName(name: _*) - counter( - if (name.length > 1) builder.withHierarchicalOnly - else builder.withDimensionalSupport - ) + counter(builder) } /** @@ -159,10 +158,7 @@ trait StatsReceiver { .withName(name: _*) .withDescription(description) - counter( - if (name.length > 1) builder.withHierarchicalOnly - else builder.withDimensionalSupport - ) + counter(builder) } /** @@ -193,10 +189,7 @@ trait StatsReceiver { .withVerbosity(verbosity) .withName(name: _*) - stat( - if (name.length > 1) builder.withHierarchicalOnly - else builder.withDimensionalSupport - ) + stat(builder) } /** @@ -210,10 +203,7 @@ trait StatsReceiver { .withName(name: _*) .withDescription(description) - stat( - if (name.length > 1) builder.withHierarchicalOnly - else builder.withDimensionalSupport - ) + stat(builder) } /** @@ -300,10 +290,7 @@ trait StatsReceiver { .withVerbosity(verbosity) .withName(name: _*) - addGauge( - if (name.length > 1) builder.withHierarchicalOnly - else builder.withDimensionalSupport - )(f) + addGauge(builder)(f) } /** @@ -322,10 +309,7 @@ trait StatsReceiver { .withName(name: _*) .withDescription(description) - addGauge( - if (name.length > 1) builder.withHierarchicalOnly - else builder.withDimensionalSupport - )(f) + addGauge(builder)(f) } /** @@ -397,27 +381,47 @@ trait StatsReceiver { */ def scope(namespace: String): StatsReceiver = { if (namespace == "") this - else { - new NameTranslatingStatsReceiver(this, namespace) { - protected def translate(name: Seq[String]): Seq[String] = namespace +: name - } - } + else + new ScopeTranslatingStatsReceiver( + this, + namespace, + NameTranslatingStatsReceiver.FullTranslation) } /** - * Add a label and prepend the label value to the names of the returned [[StatsReceiver]]. - * - * For example: - * - * {{{ - * statsReceiver.scopeAndLabel("client_name", "backend1").counter("requests") - * }}} - * - * will generate a [[Counter]] with both hierarchical name `backend1/requests` and dimensional name - * `requests {client_name="backend1"}` + * Create a new `StatsReceiver` that will add a scope that is only used when the metric is + * emitted in hierarchical form. + */ + private[finagle] def hierarchicalScope(namespace: String): StatsReceiver = { + if (namespace == "") this + else + new ScopeTranslatingStatsReceiver( + this, + namespace, + NameTranslatingStatsReceiver.HierarchicalOnly) + } + + /** + * Create a new `StatsReceiver` that will add a scope that is only used when the metric is + * emitted in dimensional form. */ - def scopeAndLabel(labelName: String, labelValue: String): StatsReceiver = - new ScopeTranslatingStatsReceiver(this, labelName, labelValue) + private[finagle] def dimensionalScope(namespace: String): StatsReceiver = { + if (namespace == "") this + else + new ScopeTranslatingStatsReceiver( + this, + namespace, + NameTranslatingStatsReceiver.DimensionalOnly) + } + + /** + * Create a new `StatsReceiver` that will add the specified label to all created metrics. + */ + private[finagle] def label(labelName: String, labelValue: String): StatsReceiver = { + require(labelName.nonEmpty) + if (labelValue == "") this + else new LabelTranslatingStatsReceiver(this, labelName, labelValue) + } /** * Prepend `namespace` and `namespaces` to the names of the returned [[StatsReceiver]]. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala index d1957718f8..8bfe82039d 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala @@ -1,5 +1,7 @@ package com.twitter.finagle.stats +import com.twitter.finagle.stats.MetricBuilder.Identity + /** * A StatsReceiver receiver proxy that translates all counter, stat, and gauge * names according to a `translate` function. @@ -24,3 +26,34 @@ abstract class TranslatingStatsReceiver( override def toString: String = s"Translating($self)" } + +private object TranslatingStatsReceiver { + + /** + * A [[TranslatingStatsReceiver]] for working with both dimensional and hierarchical metrics. + * + * Translates the [[MetricBuilder]] to prepend the label value as a scope in addition to adding + * it to the labels map. + */ + final class LabelTranslatingStatsReceiver( + sr: StatsReceiver, + labelName: String, + labelValue: String) + extends TranslatingStatsReceiver(sr) { + + require(labelName.nonEmpty) + require(labelValue.nonEmpty) + + private[this] val labelPair = labelName -> labelValue + + protected def translate(builder: MetricBuilder): MetricBuilder = + builder.withIdentity(newIdentity(builder.identity)) + + private[this] def newIdentity(identity: Identity): Identity = { + identity.copy(labels = identity.labels + labelPair) + } + + // We preserve this because unfortunately it is sometimes parsed + override def toString: String = s"$self/$labelValue" + } +} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 52f5afd86d..7bf538a582 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -3,7 +3,6 @@ package com.twitter.finagle.stats import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType -import com.twitter.finagle.stats.MetricBuilder.Identity import com.twitter.finagle.stats.exp.Expression import com.twitter.finagle.stats.exp.ExpressionSchema import com.twitter.finagle.stats.exp.ExpressionSchemaKey @@ -362,13 +361,13 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(parts.length == 3) assert( parts( - 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Full(coolGauge, Map()), List(), None, Vector(), GaugeType, InMemoryStatsReceiver)") + 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(coolGauge, coolGauge, Map(), false), List(), None, Vector(), GaugeType, InMemoryStatsReceiver)") assert( parts( - 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Hierarchical(rad/histo, Map()), List(), None, Vector(), HistogramType, InMemoryStatsReceiver/rad)") + 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(rad/histo, rad_histo, Map(), false), List(), None, Vector(), HistogramType, InMemoryStatsReceiver/rad)") assert( parts( - 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Hierarchical(sweet/counter, Map()), List(), None, Vector(), CounterType, InMemoryStatsReceiver)") + 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(sweet/counter, sweet_counter, Map(), false), List(), None, Vector(), CounterType, InMemoryStatsReceiver)") } finally { ps.close() } @@ -389,20 +388,11 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra // what we expected as hydrated metric builders val aaSchema = - MetricBuilder( - name = Seq("test", "a"), - metricType = CounterType, - statsReceiver = sr).withHierarchicalOnly + MetricBuilder(name = Seq("test", "a"), metricType = CounterType, statsReceiver = sr) val bbSchema = - MetricBuilder( - name = Seq("test", "b"), - metricType = HistogramType, - statsReceiver = sr).withHierarchicalOnly + MetricBuilder(name = Seq("test", "b"), metricType = HistogramType, statsReceiver = sr) val ccSchema = - MetricBuilder( - name = Seq("test", "c"), - metricType = GaugeType, - statsReceiver = sr).withHierarchicalOnly + MetricBuilder(name = Seq("test", "c"), metricType = GaugeType, statsReceiver = sr) val expected_expression = ExpressionSchema( "test_expression", @@ -437,144 +427,4 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(sr.getAllExpressionsWithLabel("nonexistent key", "nonexistent value").isEmpty) } - - private class SupportsDimensionalStatsReceiver extends InMemoryStatsReceiver { - var supportsDimensional: java.lang.Boolean = null - - private def hasDimensions(mb: MetricBuilder): Boolean = mb.identity match { - case Identity.Hierarchical(_, _) => false - case Identity.Full(_, _) => true - } - - def reset(): Unit = { - supportsDimensional = null - } - - override def counter(metricBuilder: MetricBuilder): ReadableCounter = { - val c = super.counter(metricBuilder) - supportsDimensional = hasDimensions(c.metadata.toMetricBuilder.get) - c - } - - override def stat(metricBuilder: MetricBuilder): ReadableStat = { - val s = super.stat(metricBuilder) - supportsDimensional = hasDimensions(s.metadata.toMetricBuilder.get) - s - } - - override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { - val g = super.addGauge(metricBuilder)(f) - supportsDimensional = hasDimensions(g.metadata.toMetricBuilder.get) - g - } - } - - test("StatsReceiver.scope.counter: dimensional metrics are disabled without a label") { - val receiver = new SupportsDimensionalStatsReceiver - val scoped = receiver.scope("foo") - - scoped.counter("bar") - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.counter: varargs metrics names longer than 1 disable dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.counter("foo", "bar") - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.counter(Some("description"), "foo", "bar") - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.counter("description", Verbosity.Default, "foo", "bar") - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.counter: varargs metrics names of 1 allow dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.counter("bar") - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.counter(Some("description"), "bar") - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.counter("description", Verbosity.Default, "bar") - assert(receiver.supportsDimensional) - } - - test("StatsReceiver.scope.addGauge: dimensional metrics are disabled without a label") { - val receiver = new SupportsDimensionalStatsReceiver - val scoped = receiver.scope("foo") - - scoped.addGauge("bar") { 1.0f } - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.addGauge: varargs metrics names longer than 1 disable dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.addGauge("foo", "bar") { 1.0f } - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.addGauge(Some("description"), "foo", "bar") { 1.0f } - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.addGauge("description", Verbosity.Default, "foo", "bar") { 1.0f } - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.addGauge: varargs metrics names of 1 allow dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.addGauge("bar") { 1.0f } - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.addGauge(Some("description"), "bar") { 1.0f } - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.addGauge("description", Verbosity.Default, "bar") { 1.0f } - assert(receiver.supportsDimensional) - } - - test("StatsReceiver.scope.stat: dimensional metrics are disabled without a label") { - val receiver = new SupportsDimensionalStatsReceiver - val scoped = receiver.scope("foo") - - scoped.stat("bar") - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.stat: varargs metrics names longer than 1 disable dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.stat("foo", "bar") - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.stat(Some("description"), "foo", "bar") - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.stat("description", Verbosity.Default, "foo", "bar") - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.stat: varargs metrics names of 1 allow dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.stat("bar") - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.stat(Some("description"), "bar") - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.stat("description", Verbosity.Default, "bar") - assert(receiver.supportsDimensional) - } - } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index f40dccdc64..2d0030ba9a 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -5,7 +5,6 @@ import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType -import com.twitter.finagle.stats.MetricBuilder.Identity import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter import com.twitter.util.Await import com.twitter.util.Future @@ -99,186 +98,63 @@ class StatsReceiverTest extends AnyFunSuite { verify(receiver, times(3)).stat("2", "chainz") } - private class SupportsDimensionalStatsReceiver extends StatsReceiver { - var supportsDimensional: java.lang.Boolean = null - - private def hasDimensions(mb: MetricBuilder): Boolean = mb.identity match { - case Identity.Hierarchical(_, _) => false - case Identity.Full(_, _) => true - } - - def reset(): Unit = { - supportsDimensional = null - } - - def repr: AnyRef = this - - def counter(metricBuilder: MetricBuilder): Counter = { - supportsDimensional = hasDimensions(metricBuilder) - new Counter { - def incr(delta: Long): Unit = () - def metadata: Metadata = metricBuilder - } - } - - def stat(metricBuilder: MetricBuilder): Stat = { - supportsDimensional = hasDimensions(metricBuilder) - new Stat { - def add(value: Float): Unit = () - def metadata: Metadata = metricBuilder - } - } - - def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { - supportsDimensional = hasDimensions(metricBuilder) - new Gauge { - def remove(): Unit = () - def metadata: Metadata = metricBuilder - } - } - } - - test("StatsReceiver.scope.counter: dimensional metrics are disabled without a label") { - val receiver = new SupportsDimensionalStatsReceiver - val scoped = receiver.scope("foo") - - scoped.counter("bar") - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.scopeAndLabel: a non-empty label name and label value are required") { + test("StatsReceiver.label: a non-empty label name is required") { val sr = new InMemoryStatsReceiver intercept[IllegalArgumentException] { - sr.scopeAndLabel("", "cool") - } - - intercept[IllegalArgumentException] { - sr.scopeAndLabel("cool", "") + sr.label("", "cool") } } - test("StatsReceiver.scopeAndLabel.counter: dimensions are supported") { - val receiver = new SupportsDimensionalStatsReceiver - val scoped = receiver.scopeAndLabel("scope", "foo") - - scoped.counter("bar") - assert(receiver.supportsDimensional) - } - - test("StatsReceiver.scopeAndLabel.scope.counter: dimensional support isn't magically added") { - val receiver = new SupportsDimensionalStatsReceiver - val scoped = receiver.scopeAndLabel("scope", "foo").scope("no_dimensions") - - scoped.counter("bar") - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.scopeAndLabel: adds both a scope and the labels") { + test("StatsReceiver.label: empty label values are effectively no-ops") { val sr = new InMemoryStatsReceiver - val c = sr.scopeAndLabel("scope", "good").counter("stuff") - val metricBuilder = c.metadata.toMetricBuilder.get - assert(metricBuilder.name == Seq("good", "stuff")) - assert(metricBuilder.identity.labels == Map("scope" -> "good")) + sr.label("foo", "").counter("bar").incr() + assert(sr.counters(Seq("bar")) == 1) + assert(sr.schemas(Seq("bar")).identity.labels.isEmpty) } - test("StatsReceiver.counter: varargs metrics names longer than 1 disable dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.counter("foo", "bar") - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.counter(Some("description"), "foo", "bar") - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.counter("description", Verbosity.Default, "foo", "bar") - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.counter: varargs metrics names of 1 allow dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.counter("bar") - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.counter(Some("description"), "bar") - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.counter("description", Verbosity.Default, "bar") - assert(receiver.supportsDimensional) - } - - test("StatsReceiver.scope.addGauge: dimensional metrics are disabled without a label") { - val receiver = new SupportsDimensionalStatsReceiver - val scoped = receiver.scope("foo") - - scoped.addGauge("bar") { 1.0f } - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.addGauge: varargs metrics names longer than 1 disable dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.addGauge("foo", "bar") { 1.0f } - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.addGauge(Some("description"), "foo", "bar") { 1.0f } - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.addGauge("description", Verbosity.Default, "foo", "bar") { 1.0f } - assert(!receiver.supportsDimensional) - } - - test("StatsReceiver.addGauge: varargs metrics names of 1 allow dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.addGauge("bar") { 1.0f } - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.addGauge(Some("description"), "bar") { 1.0f } - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.addGauge("description", Verbosity.Default, "bar") { 1.0f } - assert(receiver.supportsDimensional) - } + test("StatsReceiver.scope: adds a scope to both names and doesn't poison dimensional metrics") { + val sr = new InMemoryStatsReceiver + val scopedCounter = sr.scope("cool").counter("numbers") - test("StatsReceiver.scope.stat: dimensional metrics are disabled without a label") { - val receiver = new SupportsDimensionalStatsReceiver - val scoped = receiver.scope("foo") + val mb = scopedCounter.metadata.toMetricBuilder.get - scoped.stat("bar") - assert(!receiver.supportsDimensional) + assert(mb.identity.labels.isEmpty) + assert(mb.identity.hierarchicalName == Seq("cool", "numbers")) + assert(mb.identity.dimensionalName == Seq("cool", "numbers")) + assert(!mb.identity.hierarchicalOnly) } - test("StatsReceiver.stat: varargs metrics names longer than 1 disable dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.stat("foo", "bar") - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.stat(Some("description"), "foo", "bar") - assert(!receiver.supportsDimensional) - receiver.reset() - - receiver.stat("description", Verbosity.Default, "foo", "bar") - assert(!receiver.supportsDimensional) + test("StatsReceiver.hierarchicalScope: composes with label correctly") { + val sr = new InMemoryStatsReceiver + val scopedCounter = sr + .hierarchicalScope("clnt") + .hierarchicalScope("backend") + .label("clnt", "backend") + .counter("requests") + + val mb = scopedCounter.metadata.toMetricBuilder.get + + assert(mb.identity.hierarchicalName == Seq("clnt", "backend", "requests")) + assert(mb.identity.dimensionalName == Seq("requests")) + assert(mb.identity.labels == Map("clnt" -> "backend")) + assert(!mb.identity.hierarchicalOnly) } - test("StatsReceiver.stat: varargs metrics names of 1 allow dimensional metrics") { - val receiver = new SupportsDimensionalStatsReceiver - receiver.stat("bar") - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.stat(Some("description"), "bar") - assert(receiver.supportsDimensional) - receiver.reset() - - receiver.stat("description", Verbosity.Default, "bar") - assert(receiver.supportsDimensional) + test("StatsReceiver.dimensionalScope: composes with label correctly") { + val sr = new InMemoryStatsReceiver + val scopedCounter = sr + .dimensionalScope("foo") + .dimensionalScope("baz") + .hierarchicalScope("bar") + .label("clnt", "backend") + .counter("requests") + + val mb = scopedCounter.metadata.toMetricBuilder.get + + assert(mb.identity.hierarchicalName == Seq("bar", "requests")) + assert(mb.identity.dimensionalName == Seq("foo", "baz", "requests")) + assert(mb.identity.labels == Map("clnt" -> "backend")) + assert(!mb.identity.hierarchicalOnly) } test("StatsReceiver.scope: prefix stats by a scope string") { From eaa8670e607e9f293ca14b2617e5e97d824fdb73 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Wed, 18 May 2022 03:48:27 +0000 Subject: [PATCH 662/796] util-stats: Provide defaults for Expression APIs Problem/Solution Add `validateMetric` param to `Expression.counter`, `Expression.stat`, and `Expression.stats` APIs, when true, only create expressions if the metric is registered with the `StatsReceiver`. Problem/Solution Provide defaults for `showRollup` and `histogramComponent`. JIRA Issues: CSL-11937, CSL-11938 Differential Revision: https://phabricator.twitter.biz/D895183 --- .../finagle/stats/exp/Expression.scala | 67 ++++++++++++------- .../stats/exp/ExpressionSchemaTest.scala | 8 +-- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index a9a7927b4c..adda7291a8 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -78,40 +78,52 @@ private[twitter] object Expression { * Create an expression from creating a [[com.twitter.finagle.stats.Counter]] * with a given [[StatsReceiver]]. * - * @param statsReceiver The [[StatsReceiver]] used by the caller to record metric. - * @param showRollup When true, create an expression with all tail metrics. - * @param name The name to create the [[com.twitter.finagle.stats.Counter]]. - * @return An optional [[Expression]] carrying the counter metadata. - * Return `None` if the metric of name {@code name} is not found - * in the {@code statsReceiver}. + * @param statsReceiver The [[StatsReceiver]] used by the caller to record metric. + * @param name The name to create the [[com.twitter.finagle.stats.Counter]]. + * @param showRollup When true, create an expression with all tail metrics. The + * default is false. + * @param validateMetric When true, only create an expression if the counter with name + * {@code name} has been registered with the given + * {@code statsReceiver}, otherwise return `None`. When false, + * always return an expression. The default value is false. + * @return An optional [[Expression]] carrying the counter metadata. + * Return `None` if the metric of name {@code name} is not found + * in the {@code statsReceiver}. */ - @varargs def counter( statsReceiver: StatsReceiver, - showRollup: Boolean, - name: String* + name: Seq[String], + showRollup: Boolean = false, + validateMetric: Boolean = false ): Option[Expression] = - Some(this(statsReceiver.counter(name: _*).metadata, showRollup)) + if (validateMetric) None // return None for now, will add the validation in another change + else Some(this(statsReceiver.counter(name: _*).metadata, showRollup)) /** * Create an expression from creating a [[com.twitter.finagle.stats.Stat]] with * a given [[StatsReceiver]]. * * @param statsReceiver The [[StatsReceiver]] used by the caller to record metric. + * @param name The name to create the [[com.twitter.finagle.stats.Stat]]. * @param histogramComponent A [[HistogramComponent]] used to create a histogram * expression. - * @param name The name to create the [[com.twitter.finagle.stats.Stat]]. + * @param validateMetric When true, only create an expression if the counter with + * name {@code name} has been registered with the given + * {@code statsReceiver}, otherwise return `None`. When + * false, always return an expression. The default value is + * false. * @return An optional [[Expression]] carrying the stat metadata. * Return `None` if the metric of name {@code name} is not * found in the {@code statsReceiver}. */ - @varargs def stat( statsReceiver: StatsReceiver, + name: Seq[String], histogramComponent: HistogramComponent, - name: String* + validateMetric: Boolean = false ): Option[Expression] = - Some(this(statsReceiver.stat(name: _*).metadata, histogramComponent)) + if (validateMetric) None // return None for now, will add the validation in another change + else Some(this(statsReceiver.stat(name: _*).metadata, histogramComponent)) /** * Create a list of expressions from creating a [[com.twitter.finagle.stats.Stat]] @@ -119,25 +131,34 @@ private[twitter] object Expression { * * @param statsReceiver The [[StatsReceiver]] used by the caller to record * metric. - * @param histogramComponents A list of [[HistogramComponent]] used to create a - * histogram expression. * @param name The name to create the * [[com.twitter.finagle.stats.Stat]]. + * @param histogramComponents A list of [[HistogramComponent]] used to create a + * histogram expression. The default is + * [[DefaultExpression]]. + * @param validateMetric When true, only create an expression if the counter with + * name {@code name} has been registered with the given + * {@code statsReceiver}, otherwise return `None`. When + * false, always return an expression. The default value is + * false. * @return A sequence of [[Expression]] carrying the stat metadata. * The order of the returned sequence is in correspondence * with the oder of the {@code histogramComponents}. Return * an empty sequence if the metric of name {@code name} is * not found in the {@code statsReceiver}. */ - @varargs def stats( statsReceiver: StatsReceiver, - histogramComponents: Seq[HistogramComponent], - name: String* - ): Seq[Expression] = { - val statMetadata = statsReceiver.stat(name: _*).metadata - histogramComponents.map(this(statMetadata, _)) - } + name: Seq[String], + histogramComponents: Seq[HistogramComponent] = HistogramComponent.DefaultPercentiles, + validateMetric: Boolean = false + ): Seq[Expression] = + if (validateMetric) { + Seq.empty // return an empty list for now, will add the validation in another change + } else { + val statMetadata = statsReceiver.stat(name: _*).metadata + histogramComponents.map(this(statMetadata, _)) + } } /** diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 4ceec0eccb..423c5507a2 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -200,9 +200,9 @@ class ExpressionSchemaTest extends AnyFunSuite { "success_rate", Expression(100).multiply( Expression - .counter(sr, showRollup = false, "clnt", "aclient", "success").get.divide(Expression - .counter(sr, showRollup = false, "clnt", "aclient", "success").get.plus( - Expression.counter(sr, showRollup = true, "clnt", "aclient", "failures").get))) + .counter(sr, Seq("clnt", "aclient", "success")).get.divide(Expression + .counter(sr, Seq("clnt", "aclient", "success")).get.plus( + Expression.counter(sr, Seq("clnt", "aclient", "failures"), showRollup = true).get))) ).withRole(Client).withServiceName("aclient").build() assert(sr.expressions.values.size == 1) @@ -219,7 +219,7 @@ class ExpressionSchemaTest extends AnyFunSuite { val sr = new InMemoryStatsReceiver // create a list histogram ExpressionSchema with default percentiles val expressions: Seq[Expression] = - Expression.stats(sr, HistogramComponent.DefaultPercentiles, "clnt", "aclient", "latencies") + Expression.stats(sr, Seq("clnt", "aclient", "latencies")) expressions.map { expression => ExpressionSchema("latencies", expression).withRole(Client).withServiceName("aclient").build() } From e83544b2b20bf4d1fc2a92274116b7c39831ba62 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 24 May 2022 00:11:20 +0000 Subject: [PATCH 663/796] util: Remove the StatsReceiver field from MetricBuilder Problem MetricBuilder is a relatively nice representation of a metric definition with one exception: the statsReceiver field, which defines some behavior and isn't always possible to populate. This is primarily used for registering expressions but that is kind of fraught with danger: which `statsReceiver` do you use when registering an expression composed of multiple metrics? Solution Remove that field and instead use explicit registration of expressions with a StatsReceiver. This pattern is still not perfect but perhaps a step in the right direction and we can try to sort out how to get rid of this registration pain later. JIRA Issues: CSL-11900 Differential Revision: https://phabricator.twitter.biz/D871577 --- .../main/scala/com/twitter/jvm/JvmStats.scala | 81 +++++++------ .../stats/BroadcastStatsReceiver.scala | 4 +- .../finagle/stats/InMemoryStatsReceiver.scala | 2 +- .../twitter/finagle/stats/MetricBuilder.scala | 16 +-- .../twitter/finagle/stats/StatsReceiver.scala | 10 +- .../finagle/stats/StatsReceiverProxy.scala | 2 +- .../finagle/stats/exp/Expression.scala | 13 -- .../finagle/stats/exp/ExpressionSchema.scala | 17 +-- .../stats/InMemoryStatsReceiverTest.scala | 54 +++++---- .../finagle/stats/MetricBuilderTest.scala | 6 +- .../finagle/stats/StatsReceiverTest.scala | 10 +- .../stats/exp/ExpressionSchemaTest.scala | 111 +++++++++--------- 12 files changed, 150 insertions(+), 176 deletions(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 84ebafd291..a62bf1982a 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -59,11 +59,11 @@ object JvmStats { gauges.add(fileDescriptorCountGauge) gauges.add(stats.addGauge("fd_limit") { unix.getMaxFileDescriptorCount.toFloat }) // register expression - ExpressionSchema("file_descriptors", Expression(fileDescriptorCountGauge.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withDescription( - "Total file descriptors used by the service. If it continuously increasing over time, then potentially files or connections aren't being closed") - .build() + stats.registerExpression( + ExpressionSchema("file_descriptors", Expression(fileDescriptorCountGauge.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withDescription( + "Total file descriptors used by the service. If it continuously increasing over time, then potentially files or connections aren't being closed")) case _ => } @@ -119,13 +119,13 @@ object JvmStats { gauges.add(currentMem.addGauge(name, "max") { usage.getMax.toFloat }) // register memory usage expression - ExpressionSchema("memory_pool", Expression(usageGauge.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withLabel("kind", name) - .withUnit(Bytes) - .withDescription( - s"The current estimate of the amount of space within the $name memory pool holding allocated objects in bytes") - .build() + currentMem.registerExpression( + ExpressionSchema("memory_pool", Expression(usageGauge.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withLabel("kind", name) + .withUnit(Bytes) + .withDescription( + s"The current estimate of the amount of space within the $name memory pool holding allocated objects in bytes")) } } gauges.add(postGCStats.addGauge("used") { @@ -186,19 +186,18 @@ object JvmStats { gc.getCollectionTime.toFloat } - ExpressionSchema(s"gc_cycles", Expression(poolCycles.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withLabel("gc_pool", name) - .withDescription( - s"The total number of collections that have occurred for the $name gc pool") - .build() - ExpressionSchema(s"gc_latency", Expression(poolMsec.metadata)) + gcStats.registerExpression( + ExpressionSchema(s"gc_cycles", Expression(poolCycles.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withLabel("gc_pool", name) + .withDescription( + s"The total number of collections that have occurred for the $name gc pool")) + gcStats.registerExpression(ExpressionSchema(s"gc_latency", Expression(poolMsec.metadata)) .withLabel(ExpressionSchema.Role, "jvm") .withLabel("gc_pool", name) .withUnit(Milliseconds) .withDescription( - s"The total elapsed time spent doing collections for the $name gc pool in milliseconds") - .build() + s"The total elapsed time spent doing collections for the $name gc pool in milliseconds")) gauges.add(poolCycles) gauges.add(poolMsec) @@ -229,25 +228,25 @@ object JvmStats { gauges.add(stats.addGauge("tenuring_threshold") { jvm.tenuringThreshold.toFloat }) // register metric expressions - ExpressionSchema("jvm_uptime", Expression(uptime.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withUnit(Milliseconds) - .withDescription("The uptime of the JVM in milliseconds") - .build() - ExpressionSchema("gc_cycles", Expression(cycles.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withDescription("The total number of collections that have occurred") - .build() - ExpressionSchema("gc_latency", Expression(msec.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withUnit(Milliseconds) - .withDescription("The total elapsed time spent doing collections in milliseconds") - .build() - ExpressionSchema("memory_pool", Expression(heapUsedGauge.metadata)) - .withLabel(ExpressionSchema.Role, "jvm") - .withLabel("kind", "Heap") - .withUnit(Bytes) - .withDescription("Heap in use in bytes") - .build() + stats.registerExpression( + ExpressionSchema("jvm_uptime", Expression(uptime.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withUnit(Milliseconds) + .withDescription("The uptime of the JVM in milliseconds")) + gcStats.registerExpression( + ExpressionSchema("gc_cycles", Expression(cycles.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withDescription("The total number of collections that have occurred")) + gcStats.registerExpression( + ExpressionSchema("gc_latency", Expression(msec.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withUnit(Milliseconds) + .withDescription("The total elapsed time spent doing collections in milliseconds")) + heapStats.registerExpression( + ExpressionSchema("memory_pool", Expression(heapUsedGauge.metadata)) + .withLabel(ExpressionSchema.Role, "jvm") + .withLabel("kind", "Heap") + .withUnit(Bytes) + .withDescription("Heap in use in bytes")) } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala index e48881647a..2ea477f780 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala @@ -42,7 +42,7 @@ object BroadcastStatsReceiver { def metadata: Metadata = MultiMetadata(Seq(firstGauge.metadata, secondGauge.metadata)) } - override protected[finagle] def registerExpression( + override def registerExpression( expressionSchema: ExpressionSchema ): Try[Unit] = { Try.collect( @@ -75,7 +75,7 @@ object BroadcastStatsReceiver { def metadata: Metadata = MultiMetadata(gauges.map(_.metadata)) } - override protected[finagle] def registerExpression( + override def registerExpression( expressionSchema: ExpressionSchema ): Try[Unit] = Try.collect(srs.map(_.registerExpression(expressionSchema))) match { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index ef68bb7544..e04e2f606b 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -282,7 +282,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { /** * Designed to match the behavior of Metrics::registerExpression(). */ - override protected[finagle] def registerExpression(schema: ExpressionSchema): Try[Unit] = { + override def registerExpression(schema: ExpressionSchema): Try[Unit] = { if (expressions.contains(schema.schemaKey())) { Throw( ExpressionCollisionException( diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 28a8a6671e..9d12282874 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -61,7 +61,6 @@ object MetricBuilder { * @param percentiles used to indicate buckets for histograms, to be set by the StatsReceiver * @param metricType the type of the metric being constructed * @param isStandard whether the metric should the standard path separator or defer to the metrics store for the separator - * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder */ def apply( keyIndicator: Boolean = false, @@ -76,7 +75,6 @@ object MetricBuilder { percentiles: IndexedSeq[Double] = IndexedSeq.empty, isStandard: Boolean = false, metricType: MetricType, - statsReceiver: StatsReceiver ): MetricBuilder = { new MetricBuilder( keyIndicator, @@ -92,8 +90,7 @@ object MetricBuilder { processPath, percentiles, isStandard, - metricType, - statsReceiver + metricType ) } @@ -107,8 +104,8 @@ object MetricBuilder { * @param metricType the type of the metric being constructed. * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder. */ - def newBuilder(metricType: MetricType, statsReceiver: StatsReceiver): MetricBuilder = - apply(metricType = metricType, statsReceiver = statsReceiver) + def newBuilder(metricType: MetricType): MetricBuilder = + apply(metricType = metricType) /** * Indicate the Metric type, [[CounterType]] will create a [[Counter]], @@ -199,9 +196,7 @@ class MetricBuilder private ( // Only persisted and relevant when building histograms. val percentiles: IndexedSeq[Double], val isStandard: Boolean, - val metricType: MetricType, - @deprecated("Being removed to make MetricBuilder strictly a metric definition.", "2022-03-28") - val statsReceiver: StatsReceiver) + val metricType: MetricType) extends Metadata { /** @@ -235,7 +230,6 @@ class MetricBuilder private ( processPath = processPath, percentiles = percentiles, isStandard = isStandard, - statsReceiver = this.statsReceiver, metricType = metricType ) } @@ -376,6 +370,6 @@ class MetricBuilder private ( } override def toString(): String = { - s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $identity, $relativeName, $processPath, $percentiles, $metricType, $statsReceiver)" + s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $identity, $relativeName, $processPath, $percentiles, $metricType)" } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 9c3dafc968..564c8524ba 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -118,8 +118,9 @@ trait StatsReceiver { /** * Get a [[MetricBuilder metricBuilder]] for this StatsReceiver. */ + @deprecated("Construct a MetricBuilder using its apply method", "2022-05-11") def metricBuilder(metricType: MetricType): MetricBuilder = - MetricBuilder(metricType = metricType, statsReceiver = this) + MetricBuilder(metricType = metricType) /** * Get a [[Counter counter]] with the given `name`. @@ -453,7 +454,12 @@ trait StatsReceiver { final def scope(namespaces: String*): StatsReceiver = namespaces.foldLeft(this)((statsReceiver, name) => statsReceiver.scope(name)) - protected[finagle] def registerExpression(expressionSchema: ExpressionSchema): Try[Unit] = + /** + * Register an `ExpressionSchema`. + * + * Implementations that support expressions should override this to consume expressions. + */ + def registerExpression(expressionSchema: ExpressionSchema): Try[Unit] = Return.Unit /** diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala index 9dd6a6af62..b9ff3fa550 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala @@ -16,7 +16,7 @@ trait StatsReceiverProxy extends StatsReceiver with DelegatingStatsReceiver { def stat(metricBuilder: MetricBuilder): Stat = self.stat(metricBuilder) def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = self.addGauge(metricBuilder)(f) - override protected[finagle] def registerExpression( + override def registerExpression( expressionSchema: ExpressionSchema ): Try[Unit] = self.registerExpression(expressionSchema) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala index adda7291a8..1fb2fdd4c9 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/Expression.scala @@ -1,6 +1,5 @@ package com.twitter.finagle.stats.exp -import com.twitter.finagle.stats.LoadedStatsReceiver import com.twitter.finagle.stats.Metadata import com.twitter.finagle.stats.MetricBuilder import com.twitter.finagle.stats.MetricBuilder.HistogramType @@ -8,18 +7,6 @@ import com.twitter.finagle.stats.StatsReceiver import scala.annotation.varargs private[twitter] object Expression { - private[exp] def getStatsReceivers(expr: Expression): Set[StatsReceiver] = expr match { - case FunctionExpression(_, exprs) => - exprs.foldLeft(Set.empty[StatsReceiver]) { - case (acc, expr) => - getStatsReceivers(expr) ++ acc - } - case MetricExpression(metricBuilder, _) => Set(metricBuilder.statsReceiver) - case HistogramExpression(metricBuilder, _) => Set(metricBuilder.statsReceiver) - case ConstantExpression(_) => Set.empty - case StringExpression(_, _) => Set(LoadedStatsReceiver) - case NoExpression => Set.empty - } /** * Create a expression with a constant number in double diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index 7bd44b50ed..4465488181 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -10,7 +10,6 @@ import com.twitter.finagle.stats.exp.HistogramComponent.Max import com.twitter.finagle.stats.exp.HistogramComponent.Min import com.twitter.finagle.stats.exp.HistogramComponent.Percentile import com.twitter.finagle.stats.exp.HistogramComponent.Sum -import com.twitter.util.Try /** * ExpressionSchema is builder class that construct an expression with its metadata. @@ -58,20 +57,6 @@ case class ExpressionSchema private ( private[finagle] def withServiceName(name: String): ExpressionSchema = withLabel(ExpressionSchema.ServiceName, name) - /** - * Register this ExpressionSchema in StatsReceiver. - * - * Returns: Unit when the schema is successfully published to StatsReceiver - * Throws: [[ExpressionCollisionException]] if the schemaKey has mapped value - */ - def build(): Try[Unit] = { - Expression.getStatsReceivers(expr).toSeq match { - case Seq(sr) => sr.registerExpression(this) - case first +: _ => first.registerExpression(this) - case _ => Try.Unit // should not happen - } - } - def schemaKey(): ExpressionSchemaKey = { ExpressionSchemaKey(name, labels, namespace) } @@ -81,7 +66,7 @@ case class ExpressionSchema private ( * ExpressionSchemaKey is a class that exists to serve as a key into a Map of ExpressionSchemas. * It is simply a subset of the fields of the ExpressionSchema. Namely: * @param name - * @param serviceName + * @param labels * @param namespaces */ case class ExpressionSchemaKey( diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 7bf538a582..2fa9418329 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -130,11 +130,12 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra test("register expressions") { val stats = new InMemoryStatsReceiver() - ExpressionSchema( - "a", - Expression( - MetricBuilder(name = Seq("counter"), metricType = CounterType, statsReceiver = stats))) - stats.expressions.contains(ExpressionSchemaKey("a", Map(), Nil)) + stats.registerExpression( + ExpressionSchema( + "a", + Expression(MetricBuilder(name = Seq("counter"), metricType = CounterType)))) + + assert(stats.expressions.contains(ExpressionSchemaKey("a", Map(), Nil))) } test("print") { @@ -361,13 +362,13 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(parts.length == 3) assert( parts( - 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(coolGauge, coolGauge, Map(), false), List(), None, Vector(), GaugeType, InMemoryStatsReceiver)") + 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(coolGauge, coolGauge, Map(), false), List(), None, Vector(), GaugeType)") assert( parts( - 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(rad/histo, rad_histo, Map(), false), List(), None, Vector(), HistogramType, InMemoryStatsReceiver/rad)") + 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(rad/histo, rad_histo, Map(), false), List(), None, Vector(), HistogramType)") assert( parts( - 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(sweet/counter, sweet_counter, Map(), false), List(), None, Vector(), CounterType, InMemoryStatsReceiver)") + 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(sweet/counter, sweet_counter, Map(), false), List(), None, Vector(), CounterType)") } finally { ps.close() } @@ -380,19 +381,20 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra val bHisto = sr.scope("test").stat("b") val cGauge = sr.scope("test").addGauge("c") { 1 } - val expression = ExpressionSchema( - "test_expression", - Expression(aCounter.metadata).plus(Expression(bHisto.metadata, HistogramComponent.Min) - .plus(Expression(cGauge.metadata))) - ).build() + val expression = sr.registerExpression( + ExpressionSchema( + "test_expression", + Expression(aCounter.metadata).plus(Expression(bHisto.metadata, HistogramComponent.Min) + .plus(Expression(cGauge.metadata))) + )) // what we expected as hydrated metric builders val aaSchema = - MetricBuilder(name = Seq("test", "a"), metricType = CounterType, statsReceiver = sr) + MetricBuilder(name = Seq("test", "a"), metricType = CounterType) val bbSchema = - MetricBuilder(name = Seq("test", "b"), metricType = HistogramType, statsReceiver = sr) + MetricBuilder(name = Seq("test", "b"), metricType = HistogramType) val ccSchema = - MetricBuilder(name = Seq("test", "c"), metricType = GaugeType, statsReceiver = sr) + MetricBuilder(name = Seq("test", "c"), metricType = GaugeType) val expected_expression = ExpressionSchema( "test_expression", @@ -408,15 +410,17 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra val aCounter = sr.scope("test").counter("a") - ExpressionSchema( - "test_expression_0", - Expression(aCounter.metadata) - ).withLabel(ExpressionSchema.Role, "test").build() - - ExpressionSchema( - "test_expression_1", - Expression(aCounter.metadata) - ).withLabel(ExpressionSchema.Role, "dont appear").build() + sr.registerExpression( + ExpressionSchema( + "test_expression_0", + Expression(aCounter.metadata) + ).withLabel(ExpressionSchema.Role, "test")) + + sr.registerExpression( + ExpressionSchema( + "test_expression_1", + Expression(aCounter.metadata) + ).withLabel(ExpressionSchema.Role, "dont appear")) val withLabel = sr.getAllExpressionsWithLabel(ExpressionSchema.Role, "test") assert(withLabel.size == 1) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala index 4c29c73e87..101a6c327f 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MetricBuilderTest.scala @@ -6,10 +6,8 @@ import org.scalatest.funsuite.AnyFunSuite class MetricBuilderTest extends AnyFunSuite { test("metadataScopeSeparator affects stringification") { - val mb = MetricBuilder( - name = Seq("foo", "bar"), - metricType = CounterType, - statsReceiver = new InMemoryStatsReceiver) + val mb = MetricBuilder(name = Seq("foo", "bar"), metricType = CounterType) + assert(mb.toString.contains("foo/bar")) metadataScopeSeparator.setSeparator("-") assert(mb.toString.contains("foo-bar")) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 2d0030ba9a..99c883c1a3 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -245,13 +245,13 @@ class StatsReceiverTest extends AnyFunSuite { test("StatsReceiver validate and record metrics") { val sr = new InMemoryStatsReceiver() - val counter = MetricBuilder(name = Seq("a"), metricType = CounterType, statsReceiver = sr) + val counter = MetricBuilder(name = Seq("a"), metricType = CounterType) val counterishGauge = - MetricBuilder(name = Seq("b"), metricType = CounterishGaugeType, statsReceiver = sr) - val gauge = MetricBuilder(name = Seq("c"), metricType = GaugeType, statsReceiver = sr) - val stat = MetricBuilder(name = Seq("d"), metricType = HistogramType, statsReceiver = sr) + MetricBuilder(name = Seq("b"), metricType = CounterishGaugeType) + val gauge = MetricBuilder(name = Seq("c"), metricType = GaugeType) + val stat = MetricBuilder(name = Seq("d"), metricType = HistogramType) val unlatchedCounter = - MetricBuilder(name = Seq("e"), metricType = UnlatchedCounter, statsReceiver = sr) + MetricBuilder(name = Seq("e"), metricType = UnlatchedCounter) sr.addGauge(gauge)(1) sr.addGauge(counterishGauge)(1) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 423c5507a2..6ac7c9714b 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -17,11 +17,11 @@ class ExpressionSchemaTest extends AnyFunSuite { val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) val successMb = - MetricBuilder(name = Seq("success"), metricType = CounterType, statsReceiver = clientSR) + MetricBuilder(name = Seq("success"), metricType = CounterType) val failuresMb = - MetricBuilder(name = Seq("failures"), metricType = CounterType, statsReceiver = clientSR) + MetricBuilder(name = Seq("failures"), metricType = CounterType) val latencyMb = - MetricBuilder(name = Seq("latency"), metricType = HistogramType, statsReceiver = clientSR) + MetricBuilder(name = Seq("latency"), metricType = HistogramType) val sum = Expression(successMb).plus(Expression(failuresMb)) def slowQuery(ctl: TimeControl, succeed: Boolean): Unit = { @@ -58,8 +58,8 @@ class ExpressionSchemaTest extends AnyFunSuite { .withUnit(Milliseconds) .withDescription("The latency of the slow query") - successRate.build() - latency.build() + clientSR.registerExpression(successRate) + clientSR.registerExpression(latency) def runTheQuery(succeed: Boolean): Unit = { Time.withCurrentTimeFrozen { ctl => @@ -104,14 +104,12 @@ class ExpressionSchemaTest extends AnyFunSuite { test("histogram expressions come with default labels") { new Ctx { - val latencyP90 = - ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Percentile(0.9))) - .build() - val latencyP99 = - ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Percentile(0.99))) - .build() - val latencyAvg = - ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Avg)).build() + clientSR.registerExpression( + ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Percentile(0.9)))) + clientSR.registerExpression( + ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Percentile(0.99)))) + clientSR.registerExpression( + ExpressionSchema("latency", Expression(latencyMb, HistogramComponent.Avg))) assert(sr.expressions.contains(nameToKey("latency", Map("bucket" -> "p90")))) assert(sr.expressions.contains(nameToKey("latency", Map("bucket" -> "p99")))) @@ -121,19 +119,19 @@ class ExpressionSchemaTest extends AnyFunSuite { test("expressions with different namespaces are all stored") { new Ctx { - val successRate1 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) - .withNamespace("section1") - .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) - .withUnit(Percentage) - .withDescription("The success rate of the slow query") - .build() - - val successRate2 = ExpressionSchema("success_rate", Expression(successMb).divide(sum)) - .withNamespace("section2", "a") - .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) - .withUnit(Percentage) - .withDescription("The success rate of the slow query") - .build() + clientSR.registerExpression( + ExpressionSchema("success_rate", Expression(successMb).divide(sum)) + .withNamespace("section1") + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query")) + + clientSR.registerExpression( + ExpressionSchema("success_rate", Expression(successMb).divide(sum)) + .withNamespace("section2", "a") + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query")) assert(sr.expressions.values.size == 2) val namespaces = sr.expressions.values.toSeq.map { exprSchema => @@ -146,19 +144,19 @@ class ExpressionSchemaTest extends AnyFunSuite { test("expressions with the same key - name, labels and namespaces") { new Ctx { - val successRate1 = ExpressionSchema("success", Expression(successMb)) - .withNamespace("section1") - .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) - .withUnit(Percentage) - .withDescription("The success rate of the slow query") - .build() - - val successRate2 = ExpressionSchema("success", Expression(successMb).divide(sum)) - .withNamespace("section1") - .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) - .withUnit(Percentage) - .withDescription("The success rate of the slow query") - .build() + val successRate1 = clientSR.registerExpression( + ExpressionSchema("success", Expression(successMb)) + .withNamespace("section1") + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query")) + + val successRate2 = clientSR.registerExpression( + ExpressionSchema("success", Expression(successMb).divide(sum)) + .withNamespace("section1") + .withBounds(MonotoneThresholds(GreaterThan, 99.5, 99.75)) + .withUnit(Percentage) + .withDescription("The success rate of the slow query")) // only store the first // the second attempt will be a Throw @@ -174,13 +172,14 @@ class ExpressionSchemaTest extends AnyFunSuite { val loadedSr = LoadedStatsReceiver.self val sr = new InMemoryStatsReceiver LoadedStatsReceiver.self = sr - val successRate = ExpressionSchema( - "success_rate", - Expression(100).multiply( - Expression(Seq("clnt", "aclient", "success"), isCounter = true).divide( - Expression(Seq("clnt", "aclient", "success"), isCounter = true).plus( - Expression(Seq("clnt", "aclient", "failures"), isCounter = true)))) - ).withRole(Client).withServiceName("aclient").build() + sr.registerExpression( + ExpressionSchema( + "success_rate", + Expression(100).multiply( + Expression(Seq("clnt", "aclient", "success"), isCounter = true).divide( + Expression(Seq("clnt", "aclient", "success"), isCounter = true).plus( + Expression(Seq("clnt", "aclient", "failures"), isCounter = true)))) + ).withRole(Client).withServiceName("aclient")) assert(sr.expressions.values.size == 1) assert( @@ -196,14 +195,15 @@ class ExpressionSchemaTest extends AnyFunSuite { val sr = new InMemoryStatsReceiver // create a success rate ExpressionSchema // it is safe to call `.get` since we don't return `None` for now - ExpressionSchema( - "success_rate", - Expression(100).multiply( - Expression - .counter(sr, Seq("clnt", "aclient", "success")).get.divide(Expression - .counter(sr, Seq("clnt", "aclient", "success")).get.plus( - Expression.counter(sr, Seq("clnt", "aclient", "failures"), showRollup = true).get))) - ).withRole(Client).withServiceName("aclient").build() + sr.registerExpression( + ExpressionSchema( + "success_rate", + Expression(100).multiply( + Expression + .counter(sr, Seq("clnt", "aclient", "success")).get.divide(Expression + .counter(sr, Seq("clnt", "aclient", "success")).get.plus( + Expression.counter(sr, Seq("clnt", "aclient", "failures"), showRollup = true).get))) + ).withRole(Client).withServiceName("aclient")) assert(sr.expressions.values.size == 1) assert( @@ -221,7 +221,8 @@ class ExpressionSchemaTest extends AnyFunSuite { val expressions: Seq[Expression] = Expression.stats(sr, Seq("clnt", "aclient", "latencies")) expressions.map { expression => - ExpressionSchema("latencies", expression).withRole(Client).withServiceName("aclient").build() + sr.registerExpression( + ExpressionSchema("latencies", expression).withRole(Client).withServiceName("aclient")) } assert(sr.expressions.values.size == expressions.size) From 22701b2a8966949593cd4c1ad31e308ee2e2b8a1 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 24 May 2022 02:12:01 +0000 Subject: [PATCH 664/796] util-stats: Fix some .toString implementations in TranslatingStatsReceivers Problem Some customers use the StatsReceiver.toString method to figure out the scopes of the stats receiver. While an anti-pattern, we need to support it until we get a better solution for them. However, this only applies to hierarchical translation but we're doing it for dimensional only transformations as well. Solution Fix these situations and add some tests. Differential Revision: https://phabricator.twitter.biz/D898036 --- .../finagle/stats/NameTranslatingStatsReceiver.scala | 5 ++++- .../twitter/finagle/stats/TranslatingStatsReceiver.scala | 5 ----- .../com/twitter/finagle/stats/StatsReceiverTest.scala | 7 +++++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index d98f03680a..568792af3c 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -40,7 +40,10 @@ abstract class NameTranslatingStatsReceiver( builder.withIdentity(nextIdentity) } - override def toString: String = s"$self/$namespacePrefix" + override def toString: String = mode match { + case FullTranslation | HierarchicalOnly => s"$self/$namespacePrefix" + case DimensionalOnly => self.toString + } } private object NameTranslatingStatsReceiver { diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala index 8bfe82039d..06b5ae85c8 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala @@ -23,8 +23,6 @@ abstract class TranslatingStatsReceiver( override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { self.addGauge(translate(metricBuilder))(f) } - - override def toString: String = s"Translating($self)" } private object TranslatingStatsReceiver { @@ -52,8 +50,5 @@ private object TranslatingStatsReceiver { private[this] def newIdentity(identity: Identity): Identity = { identity.copy(labels = identity.labels + labelPair) } - - // We preserve this because unfortunately it is sometimes parsed - override def toString: String = s"$self/$labelValue" } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 99c883c1a3..2bebd771b8 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -216,6 +216,13 @@ class StatsReceiverTest extends AnyFunSuite { assert("InMemoryStatsReceiver" == inMem.toString) assert("InMemoryStatsReceiver/scope1" == inMem.scope("scope1").toString) + + assert("InMemoryStatsReceiver/scope1" == inMem.hierarchicalScope("scope1").toString) + + assert("InMemoryStatsReceiver" == inMem.dimensionalScope("scope1").toString) + + assert("InMemoryStatsReceiver" == inMem.label("a", "b").toString) + assert( "InMemoryStatsReceiver/scope1/scope2" == inMem.scope("scope1").scope("scope2").toString From 5764017a91d765f0880e025e9869c766fb275fb3 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 25 May 2022 23:25:43 +0000 Subject: [PATCH 665/796] finagle-core: Start shaping the ClientStatsReceiver for dimensional use Problem We use the ClientStatsReceiver for a lot of our metrics but it's currently not carrying the scopes or labels we will want. Solution Add those labels. We also don't want to allow people to change the metric paths so we make `def setRootScope` only change the hierarchical scopes. Differential Revision: https://phabricator.twitter.biz/D897853 --- .../main/scala/com/twitter/finagle/stats/MetricBuilder.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 9d12282874..bfdcfd8197 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -146,11 +146,11 @@ object MetricBuilder { * * @note this type is expected to undergo some churn as dimensional metric support matures. */ - final case class Identity private[stats] ( + final case class Identity( hierarchicalName: Seq[String], dimensionalName: Seq[String], labels: Map[String, String], - hierarchicalOnly: Boolean) { + hierarchicalOnly: Boolean = false) { override def toString: String = { val hname = hierarchicalName.mkString(metadataScopeSeparator()) val dname = dimensionalName.mkString(DimensionalNameScopeSeparator) From 04e69454c3cea41ad24c2af74582c412dc0fb29f Mon Sep 17 00:00:00 2001 From: Eugene Yokota Date: Thu, 26 May 2022 20:29:44 +0000 Subject: [PATCH 666/796] [Bazel] Add empty BUILD file to prevent full build Problem ------- SQ query walks up to find the affected targets, and end up causing full build for various targets. Solution -------- This adds guard `BUILD.bazel` file for all toplevel directories that do not have a BUILD file. JIRA Issues: DPB-15983, DPB-14048 Differential Revision: https://phabricator.twitter.biz/D902602 --- BUILD.bazel | 1 + 1 file changed, 1 insertion(+) create mode 100644 BUILD.bazel diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 0000000000..dbd3d06196 --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1 @@ +# This prevents SQ query from grabbing //:all since it traverses up once to find a BUILD (DPB-14048) From 1f0096a9f65d74ca934027e848de8ad997259551 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Thu, 2 Jun 2022 20:44:43 +0000 Subject: [PATCH 667/796] Add myself to owners For my past and future contributions, add myself as owners to some projects. Differential Revision: https://phabricator.twitter.biz/D906693 --- util-stats/PROJECT | 1 + 1 file changed, 1 insertion(+) diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 584008ecfa..fba5e978d3 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -3,6 +3,7 @@ owners: - banderson - beisenman + - jyan - lhe - mnakamura - roanta From 8c96a44240688d9a32a42e8c6e3e225244055dab Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Mon, 6 Jun 2022 19:59:27 +0000 Subject: [PATCH 668/796] source: Upgrade Jackson to 2.13.3 Problem / Solution New Jackson version is out. Let's upgrade 2.13.2 to 2.13.3. JIRA Issues: CSL-12047 Differential Revision: https://phabricator.twitter.biz/D906005 --- CHANGELOG.rst | 6 ++++++ build.sbt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e220163575..8a09ad4d45 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-jackson: Update Jackson library to version 2.13.3 ``PHAB_ID=D906005`` + + 22.4.0 ------ diff --git a/build.sbt b/build.sbt index a6d9927d29..04845a7d99 100644 --- a/build.sbt +++ b/build.sbt @@ -7,7 +7,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "22.5.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.13.2" +val jacksonVersion = "2.13.3" val json4sVersion = "4.0.3" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" From d96155a9c22286252356f84c9bb8b5abbc5284da Mon Sep 17 00:00:00 2001 From: Lily He Date: Fri, 10 Jun 2022 16:23:05 +0000 Subject: [PATCH 669/796] finagle-core: add "implementation" label to DefaultStatsReceiver Problem: We want to indicate which metrics are added by users as custom, application-specific metrics. Solution: Add the scope "app" to DefaultStatsReceiver. Add the label `implementation="app"`. All custom metrics will look like ``` app_serviceA_metricB {implementation="app"} ``` JIRA Issues: CSL-12028 Differential Revision: https://phabricator.twitter.biz/D901903 --- .../finagle/stats/LoadedStatsReceiver.scala | 6 +++++- .../finagle/stats/DefaultStatsReceiverTest.scala | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/DefaultStatsReceiverTest.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala index 89fcf746c6..34b8a71d8a 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala @@ -23,7 +23,11 @@ object LoadedStatsReceiver extends StatsReceiverProxy { * [[com.twitter.finagle.util.LoadService]] mechanism. */ object DefaultStatsReceiver extends StatsReceiverProxy { - def self: StatsReceiver = LoadedStatsReceiver + val self: StatsReceiver = + LoadedStatsReceiver + .dimensionalScope("app") + .label("implementation", "app") + override def repr: DefaultStatsReceiver.type = this def get: StatsReceiver = this diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/DefaultStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DefaultStatsReceiverTest.scala new file mode 100644 index 0000000000..0fdeecf1fd --- /dev/null +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DefaultStatsReceiverTest.scala @@ -0,0 +1,15 @@ +package com.twitter.finagle.stats + +import org.scalatest.funsuite.AnyFunSuite + +class DefaultStatsReceiverTest extends AnyFunSuite { + test("DefaultStatsReceiver has app in dimensional scope and label") { + LoadedStatsReceiver.self = new InMemoryStatsReceiver + val fooCounter = DefaultStatsReceiver.counter("foo") + val identity = fooCounter.metadata.toMetricBuilder.get.identity + + assert(identity.dimensionalName == Seq("app", "foo")) + assert(identity.hierarchicalName == Seq("foo")) + assert(identity.labels == Map("implementation" -> "app")) + } +} From 633cf5b0649c8377207201c633fdd7ecf6ae6758 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 13 Jun 2022 16:46:29 +0000 Subject: [PATCH 670/796] finagle-core: Move SourceRole subtypes into its object Problem We have about a million things named `Client` or `Server` and that now includes the subtypes of `SourceRole`. This can make things really difficult to navigate. Solution Move the subtypes to the object of `SourceRole` to make the relationship more clear. Differential Revision: https://phabricator.twitter.biz/D909744 --- CHANGELOG.rst | 6 +++++ .../twitter/finagle/stats/MetricBuilder.scala | 23 +++++++++++++----- .../MetricBuilderCompilationTest.java | 4 ++-- .../RoleConfiguredStatsReceiverTest.scala | 24 +++++++++---------- .../stats/exp/ExpressionSchemaTest.scala | 16 ++++++++----- 5 files changed, 47 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8a09ad4d45..0e87d68d07 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Breaking API Changes +~~~~~~~~~~~~~~~~~~~~ + +* util-stats: SourceRole subtypes have been moved into the SourceRole object to make their + relationship to the parent type more clear. ``PHAB_ID=D909744`` + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index bfdcfd8197..ba79af82d0 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -16,16 +16,28 @@ import scala.annotation.varargs * request to another service). In many cases, there is no relevant role for a metric, in which * case NoRole should be used. */ -sealed trait SourceRole { +sealed abstract class SourceRole private () { /** * Java-friendly helper for accessing the object itself. */ def getInstance(): SourceRole = this } -case object NoRoleSpecified extends SourceRole -case object Client extends SourceRole -case object Server extends SourceRole + +object SourceRole { + case object NoRoleSpecified extends SourceRole + case object Client extends SourceRole + case object Server extends SourceRole + + /** + * Java-friendly accessors + */ + def noRoleSpecified(): SourceRole = NoRoleSpecified + + def client(): SourceRole = Client + + def server(): SourceRole = Server +} /** * finagle-stats has configurable scope separators. As this package is wrapped by finagle-stats, we @@ -66,7 +78,7 @@ object MetricBuilder { keyIndicator: Boolean = false, description: String = "No description provided", units: MetricUnit = Unspecified, - role: SourceRole = NoRoleSpecified, + role: SourceRole = SourceRole.NoRoleSpecified, verbosity: Verbosity = Verbosity.Default, sourceClass: Option[String] = None, name: Seq[String] = Seq.empty, @@ -102,7 +114,6 @@ object MetricBuilder { * than being forced to use the mega apply method. * * @param metricType the type of the metric being constructed. - * @param statsReceiver used for the actual metric creation, set by the StatsReceiver when creating a MetricBuilder. */ def newBuilder(metricType: MetricType): MetricBuilder = apply(metricType = metricType) diff --git a/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java b/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java index 9148e9feff..ee0bc24b72 100644 --- a/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java +++ b/util-stats/src/test/java/com/twitter/compiletest/MetricBuilderCompilationTest.java @@ -8,7 +8,7 @@ import com.twitter.finagle.stats.MetricBuilder; import com.twitter.finagle.stats.MetricTypes; import com.twitter.finagle.stats.Microseconds; -import com.twitter.finagle.stats.NoRoleSpecified; +import com.twitter.finagle.stats.SourceRole; import com.twitter.finagle.stats.StatsReceiver; import com.twitter.finagle.stats.Verbosity; @@ -37,7 +37,7 @@ public void testWithMethods() { .withSourceClass(new Some<>("com.twitter.finagle.AwesomeClass")) .withIdentifier(new Some<>("/p/foo/bar")) .withUnits(Microseconds.getInstance()) - .withRole(NoRoleSpecified.getInstance()) + .withRole(SourceRole.noRoleSpecified()) .withName("my", "very", "cool", "name") .withRelativeName("cool", "name") .withPercentiles(0.99, 0.88, 0.77); diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala index 1ff7e4fb8e..3a3d5eae1c 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/RoleConfiguredStatsReceiverTest.scala @@ -5,30 +5,30 @@ import org.scalatest.funsuite.AnyFunSuite class RoleConfiguredStatsReceiverTest extends AnyFunSuite { test("RoleConfiguredSR configures metadata role correctly") { val mem = new InMemoryStatsReceiver - val clientSR = new RoleConfiguredStatsReceiver(mem, Client) - val serverSR = new RoleConfiguredStatsReceiver(mem, Server) + val clientSR = new RoleConfiguredStatsReceiver(mem, SourceRole.Client) + val serverSR = new RoleConfiguredStatsReceiver(mem, SourceRole.Server) clientSR.counter("foo") - assert(mem.schemas.get(Seq("foo")).get.role == Client) + assert(mem.schemas.get(Seq("foo")).get.role == SourceRole.Client) clientSR.addGauge("bar")(3) - assert(mem.schemas.get(Seq("bar")).get.role == Client) + assert(mem.schemas.get(Seq("bar")).get.role == SourceRole.Client) clientSR.stat("baz") - assert(mem.schemas.get(Seq("baz")).get.role == Client) + assert(mem.schemas.get(Seq("baz")).get.role == SourceRole.Client) serverSR.counter("fee") - assert(mem.schemas.get(Seq("fee")).get.role == Server) + assert(mem.schemas.get(Seq("fee")).get.role == SourceRole.Server) serverSR.addGauge("fi")(3) - assert(mem.schemas.get(Seq("fi")).get.role == Server) + assert(mem.schemas.get(Seq("fi")).get.role == SourceRole.Server) serverSR.stat("foe") - assert(mem.schemas.get(Seq("foe")).get.role == Server) + assert(mem.schemas.get(Seq("foe")).get.role == SourceRole.Server) } test("RoleConfiguredSR equality") { val mem = new InMemoryStatsReceiver - val clientA = RoleConfiguredStatsReceiver(mem, Client) - val clientB = RoleConfiguredStatsReceiver(mem, Client) - val serverA = RoleConfiguredStatsReceiver(mem, Server) - val serverB = RoleConfiguredStatsReceiver(mem, Server) + val clientA = RoleConfiguredStatsReceiver(mem, SourceRole.Client) + val clientB = RoleConfiguredStatsReceiver(mem, SourceRole.Client) + val serverA = RoleConfiguredStatsReceiver(mem, SourceRole.Server) + val serverB = RoleConfiguredStatsReceiver(mem, SourceRole.Server) assert(clientA == clientB) assert(clientA != serverA) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index 6ac7c9714b..a8dacc1592 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -14,7 +14,7 @@ import org.scalatest.funsuite.AnyFunSuite class ExpressionSchemaTest extends AnyFunSuite { trait Ctx { val sr = new InMemoryStatsReceiver - val clientSR = RoleConfiguredStatsReceiver(sr, Client, Some("downstream")) + val clientSR = RoleConfiguredStatsReceiver(sr, SourceRole.Client, Some("downstream")) val successMb = MetricBuilder(name = Seq("success"), metricType = CounterType) @@ -32,7 +32,9 @@ class ExpressionSchemaTest extends AnyFunSuite { } val downstreamLabel = - Map(ExpressionSchema.Role -> Client.toString, ExpressionSchema.ServiceName -> "downstream") + Map( + ExpressionSchema.Role -> SourceRole.Client.toString, + ExpressionSchema.ServiceName -> "downstream") protected[this] def nameToKey( name: String, @@ -88,7 +90,8 @@ class ExpressionSchemaTest extends AnyFunSuite { assert(sr.stats(Seq("latency")) == Seq(50, 50)) assert( - sr.expressions(nameToKey("success_rate")).labels(ExpressionSchema.Role) == Client.toString) + sr.expressions(nameToKey("success_rate")).labels( + ExpressionSchema.Role) == SourceRole.Client.toString) } } @@ -179,7 +182,7 @@ class ExpressionSchemaTest extends AnyFunSuite { Expression(Seq("clnt", "aclient", "success"), isCounter = true).divide( Expression(Seq("clnt", "aclient", "success"), isCounter = true).plus( Expression(Seq("clnt", "aclient", "failures"), isCounter = true)))) - ).withRole(Client).withServiceName("aclient")) + ).withRole(SourceRole.Client).withServiceName("aclient")) assert(sr.expressions.values.size == 1) assert( @@ -203,7 +206,7 @@ class ExpressionSchemaTest extends AnyFunSuite { .counter(sr, Seq("clnt", "aclient", "success")).get.divide(Expression .counter(sr, Seq("clnt", "aclient", "success")).get.plus( Expression.counter(sr, Seq("clnt", "aclient", "failures"), showRollup = true).get))) - ).withRole(Client).withServiceName("aclient")) + ).withRole(SourceRole.Client).withServiceName("aclient")) assert(sr.expressions.values.size == 1) assert( @@ -222,7 +225,8 @@ class ExpressionSchemaTest extends AnyFunSuite { Expression.stats(sr, Seq("clnt", "aclient", "latencies")) expressions.map { expression => sr.registerExpression( - ExpressionSchema("latencies", expression).withRole(Client).withServiceName("aclient")) + ExpressionSchema("latencies", expression) + .withRole(SourceRole.Client).withServiceName("aclient")) } assert(sr.expressions.values.size == expressions.size) From 9f15cce6c83d4992c14063ecdc8527fd6b5b9a1d Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 21 Jun 2022 21:25:48 +0000 Subject: [PATCH 671/796] finagle-core: Don't migrate all metrics yet Problem We are not sure we want to migrate users metrics to dimensional form yet, but we do want to start working on finagles metrics. Solution Make metrics hierarchical only by default but turn all finagle metrics into dimensional metrics as well via in a special proxy StatsReceiver. Differential Revision: https://phabricator.twitter.biz/D909544 --- .../twitter/finagle/stats/MetricBuilder.scala | 8 ++--- .../stats/NameTranslatingStatsReceiver.scala | 13 +++++---- .../twitter/finagle/stats/StatsReceiver.scala | 15 +++++----- .../finagle/stats/StatsReceiverProxy.scala | 3 ++ .../stats/InMemoryStatsReceiverTest.scala | 29 +++++++++++++++---- .../finagle/stats/StatsReceiverTest.scala | 10 +++---- 6 files changed, 49 insertions(+), 29 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index ba79af82d0..430b5dd222 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -97,7 +97,7 @@ object MetricBuilder { sourceClass, // For now we're not allowing construction w"ith labels. // They need to be added later via `.withLabels`. - Identity(name, name, Map.empty, false), + Identity(name, name), relativeName, processPath, percentiles, @@ -160,8 +160,8 @@ object MetricBuilder { final case class Identity( hierarchicalName: Seq[String], dimensionalName: Seq[String], - labels: Map[String, String], - hierarchicalOnly: Boolean = false) { + labels: Map[String, String] = Map.empty, + hierarchicalOnly: Boolean = true) { override def toString: String = { val hname = hierarchicalName.mkString(metadataScopeSeparator()) val dname = dimensionalName.mkString(DimensionalNameScopeSeparator) @@ -276,7 +276,7 @@ class MetricBuilder private ( copy(identity = identity.copy(hierarchicalName = name, dimensionalName = name)) } - private[finagle] def withIdentity(identity: Identity): MetricBuilder = copy(identity = identity) + private[twitter] def withIdentity(identity: Identity): MetricBuilder = copy(identity = identity) /** * The hierarchical name of the metric diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala index 568792af3c..0afedb4af1 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/NameTranslatingStatsReceiver.scala @@ -6,7 +6,6 @@ package com.twitter.finagle.stats * * @param self The underlying StatsReceiver to which translated names are passed * @param namespacePrefix the namespace used for translations - * @param mode determines which names to translate: hierarchical, dimensional, or both. */ abstract class NameTranslatingStatsReceiver( self: StatsReceiver, @@ -17,7 +16,7 @@ abstract class NameTranslatingStatsReceiver( import NameTranslatingStatsReceiver._ def this(self: StatsReceiver, namespacePrefix: String) = - this(self, namespacePrefix, NameTranslatingStatsReceiver.FullTranslation) + this(self, namespacePrefix, NameTranslatingStatsReceiver.HierarchicalOnly) def this(self: StatsReceiver) = this(self, "") @@ -40,13 +39,15 @@ abstract class NameTranslatingStatsReceiver( builder.withIdentity(nextIdentity) } - override def toString: String = mode match { - case FullTranslation | HierarchicalOnly => s"$self/$namespacePrefix" - case DimensionalOnly => self.toString + override def toString: String = { + mode match { + case FullTranslation | HierarchicalOnly => s"$self/$namespacePrefix" + case DimensionalOnly => self.toString + } } } -private object NameTranslatingStatsReceiver { +object NameTranslatingStatsReceiver { /** The mode with which to translate stats, either hierarchical, dimensional, or both. */ sealed trait Mode diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 564c8524ba..e088c8620f 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -382,18 +382,17 @@ trait StatsReceiver { */ def scope(namespace: String): StatsReceiver = { if (namespace == "") this - else - new ScopeTranslatingStatsReceiver( - this, - namespace, - NameTranslatingStatsReceiver.FullTranslation) + else new ScopeTranslatingStatsReceiver(this, namespace, scopeTranslation) } + private[finagle] def scopeTranslation: NameTranslatingStatsReceiver.Mode = + NameTranslatingStatsReceiver.HierarchicalOnly + /** * Create a new `StatsReceiver` that will add a scope that is only used when the metric is * emitted in hierarchical form. */ - private[finagle] def hierarchicalScope(namespace: String): StatsReceiver = { + private[finagle] final def hierarchicalScope(namespace: String): StatsReceiver = { if (namespace == "") this else new ScopeTranslatingStatsReceiver( @@ -406,7 +405,7 @@ trait StatsReceiver { * Create a new `StatsReceiver` that will add a scope that is only used when the metric is * emitted in dimensional form. */ - private[finagle] def dimensionalScope(namespace: String): StatsReceiver = { + private[finagle] final def dimensionalScope(namespace: String): StatsReceiver = { if (namespace == "") this else new ScopeTranslatingStatsReceiver( @@ -418,7 +417,7 @@ trait StatsReceiver { /** * Create a new `StatsReceiver` that will add the specified label to all created metrics. */ - private[finagle] def label(labelName: String, labelValue: String): StatsReceiver = { + private[finagle] final def label(labelName: String, labelValue: String): StatsReceiver = { require(labelName.nonEmpty) if (labelValue == "") this else new LabelTranslatingStatsReceiver(this, labelName, labelValue) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala index b9ff3fa550..3fba6d0148 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiverProxy.scala @@ -16,6 +16,9 @@ trait StatsReceiverProxy extends StatsReceiver with DelegatingStatsReceiver { def stat(metricBuilder: MetricBuilder): Stat = self.stat(metricBuilder) def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = self.addGauge(metricBuilder)(f) + override private[finagle] def scopeTranslation: NameTranslatingStatsReceiver.Mode = + self.scopeTranslation + override def registerExpression( expressionSchema: ExpressionSchema ): Try[Unit] = self.registerExpression(expressionSchema) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 2fa9418329..4e40fd6b79 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -362,13 +362,13 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(parts.length == 3) assert( parts( - 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(coolGauge, coolGauge, Map(), false), List(), None, Vector(), GaugeType)") + 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(coolGauge, coolGauge, Map(), true), List(), None, Vector(), GaugeType)") assert( parts( - 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(rad/histo, rad_histo, Map(), false), List(), None, Vector(), HistogramType)") + 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(rad/histo, histo, Map(), true), List(), None, Vector(), HistogramType)") assert( parts( - 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(sweet/counter, sweet_counter, Map(), false), List(), None, Vector(), CounterType)") + 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(sweet/counter, sweet_counter, Map(), true), List(), None, Vector(), CounterType)") } finally { ps.close() } @@ -390,11 +390,28 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra // what we expected as hydrated metric builders val aaSchema = - MetricBuilder(name = Seq("test", "a"), metricType = CounterType) + MetricBuilder(metricType = CounterType) + .withIdentity( + MetricBuilder.Identity( + hierarchicalName = Seq("test", "a"), + dimensionalName = Seq("a"), + labels = Map.empty)) + val bbSchema = - MetricBuilder(name = Seq("test", "b"), metricType = HistogramType) + MetricBuilder(metricType = HistogramType) + .withIdentity( + MetricBuilder.Identity( + hierarchicalName = Seq("test", "b"), + dimensionalName = Seq("b"), + labels = Map.empty)) + val ccSchema = - MetricBuilder(name = Seq("test", "c"), metricType = GaugeType) + MetricBuilder(metricType = GaugeType) + .withIdentity( + MetricBuilder.Identity( + hierarchicalName = Seq("test", "c"), + dimensionalName = Seq("c"), + labels = Map.empty)) val expected_expression = ExpressionSchema( "test_expression", diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 2bebd771b8..dfd7ecaea7 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -112,7 +112,7 @@ class StatsReceiverTest extends AnyFunSuite { assert(sr.schemas(Seq("bar")).identity.labels.isEmpty) } - test("StatsReceiver.scope: adds a scope to both names and doesn't poison dimensional metrics") { + test("StatsReceiver.scope: by default has the same behavior has hierarchicalScope") { val sr = new InMemoryStatsReceiver val scopedCounter = sr.scope("cool").counter("numbers") @@ -120,8 +120,8 @@ class StatsReceiverTest extends AnyFunSuite { assert(mb.identity.labels.isEmpty) assert(mb.identity.hierarchicalName == Seq("cool", "numbers")) - assert(mb.identity.dimensionalName == Seq("cool", "numbers")) - assert(!mb.identity.hierarchicalOnly) + assert(mb.identity.dimensionalName == Seq("numbers")) + assert(mb.identity.hierarchicalOnly) } test("StatsReceiver.hierarchicalScope: composes with label correctly") { @@ -137,7 +137,7 @@ class StatsReceiverTest extends AnyFunSuite { assert(mb.identity.hierarchicalName == Seq("clnt", "backend", "requests")) assert(mb.identity.dimensionalName == Seq("requests")) assert(mb.identity.labels == Map("clnt" -> "backend")) - assert(!mb.identity.hierarchicalOnly) + assert(mb.identity.hierarchicalOnly) } test("StatsReceiver.dimensionalScope: composes with label correctly") { @@ -154,7 +154,7 @@ class StatsReceiverTest extends AnyFunSuite { assert(mb.identity.hierarchicalName == Seq("bar", "requests")) assert(mb.identity.dimensionalName == Seq("foo", "baz", "requests")) assert(mb.identity.labels == Map("clnt" -> "backend")) - assert(!mb.identity.hierarchicalOnly) + assert(mb.identity.hierarchicalOnly) } test("StatsReceiver.scope: prefix stats by a scope string") { From 9d7852d68c78d4db5104ae5a8e13844ae4acd931 Mon Sep 17 00:00:00 2001 From: Nora Howard Date: Fri, 1 Jul 2022 02:41:59 +0000 Subject: [PATCH 672/796] [jdk-11-migration] mark targets that must stay on java8 for one scalding target Explicitly mark targets that cannot migrate to java11 as java8, so they will not be affected when the default platform is changed to java11. Targets that cannot migrate include hadoop and scalding jobs, targets that they depend on, and other special cases. See https//go.twitter.biz/jdk11migration for detailed migration docs. This patch is for src/scala/com/twitter/ads/batch/job/targeting/handle_ml/trainer:handle_targeting_training_data_gen-binary and its dependencies. JIRA Issues: VM-5402 Differential Revision: https://phabricator.twitter.biz/D920109 --- .../src/main/scala/com/twitter/app/lifecycle/BUILD | 1 + util-app/src/main/java/com/twitter/app/BUILD | 1 + util-app/src/main/scala/BUILD | 1 + util-cache/src/main/scala/BUILD | 1 + util-cache/src/main/scala/com/twitter/cache/BUILD | 1 + util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD | 1 + util-codec/src/main/scala/com/twitter/util/BUILD | 1 + util-core/BUILD | 2 ++ util-core/src/main/java/com/twitter/concurrent/BUILD | 1 + util-core/src/main/java/com/twitter/io/BUILD | 1 + util-core/src/main/java/com/twitter/service/BUILD | 1 + util-core/src/main/java/com/twitter/util/BUILD | 1 + util-core/src/main/java/com/twitter/util/javainterop/BUILD | 1 + util-core/src/main/scala/com/twitter/concurrent/BUILD | 1 + util-core/src/main/scala/com/twitter/conversions/BUILD | 1 + util-core/src/main/scala/com/twitter/io/BUILD | 1 + util-function/src/main/java/com/twitter/function/BUILD | 1 + util-hashing/src/main/scala/com/twitter/hashing/BUILD | 1 + .../src/main/java/com/twitter/util/jackson/annotation/BUILD | 1 + .../src/main/scala/com/fasterxml/jackson/databind/BUILD | 1 + util-jackson/src/main/scala/com/twitter/util/jackson/BUILD | 1 + .../src/main/scala/com/twitter/util/jackson/caseclass/BUILD | 1 + .../scala/com/twitter/util/jackson/caseclass/exceptions/BUILD | 1 + .../src/main/scala/com/twitter/util/jackson/serde/BUILD | 1 + util-jvm/src/main/scala/com/twitter/jvm/BUILD | 1 + util-lint/src/main/scala/com/twitter/util/lint/BUILD | 1 + util-logging/src/main/scala/com/twitter/logging/BUILD | 1 + util-reflect/src/main/scala/com/twitter/util/reflect/BUILD | 1 + util-registry/src/main/scala/com/twitter/util/registry/BUILD | 1 + util-routing/src/main/scala/com/twitter/util/routing/BUILD | 1 + util-security/src/main/scala/com/twitter/util/security/BUILD | 1 + util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD | 1 + .../src/main/scala/com/twitter/util/logging/BUILD | 1 + util-stats/src/main/scala/com/twitter/finagle/stats/BUILD | 1 + util-tunable/src/main/scala/com/twitter/util/tunable/BUILD | 1 + .../src/main/java/com/twitter/util/validation/BUILD | 1 + .../src/main/java/com/twitter/util/validation/constraints/BUILD | 1 + util-validator/src/main/scala/com/twitter/util/validation/BUILD | 1 + .../src/main/scala/com/twitter/util/validation/cfg/BUILD | 1 + .../com/twitter/util/validation/constraintvalidation/BUILD | 1 + .../main/scala/com/twitter/util/validation/conversions/BUILD | 1 + .../src/main/scala/com/twitter/util/validation/engine/BUILD | 1 + .../src/main/scala/com/twitter/util/validation/executable/BUILD | 1 + .../src/main/scala/com/twitter/util/validation/internal/BUILD | 1 + .../twitter/util/validation/internal/constraintvalidation/BUILD | 1 + .../scala/com/twitter/util/validation/internal/engine/BUILD | 1 + .../twitter/util/validation/internal/metadata/descriptor/BUILD | 1 + .../scala/com/twitter/util/validation/internal/validators/BUILD | 1 + .../twitter/util/validation/internal/validators/notempty/BUILD | 1 + .../com/twitter/util/validation/internal/validators/size/BUILD | 1 + .../src/main/scala/com/twitter/util/validation/metadata/BUILD | 1 + .../main/scala/org/hibernate/validator/internal/engine/BUILD | 1 + .../org/hibernate/validator/internal/properties/javabean/BUILD | 1 + 53 files changed, 54 insertions(+) diff --git a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD index a723d1b67e..5263e66e3c 100644 --- a/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD +++ b/util-app-lifecycle/src/main/scala/com/twitter/app/lifecycle/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-app-lifecycle", diff --git a/util-app/src/main/java/com/twitter/app/BUILD b/util-app/src/main/java/com/twitter/app/BUILD index 3267814a89..14f71e72de 100644 --- a/util-app/src/main/java/com/twitter/app/BUILD +++ b/util-app/src/main/java/com/twitter/app/BUILD @@ -1,5 +1,6 @@ java_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = artifact( org = "com.twitter", name = "util-app-java", diff --git a/util-app/src/main/scala/BUILD b/util-app/src/main/scala/BUILD index 831041edb1..8dd879de25 100644 --- a/util-app/src/main/scala/BUILD +++ b/util-app/src/main/scala/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-app", diff --git a/util-cache/src/main/scala/BUILD b/util-cache/src/main/scala/BUILD index 03257e94d6..adfc85a118 100644 --- a/util-cache/src/main/scala/BUILD +++ b/util-cache/src/main/scala/BUILD @@ -1,6 +1,7 @@ scala_library( # This target exists only to act as an aggregator jar. sources = ["PantsWorkaroundCache.scala"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-cache", diff --git a/util-cache/src/main/scala/com/twitter/cache/BUILD b/util-cache/src/main/scala/com/twitter/cache/BUILD index 476e56da0f..4e526fc10a 100644 --- a/util-cache/src/main/scala/com/twitter/cache/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-cache-cache", diff --git a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD index 43d840823a..cb1bc43396 100644 --- a/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/main/scala/com/twitter/cache/caffeine/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-cache-caffeine", diff --git a/util-codec/src/main/scala/com/twitter/util/BUILD b/util-codec/src/main/scala/com/twitter/util/BUILD index 631158a2b6..d88e57df30 100644 --- a/util-codec/src/main/scala/com/twitter/util/BUILD +++ b/util-codec/src/main/scala/com/twitter/util/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-codec", diff --git a/util-core/BUILD b/util-core/BUILD index d4877d3f6d..a2edd7cd0d 100644 --- a/util-core/BUILD +++ b/util-core/BUILD @@ -27,6 +27,7 @@ scala_library( "src/main/scala/com/twitter/util/*.scala", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-core-util", @@ -51,6 +52,7 @@ scala_library( # This target exists only to act as an aggregator jar. name = "scala", sources = ["src/main/scala/PantsWorkaroundCore.scala"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-core", diff --git a/util-core/src/main/java/com/twitter/concurrent/BUILD b/util-core/src/main/java/com/twitter/concurrent/BUILD index cbe1b1c4b9..51c4ae41b6 100644 --- a/util-core/src/main/java/com/twitter/concurrent/BUILD +++ b/util-core/src/main/java/com/twitter/concurrent/BUILD @@ -1,5 +1,6 @@ java_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = artifact( org = "com.twitter", name = "util-core-java-concurrent", diff --git a/util-core/src/main/java/com/twitter/io/BUILD b/util-core/src/main/java/com/twitter/io/BUILD index 902ed1fdbd..febf1f1097 100644 --- a/util-core/src/main/java/com/twitter/io/BUILD +++ b/util-core/src/main/java/com/twitter/io/BUILD @@ -1,5 +1,6 @@ java_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = artifact( org = "com.twitter", name = "util-core-java-io", diff --git a/util-core/src/main/java/com/twitter/service/BUILD b/util-core/src/main/java/com/twitter/service/BUILD index 6fd0561ef6..59f3eda853 100644 --- a/util-core/src/main/java/com/twitter/service/BUILD +++ b/util-core/src/main/java/com/twitter/service/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = artifact( org = "com.twitter", name = "util-core-java-service", diff --git a/util-core/src/main/java/com/twitter/util/BUILD b/util-core/src/main/java/com/twitter/util/BUILD index 47cf7a5d8a..38f12c51e1 100644 --- a/util-core/src/main/java/com/twitter/util/BUILD +++ b/util-core/src/main/java/com/twitter/util/BUILD @@ -1,5 +1,6 @@ java_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = artifact( org = "com.twitter", name = "util-core-java", diff --git a/util-core/src/main/java/com/twitter/util/javainterop/BUILD b/util-core/src/main/java/com/twitter/util/javainterop/BUILD index b3ef14a3a4..9ac9a13322 100644 --- a/util-core/src/main/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/main/java/com/twitter/util/javainterop/BUILD @@ -1,5 +1,6 @@ java_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = artifact( org = "com.twitter", name = "util-core-java-javainterop", diff --git a/util-core/src/main/scala/com/twitter/concurrent/BUILD b/util-core/src/main/scala/com/twitter/concurrent/BUILD index c3e2fb8676..0c1d94d45f 100644 --- a/util-core/src/main/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/main/scala/com/twitter/concurrent/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-core-concurrent", diff --git a/util-core/src/main/scala/com/twitter/conversions/BUILD b/util-core/src/main/scala/com/twitter/conversions/BUILD index ce2c90c609..e09dad9d2b 100644 --- a/util-core/src/main/scala/com/twitter/conversions/BUILD +++ b/util-core/src/main/scala/com/twitter/conversions/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-core-conversions", diff --git a/util-core/src/main/scala/com/twitter/io/BUILD b/util-core/src/main/scala/com/twitter/io/BUILD index ce279b5be3..223da6fec4 100644 --- a/util-core/src/main/scala/com/twitter/io/BUILD +++ b/util-core/src/main/scala/com/twitter/io/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-core-io", diff --git a/util-function/src/main/java/com/twitter/function/BUILD b/util-function/src/main/java/com/twitter/function/BUILD index 7be7675e1b..ce03a4870f 100644 --- a/util-function/src/main/java/com/twitter/function/BUILD +++ b/util-function/src/main/java/com/twitter/function/BUILD @@ -1,5 +1,6 @@ java_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = artifact( org = "com.twitter", name = "util-function-java", diff --git a/util-hashing/src/main/scala/com/twitter/hashing/BUILD b/util-hashing/src/main/scala/com/twitter/hashing/BUILD index aaf6592722..b4531eb750 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/main/scala/com/twitter/hashing/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-hashing", diff --git a/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD b/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD index abf69ac7be..0b686a99bb 100644 --- a/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD +++ b/util-jackson-annotations/src/main/java/com/twitter/util/jackson/annotation/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-jackson-annotations", diff --git a/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD index 434b79a7c1..d82b398136 100644 --- a/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD +++ b/util-jackson/src/main/scala/com/fasterxml/jackson/databind/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-jackson-objectmapper-copy", diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD index c5ee98857a..fdcebdf661 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-jackson-core", diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD index e011cf42e6..898dffaa96 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-jackson-caseclass", diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD index 5b2663b130..2b2bdcf4a8 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/caseclass/exceptions/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-jackson-caseclass-exceptions", diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD index 0a5c93b812..3d56d5356c 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/serde/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-jackson-serde", diff --git a/util-jvm/src/main/scala/com/twitter/jvm/BUILD b/util-jvm/src/main/scala/com/twitter/jvm/BUILD index 759853a0e7..abdf640814 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/BUILD +++ b/util-jvm/src/main/scala/com/twitter/jvm/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-jvm", diff --git a/util-lint/src/main/scala/com/twitter/util/lint/BUILD b/util-lint/src/main/scala/com/twitter/util/lint/BUILD index bd8676cd38..972662835b 100644 --- a/util-lint/src/main/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/main/scala/com/twitter/util/lint/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-lint", diff --git a/util-logging/src/main/scala/com/twitter/logging/BUILD b/util-logging/src/main/scala/com/twitter/logging/BUILD index 73efcef057..986f79252f 100644 --- a/util-logging/src/main/scala/com/twitter/logging/BUILD +++ b/util-logging/src/main/scala/com/twitter/logging/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-logging", diff --git a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD index 974f7e9415..5b38a2e602 100644 --- a/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/main/scala/com/twitter/util/reflect/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-reflect", diff --git a/util-registry/src/main/scala/com/twitter/util/registry/BUILD b/util-registry/src/main/scala/com/twitter/util/registry/BUILD index d8c88c2b2c..641b4fb983 100644 --- a/util-registry/src/main/scala/com/twitter/util/registry/BUILD +++ b/util-registry/src/main/scala/com/twitter/util/registry/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-registry", diff --git a/util-routing/src/main/scala/com/twitter/util/routing/BUILD b/util-routing/src/main/scala/com/twitter/util/routing/BUILD index c93d9fb289..137bafba10 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/main/scala/com/twitter/util/routing/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-routing", diff --git a/util-security/src/main/scala/com/twitter/util/security/BUILD b/util-security/src/main/scala/com/twitter/util/security/BUILD index 4ca8f81b88..faa70c2910 100644 --- a/util-security/src/main/scala/com/twitter/util/security/BUILD +++ b/util-security/src/main/scala/com/twitter/util/security/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-security", diff --git a/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD index ba59507a01..7b6dfa2e56 100644 --- a/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/main/scala/com/twitter/util/logging/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-slf4j", diff --git a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD index 9485be0942..cad256b797 100644 --- a/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-jul-bridge/src/main/scala/com/twitter/util/logging/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-slf4j-jul-bridge", diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD index a57ff9d56b..9b31b3e727 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-stats", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD index 3e2f06da1f..aa8915afba 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-tunable", diff --git a/util-validator-constraints/src/main/java/com/twitter/util/validation/BUILD b/util-validator-constraints/src/main/java/com/twitter/util/validation/BUILD index d3641f85b5..4a6f8e42a5 100644 --- a/util-validator-constraints/src/main/java/com/twitter/util/validation/BUILD +++ b/util-validator-constraints/src/main/java/com/twitter/util/validation/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-java", diff --git a/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/BUILD b/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/BUILD index a18b26250b..c2610076fe 100644 --- a/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/BUILD +++ b/util-validator-constraints/src/main/java/com/twitter/util/validation/constraints/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-java-constraints", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/BUILD index 6c6b8fa8ff..b715b02ab6 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-core", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD index 9882de32e7..28fe3eb9f5 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/cfg/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-cfg", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD index 36a3c73482..4adad13620 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/constraintvalidation/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-constraintvalidation", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD index 59cb1b93d1..da9927dad0 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/conversions/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-conversions", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD index 22ed6fbfc3..1b821957f3 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/engine/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-engine", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD index b40328c9c4..95829dce5a 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/executable/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-executable", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD index c342934833..577f8846a6 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD index cb8fc3ca3f..90be7adb9f 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/constraintvalidation/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal-constraintvalidation", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD index 1fd3148c58..d42615e719 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/engine/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal-engine", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD index ec8430abd1..ff9bf3312c 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/metadata/descriptor/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal-metadata-descriptor", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD index 988d90853f..dd9baf6944 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal-validators", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD index 14a934a33a..86fd42e9a9 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/notempty/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal-validators-notempty", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD index 109f82ffa8..2af10aac5d 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/validators/size/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal-validators-size", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD index 67650a7915..eba48d8d0b 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/metadata/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-metadata", diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD b/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD index 883e09bd4b..e5bff6803b 100644 --- a/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/engine/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal-hibernate-engine", diff --git a/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD index 66abf2112f..71e13675c0 100644 --- a/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD +++ b/util-validator/src/main/scala/org/hibernate/validator/internal/properties/javabean/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal-hibernate-properties-javabeans", From 3b94d2e628d6d62d678006c3177f46dfeb1e0ec9 Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Fri, 1 Jul 2022 23:58:30 +0000 Subject: [PATCH 673/796] util-jackson: Use field value type as field type for optional fields Problem When deserializing a case class, we interpret the field type with the field constructor type. For optional field, Jackson returns `Option[Object]` as the constructor type. However, field validations would require a specific field type in order to run the constraint validator. This causes an exception to throw when an optional field is validated via deserializing with Jackson. Solution Use the filed value type as the field type for optional field. Differential Revision: https://phabricator.twitter.biz/D915377 --- CHANGELOG.rst | 2 + .../util/jackson/ScalaObjectMapperTest.scala | 48 ++++++++++++++++++- .../twitter/util/jackson/caseclasses.scala | 4 ++ .../util/validation/internal/Types.scala | 19 ++++++-- 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0e87d68d07..1acbc79bf9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,6 +18,8 @@ Runtime Behavior Changes * util-jackson: Update Jackson library to version 2.13.3 ``PHAB_ID=D906005`` +* util-jackson: Deserialized case classes with validation on optional fields shouldn't throw an error. + ``PHAB_ID=D915377`` 22.4.0 ------ diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala index 11b0a1970f..8f2d3ca5ec 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala @@ -12,7 +12,6 @@ import com.fasterxml.jackson.databind.JsonMappingException import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.MapperFeature import com.fasterxml.jackson.databind.PropertyNamingStrategies -import com.fasterxml.jackson.databind.PropertyNamingStrategy import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper} import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.scala.DefaultScalaModule @@ -1359,6 +1358,53 @@ class ScalaObjectMapperTest parse[CaseClassWithOption]("""{"value":null}""") should be(CaseClassWithOption(None)) } + test( + "A case class with an Option[Int] member and validation annotation is parsable from a JSON object with value that passes the validation") { + parse[CaseClassWithOptionAndValidation]("""{ "towing_capacity": 10000 }""") should be( + CaseClassWithOptionAndValidation(Some(10000))) + } + + test( + "A case class with an Option[Int] member and validation annotation is parsable from a JSON object with null value") { + parse[CaseClassWithOptionAndValidation]("""{ "towing_capacity": null }""") should be( + CaseClassWithOptionAndValidation(None)) + } + + test( + "A case class with an Option[Int] member and validation annotation is parsable from a JSON without that field") { + parse[CaseClassWithOptionAndValidation]("""{}""") should be( + CaseClassWithOptionAndValidation(None)) + } + + test( + "A case class with an Option[Int] member and validation annotation is parsable from a JSON object with value that fails the validation") { + val e = intercept[CaseClassMappingException] { + parse[CaseClassWithOptionAndValidation]("""{ "towing_capacity": 1 }""") should be( + CaseClassWithOptionAndValidation(Some(1))) + } + e.errors.head.getMessage should be("towing_capacity: must be greater than or equal to 100") + } + + test( + "A case class with an Option[Boolean] member and incompatible validation annotation is not parsable from a JSON object") { + val e = intercept[jakarta.validation.UnexpectedTypeException] { + parse[CaseClassWithOptionAndIncompatibleValidation]("""{ "are_you": true }""") should be( + CaseClassWithOptionAndIncompatibleValidation(Some(true))) + } + } + + test( + "A case class with an Option[Boolean] member and incompatible validation annotation is parsable from a JSON object with null value") { + parse[CaseClassWithOptionAndIncompatibleValidation]("""{ "are_you": null }""") should be( + CaseClassWithOptionAndIncompatibleValidation(None)) + } + + test( + "A case class with an Option[Boolean] member and incompatible validation annotation is parsable from a JSON object without that field") { + parse[CaseClassWithOptionAndIncompatibleValidation]("""{}""") should be( + CaseClassWithOptionAndIncompatibleValidation(None)) + } + test("A case class with a JsonNode member generates a field of the given type") { generate(CaseClassWithJsonNode(new IntNode(2))) should be("""{"value":2}""") } diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala index 1915f7a1dc..2382f358e7 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/caseclasses.scala @@ -311,6 +311,10 @@ case class CaseClassWithOverloadedField(id: Long) { case class CaseClassWithOption(value: Option[String] = None) +case class CaseClassWithOptionAndValidation(@Min(100) towingCapacity: Option[Int] = None) + +case class CaseClassWithOptionAndIncompatibleValidation(@Min(100) areYou: Option[Boolean] = None) + case class CaseClassWithJsonNode(value: JsonNode) case class CaseClassWithAllTypes( diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/Types.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/Types.scala index 394dd4a179..42eae7840c 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/Types.scala +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/Types.scala @@ -1,6 +1,7 @@ package com.twitter.util.validation.internal -import org.json4s.reflect.{Reflector, ScalaType} +import org.json4s.reflect.Reflector +import org.json4s.reflect.ScalaType private[validation] object Types { @@ -28,8 +29,20 @@ private[validation] object Types { else Reflector.scalaTypeOf(value.getClass) } else if (scalaType.isOption) { - if (scalaType.typeArgs.nonEmpty) scalaType.typeArgs.head - else Reflector.scalaTypeOf(classOf[Object]) + if (scalaType.typeArgs.nonEmpty) { + // When deserializing a case class, we interpret the field + // type with the field constructor type. For optional + // field, Jackson returns Option[Object] as the constructor + // type. However, field validations would require a + // specific field type in order to run the constraint + // validator. So we use the field value type as the field + // type for optional field. + if (scalaType.typeArgs.head.erasure.isInstanceOf[Object]) + Reflector.scalaTypeOf(value.getClass) + else scalaType.typeArgs.head + } else { + Reflector.scalaTypeOf(classOf[Object]) + } } else scalaType } From f1c1cad83bce3e64e9fc3dd12342f023c529ad01 Mon Sep 17 00:00:00 2001 From: Automated go/ktf Flagger Date: Wed, 6 Jul 2022 15:13:55 +0000 Subject: [PATCH 674/796] The test target "util/util-app/src/test/scala:scala" is failing too often in master in CI. Summary: Problem: The test target "util/util-app/src/test/scala:scala" is failing too often in master in CI. Logs available here: Known to fail ticket: [CSL-12163](https://jira.twitter.biz/browse/CSL-12163) Solution: It has been marked "known-to-fail" in the BUILD file Result: This will cause it to be skipped in future builds, and its failures ignored in submit queue. This implies that code changes that further break this test will not be prevented from being submitted to the master branch, until the problem is fixed and the test is no longer marked known-to-fail. Reviewers: O5328 source:/util/util-app/! Subscribers: O56283 source:/util/util-app/ Differential Revision: https://phabricator.twitter.biz/D928757 TBR=true --- util-app/src/test/scala/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index 7f352a7b01..be166055e4 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -4,6 +4,7 @@ junit_tests( strict_deps = True, tags = [ "bazel-compatible", + "known-to-fail-jira:CSL-12163", "non-exclusive", ], dependencies = [ From b84d80be05e0c217f5109f8a2af49e7a647613fa Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 7 Jul 2022 16:56:01 +0000 Subject: [PATCH 675/796] util/util-app - Fix Flaggable InetSocketAddress test Problem The Flaggable InetSocketAddress test uses a real IP address which when parsed into an InetSocketAddress and `getHostName` is called results in a hostname lookup and SecurityManager check to see if a connection to that host is allowed. For unit test meant to run in CI the installed SecurityManager fails this check since connections off of localhost is (rightly) not allowed. Solution Don't use an external IP address. Use the loopback, `127.0.0.1` which should allow to test the same functionality without violating the SecurityManager. Additionally fix the remote getHostname asseertion. Note this is assuming that the machine `/etc/hosts` file maps `localhost` to the loopback `127.0.0.1` which should be valid in almost all cases but we may hit a case where for some reason this is not true. But for now this should work as expected. JIRA Issues: CSL-12163 Differential Revision: https://phabricator.twitter.biz/D929587 --- util-app/src/test/scala/BUILD | 1 - .../src/test/scala/com/twitter/app/FlaggableTest.scala | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index be166055e4..7f352a7b01 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -4,7 +4,6 @@ junit_tests( strict_deps = True, tags = [ "bazel-compatible", - "known-to-fail-jira:CSL-12163", "non-exclusive", ], dependencies = [ diff --git a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala index 417af43736..91bddb83bf 100644 --- a/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlaggableTest.scala @@ -1,6 +1,7 @@ package com.twitter.app -import com.twitter.util.{Duration, Time} +import com.twitter.util.Duration +import com.twitter.util.Time import java.time.LocalTime import org.scalatest.funsuite.AnyFunSuite @@ -35,10 +36,10 @@ class FlaggableTest extends AnyFunSuite { assert(local.getAddress.isAnyLocalAddress) assert(local.getPort == port) - val ip = "141.211.133.111" + val ip = "127.0.0.1" val remote = Flaggable.ofInetSocketAddress.parse(s"$ip:$port") - assert(remote.getHostName == remote.getHostName) + assert(remote.getHostName == "localhost") assert(remote.getPort == port) assert(Flaggable.ofInetSocketAddress.show(local) == s":$port") From bba655c1355cd9763635f293b4a63bdb51c80e16 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Fri, 8 Jul 2022 02:39:38 +0000 Subject: [PATCH 676/796] [mockito] remove unrequired mockito dep on 3rdparty/src/jvm/org/scalatestplus/mockito JIRA Issues: DEAD-3 Differential Revision: https://phabricator.twitter.biz/D929683 --- util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index 78fc535f3e..bb13ae1692 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -11,7 +11,7 @@ scala_library( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "3rdparty/jvm/org/scalatestplus:scalacheck", "3rdparty/jvm/org/slf4j:slf4j-api", "util/util-core/src/main/scala/com/twitter/io", From 23decadf307236ac6100113c16c390e151115c8a Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Fri, 8 Jul 2022 19:39:09 +0000 Subject: [PATCH 677/796] util: build with non-RC scala3 release Signed-off-by: Kyle Bahr Problem/Solution: Update scala cross version to use release rather than release candidate. Result Closes twitter/util#300 JIRA Issues: CSL-12125 Differential Revision: https://phabricator.twitter.biz/D930220 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 04845a7d99..780fca8bf9 100644 --- a/build.sbt +++ b/build.sbt @@ -91,7 +91,7 @@ val defaultScalaSettings = Seq( ) val defaultScala3EnabledSettings = Seq( scalaVersion := _scalaVersion, - crossScalaVersions := _crossScalaVersions ++ Seq("3.0.2-RC1") + crossScalaVersions := _crossScalaVersions ++ Seq("3.0.2") ) // Our dependencies or compiler options may differ for both Scala 2 and 3. We branch here From 29009a9a9e5aab795148b79913139c9aa64fe5aa Mon Sep 17 00:00:00 2001 From: Uriel Salazar Date: Fri, 8 Jul 2022 21:52:30 +0000 Subject: [PATCH 678/796] VM-5413: batch exclude heron binary targets from migration by marking them with platform 8 Explicitly mark targets that cannot migrate to java11 as java8, so they will not be affected when the default platform is changed to java11. Targets that cannot migrate include hadoop and scalding jobs, targets that they depend on, and other special cases. See https//go.twitter.biz/jdk11migration for detailed migration docs. This patch marks all of the heron_binarary targets and their dependencies. JIRA Issues: VM-5415 Differential Revision: https://phabricator.twitter.biz/D925912 --- util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD | 1 + util-thrift/src/main/scala/com/twitter/util/BUILD | 1 + 2 files changed, 2 insertions(+) diff --git a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD index bd4f3a57ad..2acd27445a 100644 --- a/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/main/scala/com/twitter/cache/guava/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-cache-guava", diff --git a/util-thrift/src/main/scala/com/twitter/util/BUILD b/util-thrift/src/main/scala/com/twitter/util/BUILD index 6809678a85..4187c8639a 100644 --- a/util-thrift/src/main/scala/com/twitter/util/BUILD +++ b/util-thrift/src/main/scala/com/twitter/util/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-thrift", From 78e60f911a660bbe756defd70a154d2fd15f4439 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Mon, 11 Jul 2022 16:56:36 +0000 Subject: [PATCH 679/796] vm-team: Roll back blender and friends and mark as ignored Problem blender has been seeing stability issues when they use JDK11. Solution Let's roll them and their deps back for now. We'll roll forward once we've worked things out. JIRA Issues: VM-5439, VM-5410 Differential Revision: https://phabricator.twitter.biz/D930920 --- util-stats/src/main/java/com/twitter/finagle/stats/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-stats/src/main/java/com/twitter/finagle/stats/BUILD b/util-stats/src/main/java/com/twitter/finagle/stats/BUILD index 148f15ebad..245d3d6916 100644 --- a/util-stats/src/main/java/com/twitter/finagle/stats/BUILD +++ b/util-stats/src/main/java/com/twitter/finagle/stats/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = artifact( org = "com.twitter", name = "util-stats-java", From a7a7492759056111ee11eaccd2456d350653ad46 Mon Sep 17 00:00:00 2001 From: Uriel Salazar Date: Tue, 12 Jul 2022 07:18:57 +0000 Subject: [PATCH 680/796] VM-5413: batch exclude for certain directories and their dependencies Explicitly mark targets that cannot migrate to java11 as java8, so they will not be affected when the default platform is changed to java11. Targets that cannot migrate include hadoop and scalding jobs, targets that they depend on, and other special cases. See https//go.twitter.biz/jdk11migration for detailed migration docs. This patch marks the following list of directories and their dependencies: ``` beaks beam-internal cid experiment-pipeline graph-feature-service interactive-query/looker/deploy science/frontpage src/java/com/twitter/growth_acquisition src/scala/com/twitter/frontpage src/scala/com/twitter/growth_acquisition tests/java/com/twitter/growth_acquisition tests/scala/com/twitter/frontpage tests/scala/com/twitter/growth_acquisition topicstaxonomy vm-team/jdk11-migration vm-team/src/main/python/jdk11_migration vm-team/src/test/python/jdk11_migration ``` JIRA Issues: VM-5415 Differential Revision: https://phabricator.twitter.biz/D926904 --- util-mock/src/main/scala/com/twitter/util/mock/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/util-mock/src/main/scala/com/twitter/util/mock/BUILD b/util-mock/src/main/scala/com/twitter/util/mock/BUILD index 9178b6c262..ede98b769d 100644 --- a/util-mock/src/main/scala/com/twitter/util/mock/BUILD +++ b/util-mock/src/main/scala/com/twitter/util/mock/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-mockito", From 446b39b0967f8d51b7880e37b8e21aa483363ba4 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 12 Jul 2022 18:51:47 +0000 Subject: [PATCH 681/796] finagle-core: Extract standard metrics out of StatsFilter Problem We have a bunch of metrics that we create in StatsFilter, and they're basically duplicates with different names and classification rules. We're already partially extracting them out into the RPCMetrics case class, but it's still in StatsFilter and a bit cluttered. Solution Go all the way and extract the metrics into their own class, BasicServiceMetrics. This gives us a central place to think about these metrics and the StatsFilter is now just in charge of letting BasicServiceMetrics know that a request happened. Differential Revision: https://phabricator.twitter.biz/D917812 --- .../main/scala/com/twitter/finagle/stats/MetricBuilder.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 430b5dd222..ad87263623 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -58,6 +58,8 @@ object MetricBuilder { val DimensionalNameScopeSeparator: String = "_" + val NoDescription: String = "No description provided" + /** * Construct a MethodBuilder. * @@ -76,7 +78,7 @@ object MetricBuilder { */ def apply( keyIndicator: Boolean = false, - description: String = "No description provided", + description: String = NoDescription, units: MetricUnit = Unspecified, role: SourceRole = SourceRole.NoRoleSpecified, verbosity: Verbosity = Verbosity.Default, From a5f711f97abffddd0ba84786d526e1c6f59d9e27 Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Tue, 12 Jul 2022 20:23:13 +0000 Subject: [PATCH 682/796] util, csl/src, csl/proxee: migrate scripts to bazel Problem/Solution We need to migrate scripts in util, csl/src and csl/proxee to use bazel, since pants is deprecated. JIRA Issues: CSL-12148, CSL-12140, CSL-12137 Differential Revision: https://phabricator.twitter.biz/D931631 --- util-tunable/lint-configuration | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-tunable/lint-configuration b/util-tunable/lint-configuration index 5966217244..8c8021f066 100755 --- a/util-tunable/lint-configuration +++ b/util-tunable/lint-configuration @@ -26,4 +26,4 @@ do done args=$(for x in "$@"; do echo "$x"; done) -./pants run --quiet util/util-tunable/src/main/scala/com/twitter/util/tunable/linter:configuration-linter -- $args \ No newline at end of file +./bazel run --ui_event_filters=-info,-stdout,-stderr --noshow_progress util/util-tunable/src/main/scala/com/twitter/util/tunable/linter:configuration-linter -- $args From 6146a5110e88cc63ca54c44920d33ccca75fe34a Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Fri, 15 Jul 2022 00:46:13 +0000 Subject: [PATCH 683/796] util-app: increase timeout for the test Problem c.t.util.AppTest is flaky on oss, one of the AppTest throws `TimeoutException` when closing the App. Solution Increase the timeout. JIRA Issues: CSL-12177 Differential Revision: https://phabricator.twitter.biz/D934473 --- util-app/src/test/scala/com/twitter/app/AppTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 51c10b652b..5c70b2f126 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -54,7 +54,7 @@ class WeNeverCloseButWeDoNotCare extends WeNeverClose { } trait ErrorOnExitApp extends App { - override val defaultCloseGracePeriod: Duration = 5.seconds + override val defaultCloseGracePeriod: Duration = 10.seconds override def exitOnError(throwable: Throwable): Unit = { throw throwable From c83ee0d564126894a2dca51bcb7ac1910f2ec809 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Fri, 15 Jul 2022 16:32:20 +0000 Subject: [PATCH 684/796] util: tidy up build.sbt Problem/Solution Update build.sbt to use the existing lifted function to define dependencies. Signed-off-by: Jing Yan JIRA Issues: CSL-12127 Differential Revision: https://phabricator.twitter.biz/D934588 --- build.sbt | 50 +++---------------- .../util/tunable/JsonTunableMapperTest.scala | 7 ++- 2 files changed, 13 insertions(+), 44 deletions(-) diff --git a/build.sbt b/build.sbt index 780fca8bf9..1d89283e90 100644 --- a/build.sbt +++ b/build.sbt @@ -312,17 +312,7 @@ lazy val utilApp = Project( name := "util-app", libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % "test", - ) ++ { - if (scalaVersion.value.startsWith("2")) { - Seq( - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" - ) - } else { - Seq( - "org.scalatestplus" %% "mockito-3-4" % "3.2.9.0" % "test" - ) - } - }, + ) ++ scalatestMockitoVersionedDep(scalaVersion.value) ).dependsOn(utilAppLifecycle, utilCore, utilRegistry) lazy val utilAppLifecycle = Project( @@ -409,19 +399,17 @@ lazy val utilCore = Project( caffeineLib % "test", scalacheckLib, "org.mockito" % "mockito-core" % mockitoVersion % "test", - ) ++ { + ) ++ + scalatestMockitoVersionedDep(scalaVersion.value) ++ + scalatestScalacheckVersionedDep(scalaVersion.value) ++ { if (scalaVersion.value.startsWith("2")) { Seq( "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2", - "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test" + "org.scala-lang.modules" %% "scala-parser-combinators" % "1.1.2" ) } else { Seq( - "org.scala-lang.modules" %% "scala-parser-combinators" % "2.0.0", - "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", - "org.scalatestplus" %% "mockito-3-4" % "3.2.9.0" % "test" + "org.scala-lang.modules" %% "scala-parser-combinators" % "2.0.0" ) } }, @@ -482,17 +470,7 @@ lazy val utilHashing = Project( name := "util-hashing", libraryDependencies ++= Seq( scalacheckLib - ) ++ { - if (scalaVersion.value.startsWith("2")) { - Seq( - "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", - ) - } else { - Seq( - "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test", - ) - } - }, + ) ++ scalatestScalacheckVersionedDep(scalaVersion.value) ).dependsOn(utilCore % "test") lazy val utilJacksonAnnotations = Project( @@ -585,17 +563,7 @@ lazy val utilRegistry = Project( name := "util-registry", libraryDependencies ++= Seq( "org.mockito" % "mockito-core" % mockitoVersion % "test", - ) ++ { - if (scalaVersion.value.startsWith("2")) { - Seq( - "org.scalatestplus" %% "mockito-3-3" % "3.1.2.0" % "test", - ) - } else { - Seq( - "org.scalatestplus" %% "mockito-3-4" % "3.2.9.0" % "test" - ) - } - } + ) ++ scalatestMockitoVersionedDep(scalaVersion.value) ).dependsOn(utilCore) lazy val utilRouting = Project( @@ -691,8 +659,6 @@ lazy val utilTest = Project( name := "util-test", libraryDependencies ++= Seq( "org.mockito" % "mockito-all" % "1.10.19", - "org.scalatest" %% "scalatest" % "3.1.2", - "org.scalatestplus" %% "junit-4-12" % "3.1.2.0", "org.scalatestplus" %% "mockito-1-10" % "3.1.0.0" ) ).dependsOn(utilCore, utilLogging, utilStats) diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala index f8e87a2515..759d7c4a25 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/JsonTunableMapperTest.scala @@ -5,11 +5,14 @@ import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.deser.std.StdDeserializer import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ -import com.twitter.util.{Duration, Return, StorageUnit, Throw} +import com.twitter.util.Duration +import com.twitter.util.Return +import com.twitter.util.StorageUnit +import com.twitter.util.Throw import scala.jdk.CollectionConverters._ import org.scalatest.funsuite.AnyFunSuite -// Used for veryifying custom deserialization +// Used for verifying custom deserialization case class Foo(number: Double) class JsonTunableMapperTest extends AnyFunSuite { From 994c8faba879065e0a0bbc8522fbb22910a14d2f Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 19 Jul 2022 20:04:12 +0000 Subject: [PATCH 685/796] finagle-core: Metrics using FinagleStatsReceiver get dimensional names Problem We want finagles shared resource metrics like netty threads, timer stats, etc to have dimensional names. Solution This is the first step toward giving those metrics dimensional names. It adds the root labels and scopes. Differential Revision: https://phabricator.twitter.biz/D921315 --- .../finagle/stats/TranslatingStatsReceiver.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala index 06b5ae85c8..e01e5a25ed 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/TranslatingStatsReceiver.scala @@ -27,6 +27,12 @@ abstract class TranslatingStatsReceiver( private object TranslatingStatsReceiver { + private final class IdentityTranslatingStatsReceiver(sr: StatsReceiver, f: Identity => Identity) + extends TranslatingStatsReceiver(sr) { + protected def translate(builder: MetricBuilder): MetricBuilder = + builder.withIdentity(f(builder.identity)) + } + /** * A [[TranslatingStatsReceiver]] for working with both dimensional and hierarchical metrics. * @@ -51,4 +57,8 @@ private object TranslatingStatsReceiver { identity.copy(labels = identity.labels + labelPair) } } + + /** Translate the identity of a metric */ + def translateIdentity(sr: StatsReceiver)(f: Identity => Identity): StatsReceiver = + new IdentityTranslatingStatsReceiver(sr, f) } From f12c50a1fe850827cad8abac1835d0e0fe062c9f Mon Sep 17 00:00:00 2001 From: Anthony Madrigal Date: Wed, 20 Jul 2022 18:12:28 +0000 Subject: [PATCH 686/796] VM-5465:updated BUILD files JIRA Issues: VM-5465 Differential Revision: https://phabricator.twitter.biz/D936033 --- .../src/test/scala/com/twitter/app/lifecycle/BUILD | 1 + util-app/src/test/java/BUILD | 1 + util-app/src/test/scala/BUILD | 1 + util-benchmark/src/main/scala/BUILD | 2 ++ util-benchmark/src/test/scala/BUILD | 1 + util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD | 1 + util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD | 1 + util-cache/src/test/java/BUILD | 1 + util-cache/src/test/scala/com/twitter/cache/BUILD | 2 ++ util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD | 1 + util-codec/src/test/scala/com/twitter/util/BUILD | 1 + util-core/src/test/java/com/twitter/concurrent/BUILD | 1 + util-core/src/test/java/com/twitter/io/BUILD | 1 + util-core/src/test/java/com/twitter/service/BUILD | 1 + util-core/src/test/java/com/twitter/util/BUILD | 2 ++ util-core/src/test/java/com/twitter/util/javainterop/BUILD | 1 + util-core/src/test/scala/com/twitter/concurrent/BUILD | 1 + util-core/src/test/scala/com/twitter/conversions/BUILD | 1 + util-core/src/test/scala/com/twitter/io/BUILD | 1 + util-core/src/test/scala/com/twitter/util/BUILD | 1 + util-hashing/src/test/scala/com/twitter/hashing/BUILD | 1 + util-inject/src/main/scala/com/twitter/util/inject/BUILD | 1 + util-inject/src/test/java/com/twitter/util/inject/BUILD | 1 + util-inject/src/test/scala/com/twitter/util/inject/BUILD | 1 + util-jackson/src/test/java/com/twitter/util/jackson/BUILD | 1 + .../src/test/java/com/twitter/util/jackson/tests/BUILD | 1 + util-jackson/src/test/scala/com/twitter/util/jackson/BUILD | 1 + util-jvm/src/test/java/BUILD | 1 + util-jvm/src/test/scala/BUILD | 3 +++ util-lint/src/test/scala/com/twitter/util/lint/BUILD | 1 + util-logging/src/test/java/BUILD | 1 + util-logging/src/test/scala/BUILD | 1 + util-mock/src/test/java/com/twitter/util/mock/BUILD | 1 + util-mock/src/test/scala/com/twitter/util/mock/BUILD | 1 + util-reflect/src/test/java/com/twitter/util/reflect/BUILD | 1 + util-reflect/src/test/scala/com/twitter/util/reflect/BUILD | 1 + util-registry/src/test/java/BUILD | 1 + util-registry/src/test/scala/BUILD | 1 + util-routing/src/test/java/com/twitter/util/routing/BUILD | 1 + util-routing/src/test/scala/com/twitter/util/routing/BUILD | 1 + .../src/test/scala/com/twitter/util/routing/dynamic/BUILD | 1 + .../src/test/scala/com/twitter/util/routing/simple/BUILD | 1 + util-security/src/test/java/com/twitter/util/security/BUILD | 1 + util-security/src/test/scala/com/twitter/util/security/BUILD | 1 + util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD | 1 + util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD | 2 ++ util-stats/src/test/java/BUILD | 1 + util-stats/src/test/scala/BUILD | 1 + util-test/src/main/scala/BUILD | 1 + util-test/src/main/scala/com/twitter/logging/BUILD | 1 + util-test/src/main/scala/com/twitter/util/testing/BUILD | 1 + util-test/src/test/scala/com/twitter/util/testing/BUILD | 1 + util-thrift/src/test/java/com/twitter/util/BUILD | 1 + util-thrift/src/test/scala/com/twitter/util/BUILD | 1 + .../src/main/scala/com/twitter/util/tunable/linter/BUILD | 2 ++ util-tunable/src/test/java/com/twitter/util/tunable/BUILD | 1 + util-tunable/src/test/scala/com/twitter/util/tunable/BUILD | 1 + util-validator/src/test/java/com/twitter/util/validation/BUILD | 1 + .../src/test/java/com/twitter/util/validation/tests/BUILD | 1 + .../src/test/scala/com/twitter/util/validation/BUILD | 1 + util-zk-test/src/main/scala/com/twitter/zk/BUILD | 1 + util-zk-test/src/test/java/BUILD | 1 + util-zk-test/src/test/scala/BUILD | 1 + util-zk/src/main/scala/BUILD | 1 + util-zk/src/main/scala/com/twitter/zk/BUILD | 1 + util-zk/src/main/scala/com/twitter/zk/coordination/BUILD | 1 + util-zk/src/test/scala/com/twitter/zk/BUILD | 1 + util-zk/src/test/scala/com/twitter/zk/coordination/BUILD | 1 + 68 files changed, 75 insertions(+) diff --git a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD index ee6ac23c93..ec4032ff29 100644 --- a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD +++ b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-app/src/test/java/BUILD b/util-app/src/test/java/BUILD index 5783c51559..54e9a51b22 100644 --- a/util-app/src/test/java/BUILD +++ b/util-app/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index 7f352a7b01..8443b57b73 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD index 6570923a1e..195f362472 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD @@ -4,6 +4,7 @@ scala_library( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ @@ -24,6 +25,7 @@ scala_library( jvm_binary( name = "jmh", main = "org.openjdk.jmh.Main", + platform = "java8", tags = ["bazel-compatible"], dependencies = [ ":scala", diff --git a/util-benchmark/src/test/scala/BUILD b/util-benchmark/src/test/scala/BUILD index 97c15bc825..d57272063d 100644 --- a/util-benchmark/src/test/scala/BUILD +++ b/util-benchmark/src/test/scala/BUILD @@ -3,6 +3,7 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD index b13589394c..aea988d311 100644 --- a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD index 1245078c44..cdc325f6e2 100644 --- a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD +++ b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-cache/src/test/java/BUILD b/util-cache/src/test/java/BUILD index 705f570301..a5cf81040f 100644 --- a/util-cache/src/test/java/BUILD +++ b/util-cache/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index 64b862b654..7f096b893f 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -5,6 +5,7 @@ junit_tests( "*.scala", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", @@ -32,6 +33,7 @@ scala_library( "AbstractLoadingFutureCacheTest.scala", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index b0931bfe58..f027cebb74 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-codec/src/test/scala/com/twitter/util/BUILD b/util-codec/src/test/scala/com/twitter/util/BUILD index 996e06795b..0c1e041423 100644 --- a/util-codec/src/test/scala/com/twitter/util/BUILD +++ b/util-codec/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-core/src/test/java/com/twitter/concurrent/BUILD b/util-core/src/test/java/com/twitter/concurrent/BUILD index 9444394b34..7258f00909 100644 --- a/util-core/src/test/java/com/twitter/concurrent/BUILD +++ b/util-core/src/test/java/com/twitter/concurrent/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-core/src/test/java/com/twitter/io/BUILD b/util-core/src/test/java/com/twitter/io/BUILD index 917232f1af..596b31d145 100644 --- a/util-core/src/test/java/com/twitter/io/BUILD +++ b/util-core/src/test/java/com/twitter/io/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-core/src/test/java/com/twitter/service/BUILD b/util-core/src/test/java/com/twitter/service/BUILD index 045267f323..b0a9acc072 100644 --- a/util-core/src/test/java/com/twitter/service/BUILD +++ b/util-core/src/test/java/com/twitter/service/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-core/src/test/java/com/twitter/util/BUILD b/util-core/src/test/java/com/twitter/util/BUILD index 70d15c860c..ac362447c8 100644 --- a/util-core/src/test/java/com/twitter/util/BUILD +++ b/util-core/src/test/java/com/twitter/util/BUILD @@ -2,6 +2,7 @@ java_library( name = "object_size_calculator", sources = ["ObjectSizeCalculator.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ @@ -16,6 +17,7 @@ junit_tests( "*.java", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-core/src/test/java/com/twitter/util/javainterop/BUILD b/util-core/src/test/java/com/twitter/util/javainterop/BUILD index 157a877319..8b74c78558 100644 --- a/util-core/src/test/java/com/twitter/util/javainterop/BUILD +++ b/util-core/src/test/java/com/twitter/util/javainterop/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index 0ea4d86f1a..692272da02 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-core/src/test/scala/com/twitter/conversions/BUILD b/util-core/src/test/scala/com/twitter/conversions/BUILD index 58d6c35e82..c001c9eddb 100644 --- a/util-core/src/test/scala/com/twitter/conversions/BUILD +++ b/util-core/src/test/scala/com/twitter/conversions/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 1d21d9cb2d..8566ca088d 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index 5983bc72c2..a016ab64fe 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-hashing/src/test/scala/com/twitter/hashing/BUILD b/util-hashing/src/test/scala/com/twitter/hashing/BUILD index 4d0be301d8..c0b43f401e 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/BUILD +++ b/util-hashing/src/test/scala/com/twitter/hashing/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-inject/src/main/scala/com/twitter/util/inject/BUILD b/util-inject/src/main/scala/com/twitter/util/inject/BUILD index 5790b5347e..22028bbb23 100644 --- a/util-inject/src/main/scala/com/twitter/util/inject/BUILD +++ b/util-inject/src/main/scala/com/twitter/util/inject/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-inject", diff --git a/util-inject/src/test/java/com/twitter/util/inject/BUILD b/util-inject/src/test/java/com/twitter/util/inject/BUILD index eb4a1ebe48..f745fa1628 100644 --- a/util-inject/src/test/java/com/twitter/util/inject/BUILD +++ b/util-inject/src/test/java/com/twitter/util/inject/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = [], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-inject/src/test/scala/com/twitter/util/inject/BUILD b/util-inject/src/test/scala/com/twitter/util/inject/BUILD index 5ee73e7571..f89fd93c60 100644 --- a/util-inject/src/test/scala/com/twitter/util/inject/BUILD +++ b/util-inject/src/test/scala/com/twitter/util/inject/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/BUILD index ba1cb182ca..e834d5832e 100644 --- a/util-jackson/src/test/java/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/java/com/twitter/util/jackson/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD index 1909557de1..d662bae80b 100644 --- a/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD +++ b/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD @@ -1,5 +1,6 @@ junit_tests( sources = ["**/*.java"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD index 19697e3e3c..b8221c4a97 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD @@ -5,6 +5,7 @@ junit_tests( "caseclass/*.scala", "serde/*.scala", ], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-jvm/src/test/java/BUILD b/util-jvm/src/test/java/BUILD index 0f9aa0b2c7..32ca28064d 100644 --- a/util-jvm/src/test/java/BUILD +++ b/util-jvm/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index a98d11ebc0..ca92b48517 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -15,6 +15,7 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", @@ -36,6 +37,7 @@ scala_library( # does not strictly enforce it. sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = COMMON_DEPS + ["util/util-stats/src/main/scala/com/twitter/finagle/stats:stats"], @@ -44,6 +46,7 @@ scala_library( jvm_binary( name = "bin", main = "com.twitter.jvm.EstimatorApp", + platform = "java8", tags = ["bazel-compatible"], dependencies = [":estimator_app"], ) diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index b6834e255b..d509324519 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-logging/src/test/java/BUILD b/util-logging/src/test/java/BUILD index 41a7884036..1d4116e47d 100644 --- a/util-logging/src/test/java/BUILD +++ b/util-logging/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 6126473b03..2220d14e76 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-mock/src/test/java/com/twitter/util/mock/BUILD b/util-mock/src/test/java/com/twitter/util/mock/BUILD index 639cf3823f..22faa8e0fc 100644 --- a/util-mock/src/test/java/com/twitter/util/mock/BUILD +++ b/util-mock/src/test/java/com/twitter/util/mock/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-mock/src/test/scala/com/twitter/util/mock/BUILD b/util-mock/src/test/scala/com/twitter/util/mock/BUILD index 5eef907b39..9680026e4c 100644 --- a/util-mock/src/test/scala/com/twitter/util/mock/BUILD +++ b/util-mock/src/test/scala/com/twitter/util/mock/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/BUILD b/util-reflect/src/test/java/com/twitter/util/reflect/BUILD index 4acac5497f..fb279e4d23 100644 --- a/util-reflect/src/test/java/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/java/com/twitter/util/reflect/BUILD @@ -1,6 +1,7 @@ java_library( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], ) diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD index 2f49348915..5a39dc55a7 100644 --- a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD +++ b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-registry/src/test/java/BUILD b/util-registry/src/test/java/BUILD index ea04e8fe8b..816cbab872 100644 --- a/util-registry/src/test/java/BUILD +++ b/util-registry/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index 825a2e1355..6237fbe1f0 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-routing/src/test/java/com/twitter/util/routing/BUILD b/util-routing/src/test/java/com/twitter/util/routing/BUILD index 654b453ac1..878f921bb0 100644 --- a/util-routing/src/test/java/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/java/com/twitter/util/routing/BUILD @@ -3,6 +3,7 @@ junit_tests( "*.java", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index cfe39b7d5b..da6197898e 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD index 3b46f0d23f..6333be0756 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD index 8d095944d5..621e2a799e 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD @@ -1,6 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD index 2997d293b2..1a9ce8496b 100644 --- a/util-security/src/test/java/com/twitter/util/security/BUILD +++ b/util-security/src/test/java/com/twitter/util/security/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD index 006ba93dff..cadc34321e 100644 --- a/util-security/src/test/scala/com/twitter/util/security/BUILD +++ b/util-security/src/test/scala/com/twitter/util/security/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD index 4b0fca6eb2..815cc1b44d 100644 --- a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD index bb13ae1692..72a6b2ff46 100644 --- a/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD +++ b/util-slf4j-api/src/test/scala/com/twitter/util/logging/BUILD @@ -5,6 +5,7 @@ scala_library( "*.java", "*.scala", ], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ @@ -28,6 +29,7 @@ scala_library( junit_tests( compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-stats/src/test/java/BUILD b/util-stats/src/test/java/BUILD index 3a247c28f1..ef15c458a8 100644 --- a/util-stats/src/test/java/BUILD +++ b/util-stats/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-stats/src/test/scala/BUILD b/util-stats/src/test/scala/BUILD index 7296b62097..5cb23a4da6 100644 --- a/util-stats/src/test/scala/BUILD +++ b/util-stats/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-test/src/main/scala/BUILD b/util-test/src/main/scala/BUILD index 096495e79a..5b48799dc3 100644 --- a/util-test/src/main/scala/BUILD +++ b/util-test/src/main/scala/BUILD @@ -1,6 +1,7 @@ scala_library( # This target exists only to act as an aggregator jar. sources = ["PantsWorkaround.scala"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-test", diff --git a/util-test/src/main/scala/com/twitter/logging/BUILD b/util-test/src/main/scala/com/twitter/logging/BUILD index 1b130090cb..8c33560d6c 100644 --- a/util-test/src/main/scala/com/twitter/logging/BUILD +++ b/util-test/src/main/scala/com/twitter/logging/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-test-logging", diff --git a/util-test/src/main/scala/com/twitter/util/testing/BUILD b/util-test/src/main/scala/com/twitter/util/testing/BUILD index ff7788586f..806019dbaf 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/main/scala/com/twitter/util/testing/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-test-testing", diff --git a/util-test/src/test/scala/com/twitter/util/testing/BUILD b/util-test/src/test/scala/com/twitter/util/testing/BUILD index 192e06ef14..5d09aa73e9 100644 --- a/util-test/src/test/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/test/scala/com/twitter/util/testing/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-thrift/src/test/java/com/twitter/util/BUILD b/util-thrift/src/test/java/com/twitter/util/BUILD index 53d6e32382..5f4374f0de 100644 --- a/util-thrift/src/test/java/com/twitter/util/BUILD +++ b/util-thrift/src/test/java/com/twitter/util/BUILD @@ -1,5 +1,6 @@ java_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-thrift/src/test/scala/com/twitter/util/BUILD b/util-thrift/src/test/scala/com/twitter/util/BUILD index faef2628f8..4b320f6440 100644 --- a/util-thrift/src/test/scala/com/twitter/util/BUILD +++ b/util-thrift/src/test/scala/com/twitter/util/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD index dd8f13c04c..72c37656a6 100644 --- a/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD +++ b/util-tunable/src/main/scala/com/twitter/util/tunable/linter/BUILD @@ -1,5 +1,6 @@ scala_library( name = "scala", + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ @@ -16,6 +17,7 @@ jvm_binary( name = "configuration-linter", main = "com.twitter.util.tunable.linter.ConfigurationLinter", compiler_option_sets = ["fatal_warnings"], + platform = "java8", tags = ["bazel-compatible"], dependencies = [ ":scala", diff --git a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD index faf49c68c9..629a404f97 100644 --- a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD index c5d87fc0fb..7c0e487859 100644 --- a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD +++ b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-validator/src/test/java/com/twitter/util/validation/BUILD b/util-validator/src/test/java/com/twitter/util/validation/BUILD index de6fae8ad0..d6a155c79a 100644 --- a/util-validator/src/test/java/com/twitter/util/validation/BUILD +++ b/util-validator/src/test/java/com/twitter/util/validation/BUILD @@ -4,6 +4,7 @@ java_library( "constraints/*.java", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD b/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD index fc358ab7b7..85128f9f56 100644 --- a/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD +++ b/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD @@ -1,5 +1,6 @@ junit_tests( sources = ["**/*.java"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-validator/src/test/scala/com/twitter/util/validation/BUILD b/util-validator/src/test/scala/com/twitter/util/validation/BUILD index b0671326b5..c9a121bb32 100644 --- a/util-validator/src/test/scala/com/twitter/util/validation/BUILD +++ b/util-validator/src/test/scala/com/twitter/util/validation/BUILD @@ -4,6 +4,7 @@ junit_tests( "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = ["bazel-compatible"], dependencies = [ diff --git a/util-zk-test/src/main/scala/com/twitter/zk/BUILD b/util-zk-test/src/main/scala/com/twitter/zk/BUILD index 8b51a37b97..2ce870403d 100644 --- a/util-zk-test/src/main/scala/com/twitter/zk/BUILD +++ b/util-zk-test/src/main/scala/com/twitter/zk/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-zk-test", diff --git a/util-zk-test/src/test/java/BUILD b/util-zk-test/src/test/java/BUILD index 475c98385b..07d960c355 100644 --- a/util-zk-test/src/test/java/BUILD +++ b/util-zk-test/src/test/java/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.java"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-zk-test/src/test/scala/BUILD b/util-zk-test/src/test/scala/BUILD index 286b63f686..cf92253e76 100644 --- a/util-zk-test/src/test/scala/BUILD +++ b/util-zk-test/src/test/scala/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-zk/src/main/scala/BUILD b/util-zk/src/main/scala/BUILD index 81f73e9c24..ac89a3d878 100644 --- a/util-zk/src/main/scala/BUILD +++ b/util-zk/src/main/scala/BUILD @@ -1,6 +1,7 @@ scala_library( # This target exists only to act as an aggregator jar. sources = ["PantsWorkaroundZk.scala"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-zk", diff --git a/util-zk/src/main/scala/com/twitter/zk/BUILD b/util-zk/src/main/scala/com/twitter/zk/BUILD index 855cff33d2..0041ab88b4 100644 --- a/util-zk/src/main/scala/com/twitter/zk/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-zk-zk", diff --git a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD index fadff9ef1c..0c78a36744 100644 --- a/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/main/scala/com/twitter/zk/coordination/BUILD @@ -1,5 +1,6 @@ scala_library( compiler_option_sets = ["fatal_warnings"], + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-zk-coordination", diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index ec09793534..b637fd1fee 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index 9e3c40a7c7..84a8c895a0 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -1,6 +1,7 @@ junit_tests( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], + platform = "java8", strict_deps = True, tags = [ "bazel-compatible", From d486a7dab7110ee05d24760e9d9531f75393254d Mon Sep 17 00:00:00 2001 From: Kyle Bahr Date: Wed, 20 Jul 2022 18:39:15 +0000 Subject: [PATCH 687/796] util: upgrade snakeyaml due to cve Signed-off-by: Kyle Bahr Problem/Solution: upgrade snakeyaml version to 1.28 Result Closes twitter/util#301 JIRA Issues: CSL-12126 Differential Revision: https://phabricator.twitter.biz/D930268 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1acbc79bf9..ed30d90c4e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,8 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +* util: Update snakeyaml to 1.28 ``PHAB_ID=D930268`` + Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ diff --git a/build.sbt b/build.sbt index 1d89283e90..e7d4b6c044 100644 --- a/build.sbt +++ b/build.sbt @@ -26,7 +26,7 @@ val caffeineLib = "com.github.ben-manes.caffeine" % "caffeine" % "2.9.3" val jsr305Lib = "com.google.code.findbugs" % "jsr305" % "2.0.1" val scalacheckLib = "org.scalacheck" %% "scalacheck" % "1.15.4" % "test" val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion -val snakeyaml = "org.yaml" % "snakeyaml" % "1.24" +val snakeyaml = "org.yaml" % "snakeyaml" % "1.28" def travisTestJavaOptions: Seq[String] = { // We have some custom configuration for the Travis environment From 0b11e3ab63486a1e1dee780c7769efc3e4d08afd Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Fri, 22 Jul 2022 02:20:16 +0000 Subject: [PATCH 688/796] [mockito] remove/normalize scalatestplus:mockito-3-3 Problem / Solution `3rdparty/jvm/org/scalatestplus:mockito-3-3` is an alias to `3rdparty/jvm/org/scalatestplus:mockito` (which itself a reference to `3rdparty/src/jvm/org/scalatestplus/mockito`). This confuses people since they are all the same. Continue to normalize the dependencies. JIRA Issues: DEAD-3 Differential Revision: https://phabricator.twitter.biz/D939691 --- util-app/src/test/scala/BUILD | 2 +- util-cache/src/test/scala/com/twitter/cache/BUILD | 2 +- util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD | 2 +- util-core/src/test/scala/com/twitter/concurrent/BUILD | 2 +- util-core/src/test/scala/com/twitter/io/BUILD | 2 +- util-core/src/test/scala/com/twitter/util/BUILD | 2 +- util-jvm/src/test/scala/BUILD | 2 +- util-lint/src/test/scala/com/twitter/util/lint/BUILD | 2 +- util-registry/src/test/scala/BUILD | 2 +- util-routing/src/test/scala/com/twitter/util/routing/BUILD | 2 +- util-zk/src/test/scala/com/twitter/zk/BUILD | 2 +- util-zk/src/test/scala/com/twitter/zk/coordination/BUILD | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/util-app/src/test/scala/BUILD b/util-app/src/test/scala/BUILD index 8443b57b73..e26138ec02 100644 --- a/util-app/src/test/scala/BUILD +++ b/util-app/src/test/scala/BUILD @@ -12,7 +12,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "util/util-app/src/main/scala", "util/util-app/src/test/resources", "util/util-core:util-core-util", diff --git a/util-cache/src/test/scala/com/twitter/cache/BUILD b/util-cache/src/test/scala/com/twitter/cache/BUILD index 7f096b893f..1f24e52bac 100644 --- a/util-cache/src/test/scala/com/twitter/cache/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/BUILD @@ -18,7 +18,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-core:util-core-util", diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index f027cebb74..49f71f6943 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -14,7 +14,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", diff --git a/util-core/src/test/scala/com/twitter/concurrent/BUILD b/util-core/src/test/scala/com/twitter/concurrent/BUILD index 692272da02..06206406c7 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BUILD +++ b/util-core/src/test/scala/com/twitter/concurrent/BUILD @@ -13,7 +13,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", diff --git a/util-core/src/test/scala/com/twitter/io/BUILD b/util-core/src/test/scala/com/twitter/io/BUILD index 8566ca088d..3e11414988 100644 --- a/util-core/src/test/scala/com/twitter/io/BUILD +++ b/util-core/src/test/scala/com/twitter/io/BUILD @@ -13,7 +13,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/concurrent", diff --git a/util-core/src/test/scala/com/twitter/util/BUILD b/util-core/src/test/scala/com/twitter/util/BUILD index a016ab64fe..7824c17828 100644 --- a/util-core/src/test/scala/com/twitter/util/BUILD +++ b/util-core/src/test/scala/com/twitter/util/BUILD @@ -13,7 +13,7 @@ junit_tests( "3rdparty/jvm/org/scalacheck", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "3rdparty/jvm/org/scalatestplus:scalacheck", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index ca92b48517..48b7cefce7 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -2,7 +2,7 @@ COMMON_DEPS = [ "3rdparty/jvm/junit", "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "util/util-app/src/main/scala", "util/util-core:util-core-util", "util/util-jvm/src/main/scala/com/twitter/jvm", diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index d509324519..0e8ab450cb 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -12,7 +12,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "util/util-lint/src/main/scala/com/twitter/util/lint", ], ) diff --git a/util-registry/src/test/scala/BUILD b/util-registry/src/test/scala/BUILD index 6237fbe1f0..bbf282c788 100644 --- a/util-registry/src/test/scala/BUILD +++ b/util-registry/src/test/scala/BUILD @@ -12,7 +12,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "util/util-registry/src/main/scala/com/twitter/util/registry", ], ) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD index da6197898e..639d02ca6e 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD @@ -12,7 +12,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "util/util-routing/src/main/scala/com/twitter/util/routing", "util/util-routing/src/test/scala/com/twitter/util/routing/dynamic", "util/util-routing/src/test/scala/com/twitter/util/routing/simple", diff --git a/util-zk/src/test/scala/com/twitter/zk/BUILD b/util-zk/src/test/scala/com/twitter/zk/BUILD index b637fd1fee..f705ddfd03 100644 --- a/util-zk/src/test/scala/com/twitter/zk/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/BUILD @@ -13,7 +13,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-logging/src/main/scala/com/twitter/logging", diff --git a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD index 84a8c895a0..828d368269 100644 --- a/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD +++ b/util-zk/src/test/scala/com/twitter/zk/coordination/BUILD @@ -13,7 +13,7 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito-3-3", + "3rdparty/jvm/org/scalatestplus:mockito", "util/util-core:util-core-util", "util/util-core/src/main/scala/com/twitter/conversions", "util/util-zk/src/main/scala/com/twitter/zk", From 8d06d68b477d011d912cb12264913e80c7d8b62b Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Tue, 26 Jul 2022 17:10:27 +0000 Subject: [PATCH 689/796] [util,scrooge,twitter-server,finagle,finatra]: Upgrade sbt to 1.7.1 Problem / Solution Upgrade SBT to 1.7.1 for CSL libraries and upgrade some plugins as well. JIRA Issues: CSL-12170 Differential Revision: https://phabricator.twitter.biz/D938280 --- .github/workflows/continuous-integration.yml | 2 +- project/build.properties | 2 +- project/plugins.sbt | 4 ++-- project/unidoc.sbt | 2 +- sbt | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 8a50a1be8a..eef7acf645 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -3,7 +3,7 @@ name: continuous integration env: JAVA_OPTS: "-Dsbt.log.noformat=true" TRAVIS: "true" # pretend we're TravisCI - SBT_VERSION: 1.6.1 + SBT_VERSION: 1.7.1 BUILD_VERSION: "v1" # bump this if builds are failing due to a bad cache defaults: diff --git a/project/build.properties b/project/build.properties index 3161d2146c..22af2628c4 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.6.1 +sbt.version=1.7.1 diff --git a/project/plugins.sbt b/project/plugins.sbt index d959da10ca..7afb102c77 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,6 +2,6 @@ resolvers += Classpaths.sbtPluginReleases resolvers += Resolver.sonatypeRepo("snapshots") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.1") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3") -addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.31") +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.10.1") diff --git a/project/unidoc.sbt b/project/unidoc.sbt index a394be19d8..56bff7630e 100644 --- a/project/unidoc.sbt +++ b/project/unidoc.sbt @@ -1 +1 @@ -addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.3") +addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0") diff --git a/sbt b/sbt index 1d62b30232..d08998b12d 100755 --- a/sbt +++ b/sbt @@ -2,9 +2,9 @@ set -eo pipefail -sbtver="1.6.1" +sbtver="1.7.1" sbtjar="sbt-launch-$sbtver.jar" -sbtsha128="b69ab7f114d09d859b5e349acdb05d32c9d49072" +sbtsha128="468efdd45baf58dbb575f9b4369c5234f8cd54ba" sbtrepo="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch" From 47b7acd8dfea8aa3c0267484d17a6961498d5dc7 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Tue, 26 Jul 2022 21:11:33 +0000 Subject: [PATCH 690/796] util-security: support scala 3 build for util-security Problem/Solution Upgrade util-security to scala 3 Signed-off-by: Jing Yan JIRA Issues: CSL-12128 Differential Revision: https://phabricator.twitter.biz/D935481 --- build.sbt | 5 ++--- .../twitter/util/security/X509CertificateDeserializer.scala | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.sbt b/build.sbt index e7d4b6c044..06c0aaa7db 100644 --- a/build.sbt +++ b/build.sbt @@ -603,14 +603,13 @@ lazy val utilSecurity = Project( id = "util-security", base = file("util-security") ).settings( - sharedSettings + sharedScala3EnabledSettings ).settings( name := "util-security", libraryDependencies ++= Seq( scalacheckLib, - "org.scalatestplus" %% "scalacheck-1-14" % "3.1.2.0" % "test", snakeyaml - ) + ) ++ scalatestScalacheckVersionedDep(scalaVersion.value) ).dependsOn(utilCore, utilLogging, utilSecurityTestCerts % "test") lazy val utilSecurityTestCerts = Project( diff --git a/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala b/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala index e208cb74c7..9b2b36f34d 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala @@ -3,7 +3,8 @@ package com.twitter.util.security import com.twitter.util.Try import java.io.ByteArrayInputStream -import java.security.cert.{CertificateFactory, X509Certificate} +import java.security.cert.CertificateFactory +import java.security.cert.X509Certificate /** * A helper object to deserialize PEM-encoded X.509 Certificates. @@ -15,7 +16,7 @@ import java.security.cert.{CertificateFactory, X509Certificate} */ object X509CertificateDeserializer { private[this] val MessageType: String = "CERTIFICATE" - private[this] val deserializeX509: Array[Byte] => X509Certificate = { certBytes: Array[Byte] => + private[this] val deserializeX509: Array[Byte] => X509Certificate = { (certBytes: Array[Byte]) => val certFactory = CertificateFactory.getInstance("X.509") val certificate = certFactory .generateCertificate(new ByteArrayInputStream(certBytes)) From dcc6232cc0e8a41c73244249ea21c47e44b8064e Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 26 Jul 2022 21:27:19 +0000 Subject: [PATCH 691/796] twitter-server: Use default HTTP implementation (HTTP2) for admin server Problem TwitterServer's admin HTTP server was forcing HTTP/1.1. Solution Use Finagle's default HTTP protocol version (happens to be HTTP/2 at the moment). JIRA Issues: CSL-12117 Differential Revision: https://phabricator.twitter.biz/D920963 --- .../src/main/scala/com/twitter/app/App.scala | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index d69d7314f1..cb4c916550 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -75,7 +75,7 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { /** Exit on error with the given Throwable */ protected def exitOnError(throwable: Throwable): Unit = { - throwable.printStackTrace() + throwable.printStackTrace(System.out) throwable match { case _: CloseException => // exception occurred while closing, do not attempt to close again @@ -104,7 +104,7 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { System.err.println(details) // if we have gotten here we may not yet have attempted to close, ensure we do. try { - Await.ready(close(), closeDeadline - Time.now) + Await.ready(close(), (closeDeadline + SecondChanceGrace).sinceNow) } catch { case NonFatal(exc) => exitOnError(newCloseException(Seq(exc))) @@ -211,6 +211,13 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { /** Minimum duration to allow for exits to be processed. */ final val MinGrace: Duration = 1.second + /** + * Each resource is given a "grace period" to close gracefully. If it's not able to close within + * the allowed time, we give a "second chance" and wait a little more before forcefully + * terminating the closing. + */ + private[this] final val SecondChanceGrace: Duration = 1.second + /** * Default amount of time to wait for shutdown. * This value is not used as a default if `close()` is called without parameters. It simply @@ -253,7 +260,7 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { // after `close()` completes and `closing` is satisfied close(closeDeadline) .transform { _ => closable.close(closeDeadline) } - .by(shutdownTimer, closeDeadline) + .by(shutdownTimer, closeDeadline + SecondChanceGrace) } else { // `close()` not yet called, safe to add it lastExits.add(closable) @@ -347,7 +354,7 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { closeDeadline = deadline.max(Time.now + MinGrace) Future .collectToTry(exits.asScala.toSeq.map(_.close(closeDeadline))) - .by(shutdownTimer, closeDeadline) + .by(shutdownTimer, closeDeadline + SecondChanceGrace) .transform { case Return(results) => observeFuture(CloseExitLast) { @@ -379,7 +386,7 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { ): Future[Unit] = { Future .collectToTry(lastExits.asScala.toSeq.map(_.close(deadline))) - .by(shutdownTimer, deadline) + .by(shutdownTimer, deadline + SecondChanceGrace) .transform { case Return(results) => val errors = (onExitResults ++ results).collect { case Throw(e) => e } @@ -449,11 +456,13 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { // which is the same result that would be returned by an `Await.result` on `this`. val closeFuture = close(defaultCloseGracePeriod) - // The deadline to 'close' is advisory; we enforce it here. - if (!suppressGracefulShutdownErrors) Await.result(closeFuture, closeDeadline - Time.now) + val waitForIt = (closeDeadline + SecondChanceGrace).sinceNow + + // No need to pass a timeout to Await, close(...) enforces the timeout for us. + if (!suppressGracefulShutdownErrors) Await.result(closeFuture, waitForIt) else { try { // even if we suppress shutdown errors, we give the resources time to close - Await.ready(closeFuture, closeDeadline - Time.now) + Await.ready(closeFuture, waitForIt) } catch { case e: TimeoutException => throw e // we want TimeoutExceptions to propagate From af6b401ce5a5f33d67f4af3668da63d91f1ae1c6 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Tue, 26 Jul 2022 23:05:30 +0000 Subject: [PATCH 692/796] Twitter-oss: Prepare OSS libraries for release 22.7 Problem We want to release the next versions of our Twitter OSS libraries 22.7 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. JIRA Issues: CSL-12182 Differential Revision: https://phabricator.twitter.biz/D942680 --- CHANGELOG.rst | 5 +++++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ed30d90c4e..803bc10b05 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.7.0 +------ + +No Changes + * util: Update snakeyaml to 1.28 ``PHAB_ID=D930268`` Breaking API Changes diff --git a/README.md b/README.md index f897f3d888..e8fbc17f61 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) @@ -40,7 +40,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "22.4.0" +val utilCore = "com.twitter" %% "util-core" % "22.7.0" ``` # Units diff --git a/build.sbt b/build.sbt index 06c0aaa7db..3171a9ab2c 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.5.0-SNAPSHOT" +val releaseVersion = "22.7.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.3" From 936ceec275ea8eb858c39b2138ef616c9da64075 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Wed, 27 Jul 2022 02:36:08 +0000 Subject: [PATCH 693/796] Twitter-oss: update OSS libraries post-release to 22.10.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 22.10.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D942842 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e8fbc17f61..fa2c09e7f1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 3171a9ab2c..b449bd5c10 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.7.0" +val releaseVersion = "22.10.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.3" From a9fc6b9fe1d81c11cd34a6e5f2262a55c965114d Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 28 Jul 2022 16:34:57 +0000 Subject: [PATCH 694/796] Revert "Twitter-oss: update OSS libraries post-release to 22.10.0-SNAPSHOT" This reverts commit 7915f823ec292ddd448489b79935691dcad756bf. Differential Revision: https://phabricator.twitter.biz/D943947 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fa2c09e7f1..e8fbc17f61 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index b449bd5c10..3171a9ab2c 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.10.0-SNAPSHOT" +val releaseVersion = "22.7.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.3" From 92766ca75fb7f1d167ec50d13e98cb2fd9f98fea Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Thu, 28 Jul 2022 16:38:43 +0000 Subject: [PATCH 695/796] Downgrade Scoverage plugin version Problem There is a conflict between scalatest and scoverage with the current version combination that is causing some of the builds to fail. Solution Let's just downgrade scoverage until we get a better solution. JIRA Issues: CSL-12207 Differential Revision: https://phabricator.twitter.biz/D943932 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 7afb102c77..264af56bf0 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,6 +2,6 @@ resolvers += Classpaths.sbtPluginReleases resolvers += Resolver.sonatypeRepo("snapshots") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.1") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3") addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.10.1") From 4b129a263b1680e9ef96a1d5a69f6be636c68363 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 28 Jul 2022 17:58:45 +0000 Subject: [PATCH 696/796] Twitter-oss: update OSS libraries post-release to 22.10.0-SNAPSHOT (revert of a revert) This reverts commit 04ecaac33a1bd2ddb9eede8e8264ede5af3fd161. Differential Revision: https://phabricator.twitter.biz/D944382 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e8fbc17f61..fa2c09e7f1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) diff --git a/build.sbt b/build.sbt index 3171a9ab2c..b449bd5c10 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.7.0" +val releaseVersion = "22.10.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.13.3" From b34c138d1844265fecb5ab570bca72313b05a03c Mon Sep 17 00:00:00 2001 From: Ian Bennett Date: Thu, 28 Jul 2022 21:15:37 +0000 Subject: [PATCH 697/796] util-core: introduce Future.fromCompletableFuture Problem/Solution There exists a way to convert from a `c.t.util.Future` to Java's `j.u.concurrent.CompletableFuture`, but no way to get from a `j.u.concurrent.CompletableFuture` to a `c.t.util.Future`. Let's create a shared utility that also properly propagates the `c.t.util.Future` interrupts. Differential Revision: https://phabricator.twitter.biz/D940686 --- CHANGELOG.rst | 11 +++++--- .../main/scala/com/twitter/util/Future.scala | 26 +++++++++++++++++++ .../scala/com/twitter/util/FutureTest.scala | 7 +++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 803bc10b05..b65d0f2c9b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,12 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -22.7.0 ------- +New Features +~~~~~~~~~~~~ -No Changes +* util-core: Introduce Future.fromCompletableFuture ``PHAB_ID=D940686`` -* util: Update snakeyaml to 1.28 ``PHAB_ID=D930268`` +22.7.0 +------ Breaking API Changes ~~~~~~~~~~~~~~~~~~~~ @@ -28,6 +29,8 @@ Runtime Behavior Changes * util-jackson: Deserialized case classes with validation on optional fields shouldn't throw an error. ``PHAB_ID=D915377`` +* util: Update snakeyaml to 1.28 ``PHAB_ID=D930268`` + 22.4.0 ------ diff --git a/util-core/src/main/scala/com/twitter/util/Future.scala b/util-core/src/main/scala/com/twitter/util/Future.scala index 82e02ae92b..fe334187c0 100644 --- a/util-core/src/main/scala/com/twitter/util/Future.scala +++ b/util-core/src/main/scala/com/twitter/util/Future.scala @@ -1436,6 +1436,32 @@ def join[%s](%s): Future[(%s)] = join(Seq(%s)).map { _ => (%s) }""".format( f.compose(it => it.toSeq)) ) } + + /** + * Convert a Java native [[CompletableFuture]] to a Twitter [[Future]]. + * + * @note The Twitter Future's cancellation will be propagated to the underlying [[CompletableFuture]] + * @see [[https://download.oracle.com/javase/6/docs/api/java/util/concurrent/Future.html#cancel(boolean)]] + */ + def fromCompletableFuture[T](completableFuture: CompletableFuture[T]): Future[T] = { + val p = new Promise[T] + + // ensure that we propagate cancellation/errors from the Promise to the CompletableFuture. + // without this propagation, work contained within the CompletableFuture chain will continue, + // despite the Promise completing in error. we expect the CompletableFuture chain to reflect + // that an error has occurred via cancellation and stop work that would otherwise be wasted. + p.setInterruptHandler { + case _ => completableFuture.cancel(true) + } + + // ensure the state of the CompletableFuture propagates back to the Promise on completion + completableFuture.whenComplete { + case (_, thrown: Throwable) if thrown != null => p.setException(thrown) + case (t, _) => p.setValue(t) + } + + p + } } /** diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index fce42e6358..b8c19a9b14 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -1744,6 +1744,13 @@ class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenProp new MkConst { def apply[A](r: Try[A]): Future[A] = new Promise(r) }) + test( + "Future.fromCompletableFuture", + new MkConst { + def apply[A](r: Try[A]): Future[A] = + Future.fromCompletableFuture(new Promise(r).toCompletableFuture[A]) + } + ) "Future.apply" should { "fail on NLRC" in { From 26bcb04a1ebf5bfeb072c2967aec1a16654ca244 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Thu, 28 Jul 2022 22:43:43 +0000 Subject: [PATCH 698/796] util/docs - Fix inline code blocks Problem Some of the `code::` blocks are not rendering, specifically anything using the `:emphasize-lines:` directive. Solution Use `code-block::` instead which should work properly. Result Code examples are rendered properly. Differential Revision: https://phabricator.twitter.biz/D945057 --- doc/src/sphinx/util-jackson/index.rst | 56 +++++++-------- doc/src/sphinx/util-validator/index.rst | 92 ++++++++++++------------- 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/doc/src/sphinx/util-jackson/index.rst b/doc/src/sphinx/util-jackson/index.rst index 1ec7207db6..9e54d0923a 100644 --- a/doc/src/sphinx/util-jackson/index.rst +++ b/doc/src/sphinx/util-jackson/index.rst @@ -55,21 +55,21 @@ Basic Usage Let's assume we have these two case classes: -.. code:: scala +.. code-block:: scala case class Bar(d: String) case class Foo(a: String, b: Int, c: Bar) To **serialize** a case class into a JSON string, use -.. code:: scala +.. code-block:: scala ScalaObjectMapper#writeValueAsString(any: Any): String // or JSON#write(any: Any) For example: -.. code:: scala +.. code-block:: scala :emphasize-lines: 10, 15, 30, 33, 38 Welcome to Scala 2.12.12 (JDK 64-Bit Server VM, Java 1.8.0_242). @@ -121,14 +121,14 @@ For example: To **deserialize** a JSON string into a case class, use -.. code:: scala +.. code-block:: scala ScalaObjectMapper#parse[T](s: String): T // or JSON#parse[T](s: String): T For example, assuming the same `Bar` and `Foo` case classes defined above: -.. code:: scala +.. code-block:: scala :emphasize-lines: 10, 21 Welcome to Scala 2.12.12 (JDK 64-Bit Server VM, Java 1.8.0_242). @@ -205,7 +205,7 @@ the `ScalaObject#Builder` to specify custom configuration. The companion object defines `apply` methods for creation of a |ScalaObjectMapper|_ configured with the defaults listed above: -.. code:: scala +.. code-block:: scala import com.twitter.util.jackson.ScalaObjectMapper import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper} @@ -229,7 +229,7 @@ Companion Object Wrappers The companion object also defines other methods to easily obtain some specifically configured |ScalaObjectMapper|_ which wraps an *already configured* Jackson `ObjectMapper`: -.. code:: scala +.. code-block:: scala import com.twitter.util.jackson.ScalaObjectMapper import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper} @@ -288,7 +288,7 @@ constructing a customized mapper. E.g., to set a `PropertyNamingStrategy` different than the default: -.. code:: scala +.. code-block:: scala :emphasize-lines: 3 val objectMapper: ScalaObjectMapper = @@ -298,7 +298,7 @@ E.g., to set a `PropertyNamingStrategy` different than the default: Or to set additional modules or configuration: -.. code:: scala +.. code-block:: scala :emphasize-lines: 4, 5, 6, 7 val objectMapper: ScalaObjectMapper = @@ -312,7 +312,7 @@ Or to set additional modules or configuration: You can also get a `camelCase`, `snake_case`, or even a YAML configured mapper. -.. code:: scala +.. code-block:: scala :emphasize-lines: 7, 15, 23 val camelCaseObjectMapper: ScalaObjectMapper = @@ -346,7 +346,7 @@ The |ScalaObjectMapper|_ is a thin wrapper around a configured Jackson |jackson- `com.fasterxml.jackson.module.scala.ScalaObjectMapper`, thus you can always access the underlying Jackson object mapper by calling `underlying`: -.. code:: scala +.. code-block:: scala :emphasize-lines: 7 import com.fasterxml.jackson.databind.ObjectMapper @@ -369,7 +369,7 @@ Add a Jackson Module to a |ScalaObjectMapper|_ Follow the steps to create a Jackson Module for the custom serializer or deserializer then register the module to the underlying Jackson mapper from the |ScalaObjectMapper|_ instance: -.. code:: scala +.. code-block:: scala :emphasize-lines: 13, 14, 15, 43, 44, 45, 46, 47, 48, 52, 53 import com.fasterxml.jackson.databind.JsonDeserializer @@ -464,7 +464,7 @@ For example, you can annotate a method on the companion object for the case clas factory for instantiation. Any static factory method to use for instantiation **MUST** be specified on the companion object for case class: -.. code:: scala +.. code-block:: scala :emphasize-lines: 4 case class MySimpleCaseClass(int: Int) @@ -476,7 +476,7 @@ on the companion object for case class: Or to specify a secondary constructor to use for case class instantiation: -.. code:: scala +.. code-block:: scala :emphasize-lines: 2 case class MyCaseClassWithMultipleConstructors(number1: Long, number2: Long, number3: Long) { @@ -494,7 +494,7 @@ Or to specify a secondary constructor to use for case class instantiation: To annotate the primary constructor (as the syntax can seem non-intuitive because the `()` is required): - .. code:: scala + .. code-block:: scala :emphasize-lines: 1 case class MyCaseClassWithMultipleConstructors @JsonCreator()(number1: Long, number2: Long, number3: Long) { @@ -530,7 +530,7 @@ the `TimeStringDeserializer`. Thus if you had a case class defined: -.. code:: scala +.. code-block:: scala :emphasize-lines: 7 import com.fasterxml.jackson.annotation.JsonFormat @@ -544,7 +544,7 @@ Thus if you had a case class defined: The following JSON: -.. code:: json +.. code-block:: json { "id": 42, @@ -555,7 +555,7 @@ The following JSON: Will always deserialize properly into the case class regardless of the pattern configured on the `ObjectMapper` or as the default of a contextualized deserializer: -.. code:: scala +.. code-block:: scala Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). Type in expressions for evaluation. Or try :help. @@ -632,7 +632,7 @@ for specifying field annotations during deserialization with the `case class des For example, to deserialize JSON into the following classes that are not yours to annotate: -.. code:: scala +.. code-block:: scala case class Point(x: Int, y: Int) { def area: Int = x * y @@ -643,7 +643,7 @@ For example, to deserialize JSON into the following classes that are not yours t However, you want to enforce field constraints with `validations <../util-validator/index.html>`_ during deserialization. You can define a `Mix-in`, -.. code:: scala +.. code-block:: scala import com.fasterxml.jackson.annotation.JsonIgnore import jakarta.validation.constraints.{Max, Min} @@ -659,7 +659,7 @@ Then register this `Mix-in` for the `Point` class type. There are several ways t Follow the steps to create a Jackson Module for the `Mix-in` then register the module to the underlying Jackson mapper from the |ScalaObjectMapper|_ instance: -.. code:: scala +.. code-block:: scala import com.fasterxml.jackson.databind.module.SimpleModule import com.twitter.util.jackson.ScalaObjectMapper @@ -675,7 +675,7 @@ underlying Jackson mapper from the |ScalaObjectMapper|_ instance: Or register the `Mix-in` for the class type directly on the underlying Jackson mapper (without a Jackson Module): -.. code:: scala +.. code-block:: scala import com.twitter.util.jackson.ScalaObjectMapper @@ -684,7 +684,7 @@ Or register the `Mix-in` for the class type directly on the underlying Jackson m Deserializing this JSON would then error with failed validations: -.. code:: json +.. code-block:: json { "points": [ @@ -693,7 +693,7 @@ Deserializing this JSON would then error with failed validations: ] } -.. code:: scala +.. code-block:: scala :emphasize-lines: 51, 52, 53 Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). @@ -810,7 +810,7 @@ than the `scala.util.parsing.json.JSON` utility. Users should prefer using a configured |ScalaObjectMapper|_ to perform validations in order to be able to properly handle validation exceptions. -.. code:: scala +.. code-block:: scala :emphasize-lines: 7, 13, 22, 25 Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). @@ -853,7 +853,7 @@ Similarly, there is also a utility for `YAML `__ serde operat the same methods as `c.t.util.jackson.JSON <#c-t-util-jackson-json>`__ using a default |ScalaObjectMapper|_ configured with a `YAMLFactory`. -.. code:: scala +.. code-block:: scala :emphasize-lines: 8, 9, 10, 11, 12, 18, 19, 20, 29 Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). @@ -919,7 +919,7 @@ populate the exception message. For example: -.. code:: scala +.. code-block:: scala :emphasize-lines: 13, 19, 54 Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). @@ -1020,7 +1020,7 @@ Normalization Both API methods accept a "normalize function" which is a function to apply on the *actual* to "normalize" any fields -- such as a timestamp -- before comparing to the *expected*. -.. code:: scala +.. code-block:: scala :emphasize-lines: 43, 44, 45, 46, 49, 52 Welcome to Scala 2.12.13 (JDK 64-Bit Server VM, Java 1.8.0_242). diff --git a/doc/src/sphinx/util-validator/index.rst b/doc/src/sphinx/util-validator/index.rst index a00d8e1595..6b497aae11 100644 --- a/doc/src/sphinx/util-validator/index.rst +++ b/doc/src/sphinx/util-validator/index.rst @@ -35,13 +35,13 @@ as our guide is largely based on this information. In order to use the `util-validator` add a dependency on the project, e.g. in `sbt `__: -.. code:: scala +.. code-block:: scala "com.twitter" %% "util-validator" % utilValidatorVersion or with `Maven `__: -.. code:: xml +.. code-block:: xml com.twitter @@ -57,7 +57,7 @@ Applying constraints Let's start with a basic example to see how to apply constraints. Let's assume you have a case class defined as such: -.. code:: scala +.. code-block:: scala package com.example @@ -84,7 +84,7 @@ Validating constraints To perform a validation of these constraints, you use an instance of the `ScalaValidator`. Let’s take a look at a test for validating `Car` instances: -.. code:: scala +.. code-block:: scala :emphasize-lines: 15,25,35,45 package com.example @@ -162,7 +162,7 @@ rather than returning a `Set[ConstraintViolation[_]`. Instead of calling `ScalaValidator#validate`, use `ScalaValidator#verify`: -.. code:: scala +.. code-block:: scala :emphasize-lines: 16,28,40,52 package com.example @@ -254,7 +254,7 @@ a setter, and if the field is annotated with `@BeanProperty`, a bean getter and Thus, if you had the following class definition: -.. code:: scala +.. code-block:: scala class C(@myAnnot var c: Int) @@ -268,7 +268,7 @@ If you wanted Scala to copy the annotation to the generated field, you would nee For example, using `class C` again from above: -.. code:: scala +.. code-block:: scala class C(@(myAnnot @field) var c: Int) @@ -285,7 +285,7 @@ Constructor param constraints Constraints can be expressed by annotating the constructor param of a case class. E.g., -.. code:: scala +.. code-block:: scala package com.example @@ -391,7 +391,7 @@ For that purpose the `@ValidPassengerCount` constraint is added on the class lev that constraint has access to the complete `Car` object, allowing to compare the numbers of seats and passengers. -.. code:: scala +.. code-block:: scala :emphasize-lines: 5 package com.example @@ -413,7 +413,7 @@ When a class implements a trait or extends an abstract class, all constraint ann on the super-type apply in the same manner as the constraints specified in the class itself. For example, if we had a trait, `Car` with a `RentalCar` implementation: -.. code:: scala +.. code-block:: scala trait Car { @NotEmpty def manufacturer: String @@ -508,7 +508,7 @@ The same would be true, if `Car` was not a trait but an abstract class implement also specifies any constraints on the `manufacturer` field, it would be evaluated in addition to the `@NotEmpty` constraint from the `Car` trait: -.. code:: bash +.. code-block:: bash :emphasize-lines: 7,8,22,23 scala> case class RentalCar(@Size(min = 2, max = 14) manufacturer: String, @NotEmpty rentalStation: String) extends Car @@ -552,7 +552,7 @@ The Jakarta Bean Validation API does not only allow to validate single case clas complete object graphs via cascaded validation. To do so, just annotate a field or property representing a reference to another case class with `@Valid`. -.. code:: scala +.. code-block:: scala case class Person(@NotEmpty name: String) @@ -576,7 +576,7 @@ Unlike constraints, object graph validation works for some container elements (t an `Iterable[_]`). That means the field of the container can be annotated with `@Valid`, which will cause each contained element to be validated when the parent object is validated. -.. code:: scala +.. code-block:: scala case class Person(@NotEmpty name: String) @@ -657,7 +657,7 @@ Obtaining a Validator instance The `ScalaValidator` has an `apply()` function which will return an instance of a `ScalaValidator` configured with defaults. -.. code:: scala +.. code-block:: scala val validator: ScalaValidator = ScalaValidator() @@ -666,7 +666,7 @@ customized validator. E.g., to configure a custom size for the descriptor cache: -.. code:: scala +.. code-block:: scala :emphasize-lines: 3 val validator: ScalaValidator = @@ -676,7 +676,7 @@ E.g., to configure a custom size for the descriptor cache: Or to add custom validations: -.. code:: scala +.. code-block:: scala :emphasize-lines: 10 import com.twitter.util.validation.cfg.ConstraintMapping @@ -693,7 +693,7 @@ Or to add custom validations: or to specify multiple constraint mappings: -.. code:: scala +.. code-block:: scala :emphasize-lines: 17 import com.twitter.util.validation.cfg.ConstraintMapping @@ -744,7 +744,7 @@ The topic of validation groups is discussed in detail in Hibernate's `Chapter 5, Use `validate` to perform validation of all constraints of a given case class. -.. code:: scala +.. code-block:: scala :emphasize-lines: 8 import jakarta.validation.ConstraintViolation @@ -769,7 +769,7 @@ one `ConstraintViolation[Car]` object. The `verify` method is the same as `validate` but instead of returning `Set[ConstraintViolation[T]]` the method throws a `ConstraintViolationException `__. -.. code:: scala +.. code-block:: scala :emphasize-lines: 9 import com.twitter.util.Try @@ -789,7 +789,7 @@ the method throws a `ConstraintViolationException `__. -.. code:: scala +.. code-block:: scala :emphasize-lines: 32,33,34,35 import com.twitter.util.validation.ScalaValidator @@ -1236,7 +1236,7 @@ to the `underlying Hibernate validator `__. -.. code:: scala +.. code-block:: scala :emphasize-lines: 27,28,29,30 import com.twitter.util.validation.ScalaValidator @@ -1281,7 +1281,7 @@ Validates return value constraints of the given method. This method directly pro Validates all constraints placed on the parameters of the given constructor. This method directly proxies to the `underlying Hibernate validator `__. -.. code:: scala +.. code-block:: scala :emphasize-lines: 27,28,29 package com.example @@ -1323,7 +1323,7 @@ proxies to the `underlying Hibernate validator `__. -.. code:: scala +.. code-block:: scala :emphasize-lines: 25,26,27 package com.example @@ -1414,7 +1414,7 @@ The constraint validator Having defined the annotation, we now need to create an implementation of a `ConstraintValidator `__, which is able to validate fields with a `@CheckCase` annotation. To do so, we implement the Jakarta Bean Validation interface `ConstraintValidator `__: -.. code:: scala +.. code-block:: scala package com.example.constraint @@ -1483,7 +1483,7 @@ The last piece is an error message which should be used when a `@CheckCase` cons You have a few options. You can specify the default error message directly within the `message()` attribute of the constraint annotation, e.g., -.. code:: java +.. code-block:: java String message() default "Case mode must be {value}"; @@ -1491,14 +1491,14 @@ However, if you want to be able to `internationalize `__, specifying the resource key as the `message()` attribute in the constraint annotation. -.. code:: java +.. code-block:: java String message() default "{com.example.constraint.CheckCase.message}"; Create a `properties `__ file, *ValidationMessages.properties* with the following contents: -.. code:: text +.. code-block:: text com.example.constraint.CheckCase.message=Case mode must be {value} @@ -1541,7 +1541,7 @@ By default, Expression Language support **is not enabled** for custom violations be necessary. If you add an expression variable via the `TwitterConstraintValidatorContext`, Expression Language support will be enabled when adding a constraint violation. -.. code:: scala +.. code-block:: scala :emphasize-lines: 25,26,27,28,29 import com.twitter.util.validation.constraintvalidation.TwitterConstraintValidatorContext @@ -1584,7 +1584,7 @@ Using the constraint To use the custom `@CheckCase` constraint, annotate a case class field: -.. code:: scala +.. code-block:: scala :emphasize-lines: 7 package com.example @@ -1598,7 +1598,7 @@ To use the custom `@CheckCase` constraint, annotate a case class field: During validation, a `licensePlate` field with an invalid case will now cause validation to fail. -.. code:: scala +.. code-block:: scala // invalid license plate val car: Car = Car("Morris", "dd-ab-123", 4) @@ -1619,7 +1619,7 @@ the entire case class instance. These constraints are defined in the same way as Here we'll see how to implement the `@ValidPassengerCount` annotation seen in an earlier example: -.. code:: scala +.. code-block:: scala :emphasize-lines: 6 package com.example @@ -1634,7 +1634,7 @@ Here we'll see how to implement the `@ValidPassengerCount` annotation seen in an First the annotation definition could be written as: -.. code:: java +.. code-block:: java package com.example.constraint; @@ -1653,13 +1653,13 @@ First the annotation definition could be written as: We define an error message in a *Validation.properties* file: -.. code:: text +.. code-block:: text com.example.constraint.ValidPassengerCount.message=invalid number of passengers Then the `ConstraintValidator` implementation which is specified to a `Car` type: -.. code:: scala +.. code-block:: scala package com.example @@ -1683,7 +1683,7 @@ access the complete object state to decide whether the given instance is valid o During validation, an instance which does not satisfy the constraint will fail the validation. -.. code:: scala +.. code-block:: scala package com.example @@ -1752,7 +1752,7 @@ In a case class, define a no-arg method that returns a `MethodValidationResult`. with `@MethodValidation`, optionally specifying validated fields of the case class for error-reporting. -.. code:: scala +.. code-block:: scala :emphasize-lines: 11,12 import com.twitter.util.validation.MethodValidation @@ -1798,7 +1798,7 @@ Best Practices `@MethodValidation`-annotated method should be able to be done eagerly and without side-effects. - Case classes with generic type params should be considered **not supported** for validation. E.g., - .. code:: scala + .. code-block:: scala case class GenericCaseClass[T](@NotEmpty @Valid data: T) From 63d2c36b5eb38445d832919aa81bcee79d38b20a Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Fri, 29 Jul 2022 18:38:40 +0000 Subject: [PATCH 699/796] Remove scoverage integration Problem We've run into more problems with scoverage integration. Solution Remove scoverage. It's a cool feature but effectively never utilized anyway, resulting in all pain no gain. JIRA Issues: CSL-12211 Differential Revision: https://phabricator.twitter.biz/D945589 --- .github/workflows/continuous-integration.yml | 6 +----- README.md | 1 - build.sbt | 3 --- project/plugins.sbt | 1 - 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index eef7acf645..8c34d0eafb 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -52,8 +52,4 @@ jobs: find ~/.sbt -name "*.lock" -delete fi - name: test - run: ${{ format('./sbt ++{0} clean coverage test coverageReport', matrix.scala) }} - - name: coverage - run: ${{ format('./sbt ++{0} coverageAggregate', matrix.scala) }} - - name: upload coverage - uses: codecov/codecov-action@v2 + run: ${{ format('./sbt ++{0} clean test', matrix.scala) }} diff --git a/README.md b/README.md index fa2c09e7f1..3f46da943f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Twitter Util [![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) -[![Codecov](https://codecov.io/gh/twitter/util/branch/develop/graph/badge.svg)](https://codecov.io/gh/twitter/util) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12) diff --git a/build.sbt b/build.sbt index b449bd5c10..c8ecd4bed1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,5 +1,3 @@ -import scoverage.ScoverageKeys - Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure @@ -141,7 +139,6 @@ val baseSettings = Seq( if (scalaVersion.value.startsWith("3")) Seq.empty else (Compile / doc / sources).value }, - ScoverageKeys.coverageHighlighting := true, resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", scalacOptions := Seq( diff --git a/project/plugins.sbt b/project/plugins.sbt index 264af56bf0..c6af984fd4 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,6 +2,5 @@ resolvers += Classpaths.sbtPluginReleases resolvers += Resolver.sonatypeRepo("snapshots") addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.1") addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3") addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.10.1") From efff66ac615b20b360cdf21ab07b8779687bcde1 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Fri, 5 Aug 2022 00:56:59 +0000 Subject: [PATCH 700/796] util-core: Use SortedSet in UpdatableVar Problem We can run into undesirable asymptotic behavior with the sorted List approach. Solution Just use a SortedSet which has better asymptotic behavior. JIRA Issues: IM-4160 Differential Revision: https://phabricator.twitter.biz/D949512 --- .../src/main/scala/com/twitter/util/Var.scala | 66 +++---------------- .../test/scala/com/twitter/util/VarTest.scala | 34 ---------- 2 files changed, 8 insertions(+), 92 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Var.scala b/util-core/src/main/scala/com/twitter/util/Var.scala index 643289fd49..984cbd42ab 100644 --- a/util-core/src/main/scala/com/twitter/util/Var.scala +++ b/util-core/src/main/scala/com/twitter/util/Var.scala @@ -1,11 +1,10 @@ package com.twitter.util import java.util.{List => JList} -import scala.annotation.tailrec import scala.collection.{Seq => AnySeq} import scala.collection.compat.immutable.ArraySeq +import scala.collection.immutable.SortedSet import scala.collection.mutable.Buffer -import scala.collection.mutable.ListBuffer import scala.jdk.CollectionConverters._ import scala.language.higherKinds @@ -475,11 +474,11 @@ object Var { private object UpdatableVar { import Var.Observer - final case class Party[T](observer: Observer[T], depth: Int, sequence: Long) { + private final class Party[T](val observer: Observer[T], val depth: Int, val sequence: Long) { @volatile var active: Boolean = true } - val partyOrder: Ordering[Party[_]] = new Ordering[Party[_]] { + private val partyOrder: Ordering[Party[_]] = new Ordering[Party[_]] { def compare(a: Party[_], b: Party[_]): Int = { val c1 = java.lang.Integer.compare(a.depth, b.depth) if (c1 != 0) return c1 @@ -497,13 +496,8 @@ private[util] class UpdatableVar[T](init: T) extends Var[T] with Updatable[T] wi @volatile private[this] var value = init private[this] var version = 0L private[this] var partySequence = 0L - // Exposed for testing. - // - // Parties is a sorted set. We implement it ourselves (out top of vanilla list) at the cost - // of O(n) insert/remove in order to save on memory footprint. Similar to Future's WaitQueue, - // there just a handful of observers registering to each Var so plain old linked list works - // the best. - @volatile private[util] var parties = List.empty[Party[T]] + @volatile private[this] var parties = + SortedSet.empty[Party[T]](partyOrder.asInstanceOf[Ordering[Party[T]]]) def apply(): T = value @@ -528,8 +522,8 @@ private[util] class UpdatableVar[T](init: T) extends Var[T] with Updatable[T] wi obs.claim(this) val (p, curValue, curVersion) = synchronized { - val party = Party(obs, depth, partySequence) - insertParty(party) + val party = new Party(obs, depth, partySequence) + parties = parties + party partySequence += 1 (party, value, version) } @@ -540,57 +534,13 @@ private[util] class UpdatableVar[T](init: T) extends Var[T] with Updatable[T] wi def close(deadline: Time): Future[Unit] = { p.active = false UpdatableVar.this.synchronized { - removeParty(p) + parties = parties - p } Future.Done } } } - // Must be called while synchronized on this. - def removeParty(p: Party[T]): Unit = { - @tailrec - def loop(current: List[Party[T]], result: ListBuffer[Party[T]]): List[Party[T]] = - current match { - case Nil => result.toList - case q :: next => - if (q eq p) (result ++= next).toList - else loop(next, result += q) - } - - parties = parties match { - case q :: next if q eq p => next - case ps => loop(ps, ListBuffer.empty) - } - } - - // Must be called while synchronized on this. - def insertParty(p: Party[T]): Unit = { - @tailrec - def loop(current: List[Party[T]], result: ListBuffer[Party[T]]): List[Party[T]] = - current match { - case Nil => (result += p).toList - case q :: next => - val order = partyOrder.compare(p, q) - if (order < 0) { - // p < q, insert here - result += p - (result ++= current).toList - } else if (order > 0) { - // p > q, the insert position is not found yet - loop(next, result += q) - } else { - // p == q, ignore p and return the original queue - parties - } - } - - parties = parties match { - case Nil => p :: Nil - case ps => loop(ps, ListBuffer.empty) - } - } - override def toString: String = "Var(" + value + ")@" + hashCode } diff --git a/util-core/src/test/scala/com/twitter/util/VarTest.scala b/util-core/src/test/scala/com/twitter/util/VarTest.scala index 01d6c1b487..3bf1c3796a 100644 --- a/util-core/src/test/scala/com/twitter/util/VarTest.scala +++ b/util-core/src/test/scala/com/twitter/util/VarTest.scala @@ -1,9 +1,7 @@ package com.twitter.util -import com.twitter.util.UpdatableVar.Party import java.util.concurrent.atomic.AtomicReference import org.scalacheck.Arbitrary.arbitrary -import org.scalacheck.Gen import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable import org.scalatest.funsuite.AnyFunSuite @@ -484,36 +482,4 @@ class VarTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { } } } - - test("Hand-rolled observers queue behaves exactly like SortedSet") { - implicit val anyOrder: Ordering[Party[Any]] = - UpdatableVar.partyOrder.asInstanceOf[Ordering[Party[Any]]] - - val arbParties = for { - n <- Gen.choose(1, 100) - depths <- Gen.listOfN(n, Gen.posNum[Int]) - seqs <- Gen.listOfN(n, Gen.posNum[Int]) - } yield { - depths.zip(seqs).map { - case (d, s) => Party[Any](null, d, s) - } - } - - forAll(arbParties) { parties => - var expected = scala.collection.immutable.SortedSet.empty[Party[Any]] - expected ++= parties - - val v = new UpdatableVar[Any](null) - parties.foreach(p => v.insertParty(p)) - - // test insert - assert(expected.toList == v.parties) - - expected -= parties.head - v.removeParty(parties.head) - - // test remove - assert(expected.toList == v.parties) - } - } } From 4a22431fb8cda69a9aee07818a91249b148c8938 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Wed, 10 Aug 2022 22:18:16 +0000 Subject: [PATCH 701/796] util-benchmark: Use bazel to run JMH benchmarks Problem / Solution Bazel can now run Scala JMH benchmarks. Let's leverage that. JIRA Issues: CSL-12233 Differential Revision: https://phabricator.twitter.biz/D952195 --- util-benchmark/BUILD | 6 ------ .../main/java/com/twitter/json/BUILD.bazel | 5 +++++ .../com/twitter/json/TestDemographic.java | 0 .../com/twitter/json/TestFormat.java | 0 .../src/main/scala/{BUILD => BUILD.bazel} | 17 +++------------ .../concurrent/SchedulerBenchmark.scala | 2 +- .../stats/CumulativeGaugeBenchmark.scala | 2 +- .../twitter/finagle/stats/StatBenchmark.scala | 10 ++++++--- .../scala/com/twitter/io/BufBenchmark.scala | 2 +- .../scala/com/twitter/io/PipeBenchmark.scala | 11 +++++++--- .../com/twitter/json/JacksonBenchmark.scala | 2 +- .../string/StringConcatenationBenchmark.scala | 2 +- .../com/twitter/util/DurationBenchmark.scala | 2 +- .../com/twitter/util/FutureBenchmark.scala | 2 +- .../com/twitter/util/MonitorBenchmark.scala | 6 ++++-- .../com/twitter/util/TimeBenchmark.scala | 2 +- .../com/twitter/util/TimerBenchmark.scala | 2 +- .../scala/com/twitter/util/TryBenchmark.scala | 2 +- .../com/twitter/util/TypesBenchmark.scala | 2 +- .../twitter/util/ValidationBenchmark.scala | 12 +++++++---- util-benchmark/src/test/scala/BUILD | 20 ------------------ .../util/ValidationBenchmarkTest.scala | 21 ------------------- 22 files changed, 46 insertions(+), 84 deletions(-) delete mode 100644 util-benchmark/BUILD create mode 100644 util-benchmark/src/main/java/com/twitter/json/BUILD.bazel rename util-benchmark/src/main/{scala => java}/com/twitter/json/TestDemographic.java (100%) rename util-benchmark/src/main/{scala => java}/com/twitter/json/TestFormat.java (100%) rename util-benchmark/src/main/scala/{BUILD => BUILD.bazel} (76%) delete mode 100644 util-benchmark/src/test/scala/BUILD delete mode 100644 util-benchmark/src/test/scala/com/twitter/util/ValidationBenchmarkTest.scala diff --git a/util-benchmark/BUILD b/util-benchmark/BUILD deleted file mode 100644 index bf846a7ed7..0000000000 --- a/util-benchmark/BUILD +++ /dev/null @@ -1,6 +0,0 @@ -target( - tags = ["bazel-compatible"], - dependencies = [ - "util/util-benchmark/src/main/scala", - ], -) diff --git a/util-benchmark/src/main/java/com/twitter/json/BUILD.bazel b/util-benchmark/src/main/java/com/twitter/json/BUILD.bazel new file mode 100644 index 0000000000..de1e74b94f --- /dev/null +++ b/util-benchmark/src/main/java/com/twitter/json/BUILD.bazel @@ -0,0 +1,5 @@ +java_library( + sources = [ + "**/*.java", + ], +) diff --git a/util-benchmark/src/main/scala/com/twitter/json/TestDemographic.java b/util-benchmark/src/main/java/com/twitter/json/TestDemographic.java similarity index 100% rename from util-benchmark/src/main/scala/com/twitter/json/TestDemographic.java rename to util-benchmark/src/main/java/com/twitter/json/TestDemographic.java diff --git a/util-benchmark/src/main/scala/com/twitter/json/TestFormat.java b/util-benchmark/src/main/java/com/twitter/json/TestFormat.java similarity index 100% rename from util-benchmark/src/main/scala/com/twitter/json/TestFormat.java rename to util-benchmark/src/main/java/com/twitter/json/TestFormat.java diff --git a/util-benchmark/src/main/scala/BUILD b/util-benchmark/src/main/scala/BUILD.bazel similarity index 76% rename from util-benchmark/src/main/scala/BUILD rename to util-benchmark/src/main/scala/BUILD.bazel index 195f362472..135fde91f3 100644 --- a/util-benchmark/src/main/scala/BUILD +++ b/util-benchmark/src/main/scala/BUILD.bazel @@ -1,17 +1,16 @@ -scala_library( +scala_benchmark_jmh( + name = "jmh", sources = [ - "**/*.java", "**/*.scala", ], compiler_option_sets = ["fatal_warnings"], platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], dependencies = [ "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/jakarta/validation:jakarta.validation-api", "3rdparty/jvm/org/openjdk/jmh:jmh-core", "3rdparty/jvm/org/scala-lang:scala-library", + "util/util-benchmark/src/main/java/com/twitter/json", "util/util-core:scala", "util/util-hashing/src/main/scala", "util/util-jackson/src/main/scala/com/twitter/util/jackson", @@ -21,13 +20,3 @@ scala_library( "util/util-validator/src/main/scala/com/twitter/util/validation", ], ) - -jvm_binary( - name = "jmh", - main = "org.openjdk.jmh.Main", - platform = "java8", - tags = ["bazel-compatible"], - dependencies = [ - ":scala", - ], -) diff --git a/util-benchmark/src/main/scala/com/twitter/concurrent/SchedulerBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/concurrent/SchedulerBenchmark.scala index c60e380fde..775c837060 100644 --- a/util-benchmark/src/main/scala/com/twitter/concurrent/SchedulerBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/concurrent/SchedulerBenchmark.scala @@ -5,7 +5,7 @@ import org.openjdk.jmh.annotations._ import org.openjdk.jmh.infra.Blackhole // Run this via: -// ./sbt 'project util-benchmark' 'jmh:run SchedulerBenchmark -i 10 -wi 5 -f 1 -t 8 -bm avgt -tu ns' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'SchedulerBenchmark' -i 10 -wi 5 -f 1 -t 8 -bm avgt -tu ns // // Notes: // - threads, -t, should be 2x number of logical cores. diff --git a/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala index 462532228a..bf16f546f4 100644 --- a/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/finagle/stats/CumulativeGaugeBenchmark.scala @@ -5,7 +5,7 @@ import java.util import java.util.concurrent.TimeUnit import org.openjdk.jmh.annotations._ -// ./sbt 'project util-benchmark' 'jmh:run CumulativeGaugeBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'CumulativeGaugeBenchmark' @State(Scope.Benchmark) class CumulativeGaugeBenchmark extends StdBenchAnnotations { import CumulativeGaugeBenchmark._ diff --git a/util-benchmark/src/main/scala/com/twitter/finagle/stats/StatBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/finagle/stats/StatBenchmark.scala index 405eeefb42..a48aa52478 100644 --- a/util-benchmark/src/main/scala/com/twitter/finagle/stats/StatBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/finagle/stats/StatBenchmark.scala @@ -1,10 +1,14 @@ package com.twitter.finagle.stats -import com.twitter.util.{Await, Future, StdBenchAnnotations} +import com.twitter.util.Await +import com.twitter.util.Future +import com.twitter.util.StdBenchAnnotations import java.util.concurrent.TimeUnit -import org.openjdk.jmh.annotations.{Benchmark, Scope, State} +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State -// ./sbt 'project util-benchmark' 'jmh:run StatBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'StatBenchmark' @State(Scope.Benchmark) class StatBenchmark extends StdBenchAnnotations { diff --git a/util-benchmark/src/main/scala/com/twitter/io/BufBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/io/BufBenchmark.scala index 3957c0c913..0ac4e3d445 100644 --- a/util-benchmark/src/main/scala/com/twitter/io/BufBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/io/BufBenchmark.scala @@ -6,7 +6,7 @@ import org.openjdk.jmh.annotations._ import org.openjdk.jmh.infra.Blackhole import scala.util.Random -// ./sbt 'project util-benchmark' 'jmh:run BufBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'BufBenchmark' @State(Scope.Benchmark) class BufBenchmark extends StdBenchAnnotations { diff --git a/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala index cd58eca4e1..27d0e7d68c 100644 --- a/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/io/PipeBenchmark.scala @@ -1,9 +1,14 @@ package com.twitter.io -import com.twitter.util.{Future, StdBenchAnnotations} -import org.openjdk.jmh.annotations.{Benchmark, Measurement, Scope, State, Warmup} +import com.twitter.util.Future +import com.twitter.util.StdBenchAnnotations +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.Warmup -// sbt 'project util-benchmark' 'jmh:run PipeBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'PipeBenchmark' @State(Scope.Benchmark) @Warmup(iterations = 2) @Measurement(iterations = 2) diff --git a/util-benchmark/src/main/scala/com/twitter/json/JacksonBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/json/JacksonBenchmark.scala index 4bc32a2212..a2d956b934 100644 --- a/util-benchmark/src/main/scala/com/twitter/json/JacksonBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/json/JacksonBenchmark.scala @@ -24,7 +24,7 @@ object JacksonBenchmark { } -// ./sbt 'project util-benchmark' 'jmh:run JacksonBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'JacksonBenchmark' @State(Scope.Benchmark) class JacksonBenchmark extends StdBenchAnnotations { import JacksonBenchmark._ diff --git a/util-benchmark/src/main/scala/com/twitter/string/StringConcatenationBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/string/StringConcatenationBenchmark.scala index 1b3599a4f0..e97359859b 100644 --- a/util-benchmark/src/main/scala/com/twitter/string/StringConcatenationBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/string/StringConcatenationBenchmark.scala @@ -4,7 +4,7 @@ import com.twitter.util.StdBenchAnnotations import org.openjdk.jmh.annotations._ import scala.util.Random -// ./sbt 'project util-benchmark' 'jmh:run StringConcatenationBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'StringConcatenationBenchmark' @State(Scope.Benchmark) class StringConcatenationBenchmark extends StdBenchAnnotations { diff --git a/util-benchmark/src/main/scala/com/twitter/util/DurationBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/DurationBenchmark.scala index 70e461de18..c27bd5886d 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/DurationBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/DurationBenchmark.scala @@ -2,7 +2,7 @@ package com.twitter.util import org.openjdk.jmh.annotations._ -// ./sbt 'project util-benchmark' 'jmh:run DurationBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'DurationBenchmark' @State(Scope.Benchmark) class DurationBenchmark extends StdBenchAnnotations { diff --git a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala index 65ebf75e35..8c6ffcf945 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/FutureBenchmark.scala @@ -3,7 +3,7 @@ package com.twitter.util import com.twitter.concurrent.Offer import org.openjdk.jmh.annotations._ -// ./sbt 'project util-benchmark' 'jmh:run FutureBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'FutureBenchmark' class FutureBenchmark extends StdBenchAnnotations { import FutureBenchmark._ diff --git a/util-benchmark/src/main/scala/com/twitter/util/MonitorBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/MonitorBenchmark.scala index f04b6c8ae6..7fd71a1061 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/MonitorBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/MonitorBenchmark.scala @@ -1,9 +1,11 @@ package com.twitter.util -import org.openjdk.jmh.annotations.{Benchmark, Scope, State} +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State import scala.util.control.NoStackTrace -// ./sbt 'project util-benchmark' 'jmh:run MonitorBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'MonitorBenchmark' @State(Scope.Benchmark) class MonitorBenchmark extends StdBenchAnnotations { diff --git a/util-benchmark/src/main/scala/com/twitter/util/TimeBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/TimeBenchmark.scala index cc313f4eb6..d65d89b4de 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/TimeBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/TimeBenchmark.scala @@ -2,7 +2,7 @@ package com.twitter.util import org.openjdk.jmh.annotations._ -// ./sbt 'project util-benchmark' 'jmh:run TimeBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'TimeBenchmark' @State(Scope.Benchmark) class TimeBenchmark extends StdBenchAnnotations { diff --git a/util-benchmark/src/main/scala/com/twitter/util/TimerBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/TimerBenchmark.scala index 76f0cf3e18..f2490a520d 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/TimerBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/TimerBenchmark.scala @@ -9,7 +9,7 @@ import org.openjdk.jmh.annotations._ * performance of future scheduling (particularly because this * does not wait for the tasks to be executed). */ -// ./sbt 'project util-benchmark' 'jmh:run TimerBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'TimerBenchmark' @State(Scope.Benchmark) @Warmup(batchSize = 250) @Measurement(batchSize = 1000) diff --git a/util-benchmark/src/main/scala/com/twitter/util/TryBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/TryBenchmark.scala index 339c444fb1..62472f16a0 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/TryBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/TryBenchmark.scala @@ -2,7 +2,7 @@ package com.twitter.util import org.openjdk.jmh.annotations._ -// ./sbt 'project util-benchmark' 'jmh:run TryBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'TryBenchmark' @State(Scope.Benchmark) class TryBenchmark extends StdBenchAnnotations { diff --git a/util-benchmark/src/main/scala/com/twitter/util/TypesBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/TypesBenchmark.scala index 515cba38f3..69170913fc 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/TypesBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/TypesBenchmark.scala @@ -13,7 +13,7 @@ private object TypesBenchmark { case class IsCaseClass(a: String, b: String) } -// ./sbt 'project util-benchmark' 'jmh:run TypesBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'TypesBenchmark' @State(Scope.Benchmark) class TypesBenchmark extends StdBenchAnnotations { import TypesBenchmark._ diff --git a/util-benchmark/src/main/scala/com/twitter/util/ValidationBenchmark.scala b/util-benchmark/src/main/scala/com/twitter/util/ValidationBenchmark.scala index 7f8f27143e..70c51004ad 100644 --- a/util-benchmark/src/main/scala/com/twitter/util/ValidationBenchmark.scala +++ b/util-benchmark/src/main/scala/com/twitter/util/ValidationBenchmark.scala @@ -1,10 +1,14 @@ package com.twitter.util -import com.twitter.util.validation.{MethodValidation, ScalaValidator} +import com.twitter.util.validation.MethodValidation +import com.twitter.util.validation.ScalaValidator import com.twitter.util.validation.engine.MethodValidationResult import jakarta.validation.ValidationException -import jakarta.validation.constraints.{Min, NotEmpty} -import org.openjdk.jmh.annotations.{Benchmark, Scope, State} +import jakarta.validation.constraints.Min +import jakarta.validation.constraints.NotEmpty +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State import scala.util.control.NonFatal private object ValidationBenchmark { @@ -19,7 +23,7 @@ private object ValidationBenchmark { } } -// ./sbt 'project util-benchmark' 'jmh:run ValidationBenchmark' +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'ValidationBenchmark' @State(Scope.Benchmark) class ValidationBenchmark extends StdBenchAnnotations { import ValidationBenchmark._ diff --git a/util-benchmark/src/test/scala/BUILD b/util-benchmark/src/test/scala/BUILD deleted file mode 100644 index d57272063d..0000000000 --- a/util-benchmark/src/test/scala/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -junit_tests( - sources = [ - "**/*.scala", - ], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/org/scalacheck", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:scalacheck", - scoped( - "3rdparty/jvm/org/slf4j:slf4j-simple", - scope = "runtime", - ), - "util/util-benchmark/src/main/scala", - ], -) diff --git a/util-benchmark/src/test/scala/com/twitter/util/ValidationBenchmarkTest.scala b/util-benchmark/src/test/scala/com/twitter/util/ValidationBenchmarkTest.scala deleted file mode 100644 index 41bb2b979f..0000000000 --- a/util-benchmark/src/test/scala/com/twitter/util/ValidationBenchmarkTest.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.twitter.util - -import org.junit.runner.RunWith -import org.scalatest.funsuite.AnyFunSuite -import org.scalatestplus.junit.JUnitRunner - -@RunWith(classOf[JUnitRunner]) -class ValidationBenchmarkTest extends AnyFunSuite { - private[this] val benchmark = new ValidationBenchmark() - - test("validate case class") { - benchmark.withValidUser() - benchmark.withInvalidUser() - } - - test("validate nested case class") { - benchmark.withNestedValidUser() - benchmark.withNestedInvalidUser() - benchmark.withNestedDuplicateUser() - } -} From 0da7550217c1c21588903c03f33e48566b6c7e82 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Fri, 12 Aug 2022 23:08:41 +0000 Subject: [PATCH 702/796] util-core: export util from util-conversions Problem util-core/.../conversions exposes the types of util-core and thus all users of the conversions target much depend on util-core. This isn't exposed in the build graph. Solution Expose this in the build graph Differential Revision: https://phabricator.twitter.biz/D955468 --- util-core/src/main/scala/com/twitter/conversions/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util-core/src/main/scala/com/twitter/conversions/BUILD b/util-core/src/main/scala/com/twitter/conversions/BUILD index e09dad9d2b..d45b717d2e 100644 --- a/util-core/src/main/scala/com/twitter/conversions/BUILD +++ b/util-core/src/main/scala/com/twitter/conversions/BUILD @@ -11,4 +11,7 @@ scala_library( dependencies = [ "util/util-core:util-core-util", ], + exports = [ + "util/util-core:util-core-util", + ], ) From db9d127cde653a5f3e7d45c752956cbfdbd6f0c3 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Sat, 13 Aug 2022 20:16:35 +0000 Subject: [PATCH 703/796] finagle: Introduce configurable summary for histograms Problem We want to allow the developers to decide the verbosity of the histograms that they create with`StatsReceiver`. Solution Introduce a new param for `MetricsBuilder` to override the default format for histograms (`HistogramFormat`). A few formats are already supported, including `NoSummary` and `ShortSummary`, both of which will reduce the histogram footprint. The exporting subsystem (TwitterServer) already respects the new parameter. The default histogram format remains unchanged and is the same as `FullSummary`. That said, this commit doesn't introduce a runtime change. JIRA Issues: CSL-12237 Differential Revision: https://phabricator.twitter.biz/D908189 --- CHANGELOG.rst | 3 ++ .../twitter/finagle/stats/MetricBuilder.scala | 47 ++++++++++++++++++- .../stats/InMemoryStatsReceiverTest.scala | 6 +-- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b65d0f2c9b..18a73342d7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,9 @@ New Features * util-core: Introduce Future.fromCompletableFuture ``PHAB_ID=D940686`` +* util-stats: `MetricBuilder` now has API to configure a format in which histograms are exported. + Use `.withHistogramFormat` to access it. ``PHAB_ID=D908189`` + 22.7.0 ------ diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index ad87263623..9160ecf854 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -39,6 +39,23 @@ object SourceRole { def server(): SourceRole = Server } +/** + * A format for when histogram is exported. A few formats are supported: + * + * - `NoSummary`: only percentiles (p50, ..., p9999) are exported + * - `ShortSummary`: percentiles and `count`, and `sum` are exported + * - `FullSummary`: percentiles, `count`, `sum`, `avg`, `min`, and `max` are exported + * - `Default`: let the exporting sub-system (usually, TwitterServer) decide + */ +sealed abstract class HistogramFormat private (override val toString: String) + +object HistogramFormat { + case object Default extends HistogramFormat("default") + case object NoSummary extends HistogramFormat("no_summary") + case object ShortSummary extends HistogramFormat("short_summary") + case object FullSummary extends HistogramFormat("full_summary") +} + /** * finagle-stats has configurable scope separators. As this package is wrapped by finagle-stats, we * cannot retrieve it from finagle-stats. Consequently, we created this object so that the @@ -86,6 +103,7 @@ object MetricBuilder { name: Seq[String] = Seq.empty, relativeName: Seq[String] = Seq.empty, processPath: Option[String] = None, + histogramFormat: HistogramFormat = HistogramFormat.Default, percentiles: IndexedSeq[Double] = IndexedSeq.empty, isStandard: Boolean = false, metricType: MetricType, @@ -102,6 +120,7 @@ object MetricBuilder { Identity(name, name), relativeName, processPath, + histogramFormat, percentiles, isStandard, metricType @@ -120,6 +139,24 @@ object MetricBuilder { def newBuilder(metricType: MetricType): MetricBuilder = apply(metricType = metricType) + /** + * Construct a new `MetricBuilder` that will build counters. The result is the same as calling + * `newBuilder(CounterType)`. + */ + val forCounter: MetricBuilder = newBuilder(MetricBuilder.CounterType) + + /** + * Construct a new `MetricBuilder` that will build stats (histograms). The result is the same as + * calling `newBuilder(HistogramType)`. + */ + val forStat: MetricBuilder = newBuilder(MetricBuilder.HistogramType) + + /** + * Construct a new `MetricBuilder` that will build gauges. The result is the same as calling + * `newBuilder(GaugeType)`. + */ + val forGauge: MetricBuilder = newBuilder(MetricBuilder.GaugeType) + /** * Indicate the Metric type, [[CounterType]] will create a [[Counter]], * [[GaugeType]] will create a standard [[Gauge]], and [[HistogramType]] will create a [[Stat]]. @@ -207,6 +244,7 @@ class MetricBuilder private ( val relativeName: Seq[String], val processPath: Option[String], // Only persisted and relevant when building histograms. + val histogramFormat: HistogramFormat, val percentiles: IndexedSeq[Double], val isStandard: Boolean, val metricType: MetricType) @@ -228,6 +266,7 @@ class MetricBuilder private ( relativeName: Seq[String] = this.relativeName, processPath: Option[String] = this.processPath, isStandard: Boolean = this.isStandard, + histogramFormat: HistogramFormat = this.histogramFormat, percentiles: IndexedSeq[Double] = this.percentiles, metricType: MetricType = this.metricType ): MetricBuilder = { @@ -241,6 +280,7 @@ class MetricBuilder private ( identity = identity, relativeName = relativeName, processPath = processPath, + histogramFormat = histogramFormat, percentiles = percentiles, isStandard = isStandard, metricType = metricType @@ -304,6 +344,9 @@ class MetricBuilder private ( def withRelativeName(relativeName: String*): MetricBuilder = if (this.relativeName == Seq.empty) this.copy(relativeName = relativeName) else this + def withHistogramFormat(histogramFormat: HistogramFormat): MetricBuilder = + this.copy(histogramFormat = histogramFormat) + @varargs def withPercentiles(percentiles: Double*): MetricBuilder = this.copy(percentiles = percentiles.toIndexedSeq) @@ -360,6 +403,7 @@ class MetricBuilder private ( identity == that.identity && relativeName == that.relativeName && processPath == that.processPath && + histogramFormat == histogramFormat && percentiles == that.percentiles && metricType == that.metricType case _ => false @@ -376,6 +420,7 @@ class MetricBuilder private ( identity, relativeName, processPath, + histogramFormat, percentiles, metricType) @@ -383,6 +428,6 @@ class MetricBuilder private ( } override def toString(): String = { - s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $identity, $relativeName, $processPath, $percentiles, $metricType)" + s"MetricBuilder($keyIndicator, $description, $units, $role, $verbosity, $sourceClass, $identity, $relativeName, $processPath, $histogramFormat, $percentiles, $metricType)" } } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 4e40fd6b79..65ff2f2bdc 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -362,13 +362,13 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(parts.length == 3) assert( parts( - 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(coolGauge, coolGauge, Map(), true), List(), None, Vector(), GaugeType)") + 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(coolGauge, coolGauge, Map(), true), List(), None, default, Vector(), GaugeType)") assert( parts( - 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(rad/histo, histo, Map(), true), List(), None, Vector(), HistogramType)") + 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(rad/histo, histo, Map(), true), List(), None, default, Vector(), HistogramType)") assert( parts( - 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(sweet/counter, sweet_counter, Map(), true), List(), None, Vector(), CounterType)") + 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(sweet/counter, sweet_counter, Map(), true), List(), None, default, Vector(), CounterType)") } finally { ps.close() } From 1f4f5af5528700b76829f424c20f42511f068b36 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Mon, 15 Aug 2022 20:29:01 +0000 Subject: [PATCH 704/796] util-stats, twitter-server: Add a ShortLived verbosity level Problem Some metrics are not valuable for a long time and it would be nice if the backend metrics store could drop them early to avoid the cost of longer term storage. Solution Add a ShortLived metrics verbosity level that can signal to the metrics store through the metric metadata endpoint that this metrics is of limited value for long duration. Differential Revision: https://phabricator.twitter.biz/D954203 --- .../com/twitter/finagle/stats/StatsReceiver.scala | 13 +++++++++++-- .../finagle/stats/InMemoryStatsReceiverTest.scala | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index e088c8620f..01131f0bed 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -26,12 +26,21 @@ object Verbosity { /** * Indicates that a given metric is for standard operations. */ - val Default: Verbosity = new Verbosity("Verbosity(default)") + val Default: Verbosity = new Verbosity("default") /** * Indicates that a given metric may only be useful for a debugging/troubleshooting purposes. */ - val Debug: Verbosity = new Verbosity("Verbosity(debug)") + val Debug: Verbosity = new Verbosity("debug") + + /** + * Indicates that the metrics value is short lived and the backend metric store is free to + * drop it more eagerly than normal metrics. + * + * @note there is no guarantee as to how or if the metrics store will honor the designation + * and if not implementations should interpret it the same as `Default`. + */ + val ShortLived: Verbosity = new Verbosity("short_lived") } object StatsReceiver { diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 65ff2f2bdc..33cdb12893 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -362,13 +362,13 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(parts.length == 3) assert( parts( - 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(coolGauge, coolGauge, Map(), true), List(), None, default, Vector(), GaugeType)") + 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, default, None, Identity(coolGauge, coolGauge, Map(), true), List(), None, default, Vector(), GaugeType)") assert( parts( - 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(rad/histo, histo, Map(), true), List(), None, default, Vector(), HistogramType)") + 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, default, None, Identity(rad/histo, histo, Map(), true), List(), None, default, Vector(), HistogramType)") assert( parts( - 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, Verbosity(default), None, Identity(sweet/counter, sweet_counter, Map(), true), List(), None, default, Vector(), CounterType)") + 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, default, None, Identity(sweet/counter, sweet_counter, Map(), true), List(), None, default, Vector(), CounterType)") } finally { ps.close() } From bc2c08f5dfa0cc48f2c4dc3f96d7c228ac056e1a Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Thu, 18 Aug 2022 00:18:24 +0000 Subject: [PATCH 705/796] [Bazel migration] migrate 0 dependee /test/{scala,java} to bazel only JIRA Issues: PSEC-13747 Differential Revision: https://phabricator.twitter.biz/D954799 --- .../test/scala/com/twitter/app/lifecycle/{BUILD => BUILD.bazel} | 0 .../java/com/twitter/util/jackson/tests/{BUILD => BUILD.bazel} | 0 .../src/test/scala/com/twitter/util/mock/{BUILD => BUILD.bazel} | 0 .../test/scala/com/twitter/util/reflect/{BUILD => BUILD.bazel} | 0 .../src/test/java/com/twitter/util/routing/{BUILD => BUILD.bazel} | 0 .../test/scala/com/twitter/util/routing/{BUILD => BUILD.bazel} | 0 .../src/test/java/com/twitter/util/tunable/{BUILD => BUILD.bazel} | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/{BUILD => BUILD.bazel} (100%) rename util-jackson/src/test/java/com/twitter/util/jackson/tests/{BUILD => BUILD.bazel} (100%) rename util-mock/src/test/scala/com/twitter/util/mock/{BUILD => BUILD.bazel} (100%) rename util-reflect/src/test/scala/com/twitter/util/reflect/{BUILD => BUILD.bazel} (100%) rename util-routing/src/test/java/com/twitter/util/routing/{BUILD => BUILD.bazel} (100%) rename util-routing/src/test/scala/com/twitter/util/routing/{BUILD => BUILD.bazel} (100%) rename util-tunable/src/test/java/com/twitter/util/tunable/{BUILD => BUILD.bazel} (100%) diff --git a/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD b/util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD.bazel similarity index 100% rename from util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD rename to util-app-lifecycle/src/test/scala/com/twitter/app/lifecycle/BUILD.bazel diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD.bazel similarity index 100% rename from util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD rename to util-jackson/src/test/java/com/twitter/util/jackson/tests/BUILD.bazel diff --git a/util-mock/src/test/scala/com/twitter/util/mock/BUILD b/util-mock/src/test/scala/com/twitter/util/mock/BUILD.bazel similarity index 100% rename from util-mock/src/test/scala/com/twitter/util/mock/BUILD rename to util-mock/src/test/scala/com/twitter/util/mock/BUILD.bazel diff --git a/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD b/util-reflect/src/test/scala/com/twitter/util/reflect/BUILD.bazel similarity index 100% rename from util-reflect/src/test/scala/com/twitter/util/reflect/BUILD rename to util-reflect/src/test/scala/com/twitter/util/reflect/BUILD.bazel diff --git a/util-routing/src/test/java/com/twitter/util/routing/BUILD b/util-routing/src/test/java/com/twitter/util/routing/BUILD.bazel similarity index 100% rename from util-routing/src/test/java/com/twitter/util/routing/BUILD rename to util-routing/src/test/java/com/twitter/util/routing/BUILD.bazel diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel similarity index 100% rename from util-routing/src/test/scala/com/twitter/util/routing/BUILD rename to util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel diff --git a/util-tunable/src/test/java/com/twitter/util/tunable/BUILD b/util-tunable/src/test/java/com/twitter/util/tunable/BUILD.bazel similarity index 100% rename from util-tunable/src/test/java/com/twitter/util/tunable/BUILD rename to util-tunable/src/test/java/com/twitter/util/tunable/BUILD.bazel From 04463df934bc70611162b356040a5412c4fb4af0 Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Fri, 26 Aug 2022 02:53:05 +0000 Subject: [PATCH 706/796] [Bazel migration] migrate 0 dependee test targets to bazel only JIRA Issues: PSEC-13747 TBR=true Differential Revision: https://phabricator.twitter.biz/D961872 --- util-cache-guava/src/test/java/{BUILD => BUILD.bazel} | 0 util-cache-guava/src/test/scala/{BUILD => BUILD.bazel} | 0 .../src/test/scala/com/twitter/util/inject/{BUILD => BUILD.bazel} | 0 .../src/test/java/com/twitter/util/mock/{BUILD => BUILD.bazel} | 0 .../src/test/java/com/twitter/util/reflect/{BUILD => BUILD.bazel} | 0 .../scala/com/twitter/util/routing/dynamic/{BUILD => BUILD.bazel} | 0 .../scala/com/twitter/util/routing/simple/{BUILD => BUILD.bazel} | 0 util-security/src/test/java/{BUILD => BUILD.bazel} | 0 util-security/src/test/scala/{BUILD => BUILD.bazel} | 0 util-slf4j-api/src/test/java/{BUILD => BUILD.bazel} | 0 util-tunable/src/test/scala/{BUILD => BUILD.bazel} | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename util-cache-guava/src/test/java/{BUILD => BUILD.bazel} (100%) rename util-cache-guava/src/test/scala/{BUILD => BUILD.bazel} (100%) rename util-inject/src/test/scala/com/twitter/util/inject/{BUILD => BUILD.bazel} (100%) rename util-mock/src/test/java/com/twitter/util/mock/{BUILD => BUILD.bazel} (100%) rename util-reflect/src/test/java/com/twitter/util/reflect/{BUILD => BUILD.bazel} (100%) rename util-routing/src/test/scala/com/twitter/util/routing/dynamic/{BUILD => BUILD.bazel} (100%) rename util-routing/src/test/scala/com/twitter/util/routing/simple/{BUILD => BUILD.bazel} (100%) rename util-security/src/test/java/{BUILD => BUILD.bazel} (100%) rename util-security/src/test/scala/{BUILD => BUILD.bazel} (100%) rename util-slf4j-api/src/test/java/{BUILD => BUILD.bazel} (100%) rename util-tunable/src/test/scala/{BUILD => BUILD.bazel} (100%) diff --git a/util-cache-guava/src/test/java/BUILD b/util-cache-guava/src/test/java/BUILD.bazel similarity index 100% rename from util-cache-guava/src/test/java/BUILD rename to util-cache-guava/src/test/java/BUILD.bazel diff --git a/util-cache-guava/src/test/scala/BUILD b/util-cache-guava/src/test/scala/BUILD.bazel similarity index 100% rename from util-cache-guava/src/test/scala/BUILD rename to util-cache-guava/src/test/scala/BUILD.bazel diff --git a/util-inject/src/test/scala/com/twitter/util/inject/BUILD b/util-inject/src/test/scala/com/twitter/util/inject/BUILD.bazel similarity index 100% rename from util-inject/src/test/scala/com/twitter/util/inject/BUILD rename to util-inject/src/test/scala/com/twitter/util/inject/BUILD.bazel diff --git a/util-mock/src/test/java/com/twitter/util/mock/BUILD b/util-mock/src/test/java/com/twitter/util/mock/BUILD.bazel similarity index 100% rename from util-mock/src/test/java/com/twitter/util/mock/BUILD rename to util-mock/src/test/java/com/twitter/util/mock/BUILD.bazel diff --git a/util-reflect/src/test/java/com/twitter/util/reflect/BUILD b/util-reflect/src/test/java/com/twitter/util/reflect/BUILD.bazel similarity index 100% rename from util-reflect/src/test/java/com/twitter/util/reflect/BUILD rename to util-reflect/src/test/java/com/twitter/util/reflect/BUILD.bazel diff --git a/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD.bazel similarity index 100% rename from util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD rename to util-routing/src/test/scala/com/twitter/util/routing/dynamic/BUILD.bazel diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD.bazel similarity index 100% rename from util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD rename to util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD.bazel diff --git a/util-security/src/test/java/BUILD b/util-security/src/test/java/BUILD.bazel similarity index 100% rename from util-security/src/test/java/BUILD rename to util-security/src/test/java/BUILD.bazel diff --git a/util-security/src/test/scala/BUILD b/util-security/src/test/scala/BUILD.bazel similarity index 100% rename from util-security/src/test/scala/BUILD rename to util-security/src/test/scala/BUILD.bazel diff --git a/util-slf4j-api/src/test/java/BUILD b/util-slf4j-api/src/test/java/BUILD.bazel similarity index 100% rename from util-slf4j-api/src/test/java/BUILD rename to util-slf4j-api/src/test/java/BUILD.bazel diff --git a/util-tunable/src/test/scala/BUILD b/util-tunable/src/test/scala/BUILD.bazel similarity index 100% rename from util-tunable/src/test/scala/BUILD rename to util-tunable/src/test/scala/BUILD.bazel From d5de31f5c4fbf274f0d1d5be88b0496dd6639c05 Mon Sep 17 00:00:00 2001 From: Yufan Gong Date: Mon, 29 Aug 2022 18:28:59 +0000 Subject: [PATCH 707/796] finagle-http: Disable DeadlineFilter in HTTP servers Problem We found out DeadlineFilter and BadRequestHandler are created for every Http2 stream. Their field members are also created at the same frequency, such as counters. This waste some resources. In addition, DeadlineFilter is not working in the H2 server stack because it relies on a connection shared leaky bucket to reject, but the leaky bucket is created per stream. Solution Disable DeadlineFilter for H2 server. Make BadRequestHandler sharable. It needs the stack param as a parameter and then pass to all pipelines. Make the initServer partially applied so it is actually shared. Result We see H2 server throughput increased with the change. Differential Revision: https://phabricator.twitter.biz/D964479 --- .../finagle/stats/InMemoryStatsReceiver.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index e04e2f606b..a8163d93d3 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -83,6 +83,18 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { val expressions: mutable.Map[ExpressionSchemaKey, ExpressionSchema] = new ConcurrentHashMap[ExpressionSchemaKey, ExpressionSchema]().asScala + // duplicate metric name -> duplication times + val duplicatedMetrics: mutable.Map[Seq[String], Long] = + new ConcurrentHashMap[Seq[String], Long]().asScala + + private def recordDup(metricBuilder: MetricBuilder): Unit = { + if (!duplicatedMetrics.contains(metricBuilder.name)) { + duplicatedMetrics(metricBuilder.name) = 1 + } else { + duplicatedMetrics(metricBuilder.name) = duplicatedMetrics(metricBuilder.name) + 1 + } + } + override def counter(name: String*): ReadableCounter = { val metricBuilder = this .metricBuilder(CounterType) @@ -105,6 +117,8 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { if (!counters.contains(metricBuilder.name)) { counters(metricBuilder.name) = 0 schemas(metricBuilder.name) = metricBuilder + } else { + recordDup(metricBuilder) } } @@ -144,6 +158,8 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { if (!stats.contains(metricBuilder.name)) { stats(metricBuilder.name) = Nil schemas(metricBuilder.name) = metricBuilder + } else { + recordDup(metricBuilder) } } From fd5f3d5ffeb81b79bd5c059356b23f733e166f6a Mon Sep 17 00:00:00 2001 From: Helen Woldesenbet Date: Tue, 30 Aug 2022 05:09:57 +0000 Subject: [PATCH 708/796] util-jackson: move Validation tests from ScalaObjectMapperTest Problem/Solution We should move Validation tests out of ScalaObjectMapperTest, as the tests are 2000+ lines. JIRA Issues: CSL-12132 Differential Revision: https://phabricator.twitter.biz/D956753 --- .../util/jackson/ScalaObjectMapperTest.scala | 229 +--------------- .../ValidationScalaObjectMapperTest.scala | 244 ++++++++++++++++++ 2 files changed, 249 insertions(+), 224 deletions(-) create mode 100644 util-jackson/src/test/scala/com/twitter/util/jackson/ValidationScalaObjectMapperTest.scala diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala index 8f2d3ca5ec..ace3b4cbf6 100644 --- a/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ScalaObjectMapperTest.scala @@ -2,9 +2,6 @@ package com.twitter.util.jackson import com.fasterxml.jackson.core.JsonFactory import com.fasterxml.jackson.core.JsonParser -import com.fasterxml.jackson.databind.module.SimpleModule -import com.fasterxml.jackson.databind.node.IntNode -import com.fasterxml.jackson.databind.node.TreeTraversingParser import com.fasterxml.jackson.databind.DeserializationContext import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.JsonDeserializer @@ -12,6 +9,9 @@ import com.fasterxml.jackson.databind.JsonMappingException import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.MapperFeature import com.fasterxml.jackson.databind.PropertyNamingStrategies +import com.fasterxml.jackson.databind.module.SimpleModule +import com.fasterxml.jackson.databind.node.IntNode +import com.fasterxml.jackson.databind.node.TreeTraversingParser import com.fasterxml.jackson.databind.{ObjectMapper => JacksonObjectMapper} import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.scala.DefaultScalaModule @@ -28,12 +28,11 @@ import com.twitter.util.jackson.internal.SimplePersonInPackageObject import com.twitter.util.jackson.internal.SimplePersonInPackageObjectWithoutConstructorParams import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream -import java.time.format.DateTimeFormatter import java.time.LocalDateTime import java.time.ZoneId -import java.util.concurrent.TimeUnit +import java.time.format.DateTimeFormatter import java.util.TimeZone -import java.util.UUID +import java.util.concurrent.TimeUnit import org.junit.Assert.assertNotNull import org.junit.runner.RunWith import org.scalatest.BeforeAndAfterAll @@ -1205,42 +1204,6 @@ class ScalaObjectMapperTest } } - test("fail when CaseClassWithSeqWrappedValueLongWithValidation with null array element") { - intercept[CaseClassMappingException] { - parse[CaseClassWithSeqWrappedValueLongWithValidation]("""{"seq": [null]}""") - } - } - - test("fail when CaseClassWithSeqWrappedValueLongWithValidation with invalid field") { - intercept[CaseClassMappingException] { - parse[CaseClassWithSeqWrappedValueLongWithValidation]("""{"seq": [{"value": 0}]}""") - } - } - - test("fail when CaseClassWithSeqOfCaseClassWithValidation with null array element") { - intercept[CaseClassMappingException] { - parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [null]}""") - } - } - - test("fail when CaseClassWithSeqOfCaseClassWithValidation with invalid array element") { - intercept[CaseClassMappingException] { - parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [0]}""") - } - } - - test("fail when CaseClassWithSeqOfCaseClassWithValidation with null field in object") { - intercept[CaseClassMappingException] { - parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [{"value": null}]}""") - } - } - - test("fail when CaseClassWithSeqOfCaseClassWithValidation with invalid field in object") { - intercept[CaseClassMappingException] { - parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [{"value": 0}]}""") - } - } - test("fail when CaseClassWithArrayLong with null field in object") { intercept[CaseClassMappingException] { parse[CaseClassWithArrayLong]("""{"array": [null]}""") @@ -1358,53 +1321,6 @@ class ScalaObjectMapperTest parse[CaseClassWithOption]("""{"value":null}""") should be(CaseClassWithOption(None)) } - test( - "A case class with an Option[Int] member and validation annotation is parsable from a JSON object with value that passes the validation") { - parse[CaseClassWithOptionAndValidation]("""{ "towing_capacity": 10000 }""") should be( - CaseClassWithOptionAndValidation(Some(10000))) - } - - test( - "A case class with an Option[Int] member and validation annotation is parsable from a JSON object with null value") { - parse[CaseClassWithOptionAndValidation]("""{ "towing_capacity": null }""") should be( - CaseClassWithOptionAndValidation(None)) - } - - test( - "A case class with an Option[Int] member and validation annotation is parsable from a JSON without that field") { - parse[CaseClassWithOptionAndValidation]("""{}""") should be( - CaseClassWithOptionAndValidation(None)) - } - - test( - "A case class with an Option[Int] member and validation annotation is parsable from a JSON object with value that fails the validation") { - val e = intercept[CaseClassMappingException] { - parse[CaseClassWithOptionAndValidation]("""{ "towing_capacity": 1 }""") should be( - CaseClassWithOptionAndValidation(Some(1))) - } - e.errors.head.getMessage should be("towing_capacity: must be greater than or equal to 100") - } - - test( - "A case class with an Option[Boolean] member and incompatible validation annotation is not parsable from a JSON object") { - val e = intercept[jakarta.validation.UnexpectedTypeException] { - parse[CaseClassWithOptionAndIncompatibleValidation]("""{ "are_you": true }""") should be( - CaseClassWithOptionAndIncompatibleValidation(Some(true))) - } - } - - test( - "A case class with an Option[Boolean] member and incompatible validation annotation is parsable from a JSON object with null value") { - parse[CaseClassWithOptionAndIncompatibleValidation]("""{ "are_you": null }""") should be( - CaseClassWithOptionAndIncompatibleValidation(None)) - } - - test( - "A case class with an Option[Boolean] member and incompatible validation annotation is parsable from a JSON object without that field") { - parse[CaseClassWithOptionAndIncompatibleValidation]("""{}""") should be( - CaseClassWithOptionAndIncompatibleValidation(None)) - } - test("A case class with a JsonNode member generates a field of the given type") { generate(CaseClassWithJsonNode(new IntNode(2))) should be("""{"value":2}""") } @@ -1945,106 +1861,6 @@ class ScalaObjectMapperTest ) } - test("deserialization#JsonCreator with Validation") { - val e = intercept[CaseClassMappingException] { - // fails validation - parse[TestJsonCreatorWithValidation]("""{ "s": "" }""") - } - e.errors.size should equal(1) - e.errors.head.getMessage should be("s: must not be empty") - } - - test("deserialization#JsonCreator with Validation 1") { - // works with multiple validations - var e = intercept[CaseClassMappingException] { - // fails validation - parse[TestJsonCreatorWithValidations]("""{ "s": "" }""") - } - e.errors.size should equal(2) - // errors are alpha-sorted by message to be stable for testing - e.errors.head.getMessage should be("s: not one of [42, 137]") - e.errors.last.getMessage should be("s: must not be empty") - - e = intercept[CaseClassMappingException] { - // fails validation - parse[TestJsonCreatorWithValidations]("""{ "s": "99" }""") - } - e.errors.size should equal(1) - e.errors.head.getMessage should be("s: 99 not one of [42, 137]") - - parse[TestJsonCreatorWithValidations]("""{ "s": "42" }""") should equal( - TestJsonCreatorWithValidations("42")) - - parse[TestJsonCreatorWithValidations]("""{ "s": "137" }""") should equal( - TestJsonCreatorWithValidations(137)) - } - - test("deserialization#JsonCreator with Validation 2") { - val uuid = UUID.randomUUID().toString - - var e = intercept[CaseClassMappingException] { - // fails validation - parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( - s""" - |{ - | "number_as_string1" : "", - | "number_as_string2" : "20002", - | "third_argument" : "$uuid" - |} - |""".stripMargin - ) - } - e.errors.size should equal(1) - e.errors.head.getMessage should be("number_as_string1: must not be empty") - - e = intercept[CaseClassMappingException] { - // fails validation - parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( - s""" - |{ - | "number_as_string1" : "", - | "number_as_string2" : "65789", - | "third_argument" : "$uuid" - |} - |""".stripMargin - ) - } - e.errors.size should equal(2) - // errors are alpha-sorted by message to be stable for testing - e.errors.head.getMessage should be("number_as_string1: must not be empty") - e.errors.last.getMessage should be("number_as_string2: 65789 not one of [10001, 20002, 30003]") - - e = intercept[CaseClassMappingException] { - // fails validation - parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( - s""" - |{ - | "number_as_string1" : "", - | "number_as_string2" : "65789", - | "third_argument" : "foobar" - |} - |""".stripMargin - ) - } - e.errors.size should equal(3) - // errors are alpha-sorted by message to be stable for testing - e.errors.head.getMessage should be("number_as_string1: must not be empty") - e.errors(1).getMessage should be("number_as_string2: 65789 not one of [10001, 20002, 30003]") - e.errors(2).getMessage should be("third_argument: must be a valid UUID") - - parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( - s""" - |{ - | "number_as_string1" : "12345", - | "number_as_string2" : "20002", - | "third_argument" : "$uuid" - |} - |""".stripMargin - ) should equal( - CaseClassWithMultipleConstructorsAnnotatedAndValidations(12345L, 20002L, uuid) - ) - } - test("deserialization#JsonFormat with c.t.util.Time") { val timeString = "2018-09-14T23:20:08.000-07:00" val format = new com.twitter.util.TimeFormat( @@ -2183,41 +1999,6 @@ class ScalaObjectMapperTest generate(points) should be("""{"first":{"x":1,"y":1},"second":{"x":4,"y":5}}""".stripMargin) } - test("deserialization#mixin annotations with validations") { - val json = - """ - |{ - | "first": { "x": -1, "y": 120 }, - | "second": { "x": 4, "y": 5 } - |} - |""".stripMargin - - val e = intercept[CaseClassMappingException] { - parse[Points](json) - } - e.errors.size should equal(2) - e.errors.head.getMessage should be("first.x: must be greater than or equal to 0") - e.errors.head.reason.detail match { - case ValidationError(violation, ValidationError.Field, None) => - violation.getPropertyPath.toString should equal("Point.x") - violation.getMessage should equal("must be greater than or equal to 0") - violation.getInvalidValue should equal(-1) - violation.getRootBeanClass should equal(classOf[Point]) - violation.getRootBean == null should be(true) - case _ => fail() - } - e.errors.last.getMessage should be("first.y: must be less than or equal to 100") - e.errors.last.reason.detail match { - case ValidationError(violation, ValidationError.Field, None) => - violation.getPropertyPath.toString should equal("Point.y") - violation.getMessage should equal("must be less than or equal to 100") - violation.getInvalidValue should equal(120) - violation.getRootBeanClass should equal(classOf[Point]) - violation.getRootBean == null should be(true) - case _ => fail() - } - } - test("deserialization#ignore type with no default fails") { val json = """ diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/ValidationScalaObjectMapperTest.scala b/util-jackson/src/test/scala/com/twitter/util/jackson/ValidationScalaObjectMapperTest.scala new file mode 100644 index 0000000000..18eb547cea --- /dev/null +++ b/util-jackson/src/test/scala/com/twitter/util/jackson/ValidationScalaObjectMapperTest.scala @@ -0,0 +1,244 @@ +package com.twitter.util.jackson + +import com.twitter.util.jackson.caseclass.exceptions.CaseClassFieldMappingException.ValidationError +import com.twitter.util.jackson.caseclass.exceptions.CaseClassMappingException +import java.util.UUID +import org.junit.runner.RunWith +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class ValidationScalaObjectMapperTest + extends AnyFunSuite + with BeforeAndAfterAll + with Matchers + with ScalaObjectMapperFunctions { + + /* Class under test */ + protected val mapper: ScalaObjectMapper = ScalaObjectMapper() + + override def beforeAll(): Unit = { + super.beforeAll() + mapper.registerModule(ScalaObjectMapperTest.MixInAnnotationsModule) + } + + test("fail when CaseClassWithSeqWrappedValueLongWithValidation with null array element") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqWrappedValueLongWithValidation]("""{"seq": [null]}""") + } + } + + test("fail when CaseClassWithSeqWrappedValueLongWithValidation with invalid field") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqWrappedValueLongWithValidation]("""{"seq": [{"value": 0}]}""") + } + } + + test("fail when CaseClassWithSeqOfCaseClassWithValidation with null array element") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [null]}""") + } + } + + test("fail when CaseClassWithSeqOfCaseClassWithValidation with invalid array element") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [0]}""") + } + } + + test("fail when CaseClassWithSeqOfCaseClassWithValidation with null field in object") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [{"value": null}]}""") + } + } + + test("fail when CaseClassWithSeqOfCaseClassWithValidation with invalid field in object") { + intercept[CaseClassMappingException] { + parse[CaseClassWithSeqOfCaseClassWithValidation]("""{"seq": [{"value": 0}]}""") + } + } + + test( + "A case class with an Option[Int] member and validation annotation is parsable from a JSON object with value that passes the validation") { + parse[CaseClassWithOptionAndValidation]("""{ "towing_capacity": 10000 }""") should be( + CaseClassWithOptionAndValidation(Some(10000))) + } + + test( + "A case class with an Option[Int] member and validation annotation is parsable from a JSON object with null value") { + parse[CaseClassWithOptionAndValidation]("""{ "towing_capacity": null }""") should be( + CaseClassWithOptionAndValidation(None)) + } + + test( + "A case class with an Option[Int] member and validation annotation is parsable from a JSON without that field") { + parse[CaseClassWithOptionAndValidation]("""{}""") should be( + CaseClassWithOptionAndValidation(None)) + } + + test( + "A case class with an Option[Int] member and validation annotation is parsable from a JSON object with value that fails the validation") { + val e = intercept[CaseClassMappingException] { + parse[CaseClassWithOptionAndValidation]("""{ "towing_capacity": 1 }""") should be( + CaseClassWithOptionAndValidation(Some(1))) + } + e.errors.head.getMessage should be("towing_capacity: must be greater than or equal to 100") + } + + test( + "A case class with an Option[Boolean] member and incompatible validation annotation is not parsable from a JSON object") { + val e = intercept[jakarta.validation.UnexpectedTypeException] { + parse[CaseClassWithOptionAndIncompatibleValidation]("""{ "are_you": true }""") should be( + CaseClassWithOptionAndIncompatibleValidation(Some(true))) + } + } + + test( + "A case class with an Option[Boolean] member and incompatible validation annotation is parsable from a JSON object with null value") { + parse[CaseClassWithOptionAndIncompatibleValidation]("""{ "are_you": null }""") should be( + CaseClassWithOptionAndIncompatibleValidation(None)) + } + + test( + "A case class with an Option[Boolean] member and incompatible validation annotation is parsable from a JSON object without that field") { + parse[CaseClassWithOptionAndIncompatibleValidation]("""{}""") should be( + CaseClassWithOptionAndIncompatibleValidation(None)) + } + + test("deserialization#JsonCreator with Validation") { + val e = intercept[CaseClassMappingException] { + // fails validation + parse[TestJsonCreatorWithValidation]("""{ "s": "" }""") + } + e.errors.size should equal(1) + e.errors.head.getMessage should be("s: must not be empty") + } + + test("deserialization#JsonCreator with Validation 1") { + // works with multiple validations + var e = intercept[CaseClassMappingException] { + // fails validation + parse[TestJsonCreatorWithValidations]("""{ "s": "" }""") + } + e.errors.size should equal(2) + // errors are alpha-sorted by message to be stable for testing + e.errors.head.getMessage should be("s: not one of [42, 137]") + e.errors.last.getMessage should be("s: must not be empty") + + e = intercept[CaseClassMappingException] { + // fails validation + parse[TestJsonCreatorWithValidations]("""{ "s": "99" }""") + } + e.errors.size should equal(1) + e.errors.head.getMessage should be("s: 99 not one of [42, 137]") + + parse[TestJsonCreatorWithValidations]("""{ "s": "42" }""") should equal( + TestJsonCreatorWithValidations("42")) + + parse[TestJsonCreatorWithValidations]("""{ "s": "137" }""") should equal( + TestJsonCreatorWithValidations(137)) + } + + test("deserialization#JsonCreator with Validation 2") { + val uuid = UUID.randomUUID().toString + + var e = intercept[CaseClassMappingException] { + // fails validation + parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( + s""" + |{ + | "number_as_string1" : "", + | "number_as_string2" : "20002", + | "third_argument" : "$uuid" + |} + |""".stripMargin + ) + } + e.errors.size should equal(1) + e.errors.head.getMessage should be("number_as_string1: must not be empty") + + e = intercept[CaseClassMappingException] { + // fails validation + parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( + s""" + |{ + | "number_as_string1" : "", + | "number_as_string2" : "65789", + | "third_argument" : "$uuid" + |} + |""".stripMargin + ) + } + e.errors.size should equal(2) + // errors are alpha-sorted by message to be stable for testing + e.errors.head.getMessage should be("number_as_string1: must not be empty") + e.errors.last.getMessage should be("number_as_string2: 65789 not one of [10001, 20002, 30003]") + + e = intercept[CaseClassMappingException] { + // fails validation + parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( + s""" + |{ + | "number_as_string1" : "", + | "number_as_string2" : "65789", + | "third_argument" : "foobar" + |} + |""".stripMargin + ) + } + e.errors.size should equal(3) + // errors are alpha-sorted by message to be stable for testing + e.errors.head.getMessage should be("number_as_string1: must not be empty") + e.errors(1).getMessage should be("number_as_string2: 65789 not one of [10001, 20002, 30003]") + e.errors(2).getMessage should be("third_argument: must be a valid UUID") + + parse[CaseClassWithMultipleConstructorsAnnotatedAndValidations]( + s""" + |{ + | "number_as_string1" : "12345", + | "number_as_string2" : "20002", + | "third_argument" : "$uuid" + |} + |""".stripMargin + ) should equal( + CaseClassWithMultipleConstructorsAnnotatedAndValidations(12345L, 20002L, uuid) + ) + } + + test("deserialization#mixin annotations with validations") { + val json = + """ + |{ + | "first": { "x": -1, "y": 120 }, + | "second": { "x": 4, "y": 5 } + |} + |""".stripMargin + + val e = intercept[CaseClassMappingException] { + parse[Points](json) + } + e.errors.size should equal(2) + e.errors.head.getMessage should be("first.x: must be greater than or equal to 0") + e.errors.head.reason.detail match { + case ValidationError(violation, ValidationError.Field, None) => + violation.getPropertyPath.toString should equal("Point.x") + violation.getMessage should equal("must be greater than or equal to 0") + violation.getInvalidValue should equal(-1) + violation.getRootBeanClass should equal(classOf[Point]) + violation.getRootBean == null should be(true) + case _ => fail() + } + e.errors.last.getMessage should be("first.y: must be less than or equal to 100") + e.errors.last.reason.detail match { + case ValidationError(violation, ValidationError.Field, None) => + violation.getPropertyPath.toString should equal("Point.y") + violation.getMessage should equal("must be less than or equal to 100") + violation.getInvalidValue should equal(120) + violation.getRootBeanClass should equal(classOf[Point]) + violation.getRootBean == null should be(true) + case _ => fail() + } + } +} From 53c596dcdb756c6d7d0b2ecee2c1661391f8f71b Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Wed, 31 Aug 2022 06:10:59 +0000 Subject: [PATCH 709/796] [mockito] clean up false mockito deps Problem go/mockito-problem Solution cleab up some unused deps Differential Revision: https://phabricator.twitter.biz/D967035 --- util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD | 2 -- util-lint/src/test/scala/com/twitter/util/lint/BUILD | 2 -- util-logging/src/test/scala/BUILD | 1 - 3 files changed, 5 deletions(-) diff --git a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD index 49f71f6943..36a1d33fa3 100644 --- a/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD +++ b/util-cache/src/test/scala/com/twitter/cache/caffeine/BUILD @@ -11,10 +11,8 @@ junit_tests( "3rdparty/jvm/com/github/ben-manes/caffeine", "3rdparty/jvm/com/google/code/findbugs:jsr305", "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito", "util/util-cache/src/main/scala/com/twitter/cache", "util/util-cache/src/main/scala/com/twitter/cache/caffeine", "util/util-cache/src/test/scala/com/twitter/cache:abstract_tests", diff --git a/util-lint/src/test/scala/com/twitter/util/lint/BUILD b/util-lint/src/test/scala/com/twitter/util/lint/BUILD index 0e8ab450cb..bfc907c64e 100644 --- a/util-lint/src/test/scala/com/twitter/util/lint/BUILD +++ b/util-lint/src/test/scala/com/twitter/util/lint/BUILD @@ -9,10 +9,8 @@ junit_tests( ], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito", "util/util-lint/src/main/scala/com/twitter/util/lint", ], ) diff --git a/util-logging/src/test/scala/BUILD b/util-logging/src/test/scala/BUILD index 2220d14e76..07b37c9ff5 100644 --- a/util-logging/src/test/scala/BUILD +++ b/util-logging/src/test/scala/BUILD @@ -9,7 +9,6 @@ junit_tests( ], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "util/util-app/src/main/scala", From 106188b224542f5a4b7aad7e1fc395d606d2dbf7 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Thu, 1 Sep 2022 22:45:00 +0000 Subject: [PATCH 710/796] util-app: Use more helpful message for undefined flags Problem "flag undefined" is the very confusing error message. Solution Let's rewrite it to be less confusing. Differential Revision: https://phabricator.twitter.biz/D967923 --- util-app/src/main/scala/com/twitter/app/Flags.scala | 2 +- util-app/src/test/scala/com/twitter/app/FlagsTest.scala | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/Flags.scala b/util-app/src/main/scala/com/twitter/app/Flags.scala index b9a992a7b6..a2c7c32bc5 100644 --- a/util-app/src/main/scala/com/twitter/app/Flags.scala +++ b/util-app/src/main/scala/com/twitter/app/Flags.scala @@ -26,7 +26,7 @@ class FlagUndefinedException extends Exception(Flags.FlagUndefinedMessage) object Flags { private[app] val FlagValueRequiredMessage = "flag value is required" - private[app] val FlagUndefinedMessage = "flag undefined" + private[app] val FlagUndefinedMessage = "flag instance (c.t.app.Flag) is not defined" sealed trait FlagParseResult diff --git a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala index 59f41aa26a..cdb5ca95fe 100644 --- a/util-app/src/test/scala/com/twitter/app/FlagsTest.scala +++ b/util-app/src/test/scala/com/twitter/app/FlagsTest.scala @@ -1,6 +1,8 @@ package com.twitter.app -import com.twitter.util.registry.{Entry, GlobalRegistry, SimpleRegistry} +import com.twitter.util.registry.Entry +import com.twitter.util.registry.GlobalRegistry +import com.twitter.util.registry.SimpleRegistry import org.scalatest.funsuite.AnyFunSuite class FlagsTest extends AnyFunSuite { @@ -170,7 +172,7 @@ class FlagsTest extends AnyFunSuite { import ctx._ flag.parseArgs(Array("-undefined", "-foo", "blah")) match { case Flags.Error(reason) => - assert(reason.contains("Error parsing flag \"undefined\": flag undefined")) + assert(reason.contains("Error parsing flag \"undefined\": " + Flags.FlagUndefinedMessage)) assert(reason.contains("Error parsing flag \"foo\": For input string: \"blah\"")) assert(reason.contains("usage:")) case other => fail(s"expected a flag error, but received $other") From 2af68ac712408c08a192175a32f0894cfad42b12 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 7 Sep 2022 01:54:22 +0000 Subject: [PATCH 711/796] [2/N] Migrate top level scala_library targets to BUILD.bazel ### Problem / Solution 1. If a scala library is at the top of the build graph from Pants' perspective, then all its dependees have already migrated. 2. If the BUILD files only contain a scala library, then it would have no side effects migrating other targets in the same BUILD file. JIRA Issues: DPB-12362 Differential Revision: https://phabricator.twitter.biz/D972160 --- .../scala/com/twitter/util/inject/internal/{BUILD => BUILD.bazel} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename util-inject/src/main/scala/com/twitter/util/inject/internal/{BUILD => BUILD.bazel} (100%) diff --git a/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD b/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD.bazel similarity index 100% rename from util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD rename to util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD.bazel From a000510a74363d7dbca86514846728808c0c6a9d Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 7 Sep 2022 13:22:21 +0000 Subject: [PATCH 712/796] [3/N] Migrate top level scala_library targets to BUILD.bazel ### Problem / Solution 1. If a scala library is at the top of the build graph from Pants' perspective, then all its dependees have already migrated. 2. If the BUILD files only contain a scala library, then it would have no side effects migrating other targets in the same BUILD file. Differential Revision: https://phabricator.twitter.biz/D972301 --- .../src/main/scala/com/twitter/util/inject/{BUILD => BUILD.bazel} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename util-inject/src/main/scala/com/twitter/util/inject/{BUILD => BUILD.bazel} (100%) diff --git a/util-inject/src/main/scala/com/twitter/util/inject/BUILD b/util-inject/src/main/scala/com/twitter/util/inject/BUILD.bazel similarity index 100% rename from util-inject/src/main/scala/com/twitter/util/inject/BUILD rename to util-inject/src/main/scala/com/twitter/util/inject/BUILD.bazel From cd2b37404a5c472f481ca8125910f288939c3eeb Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Wed, 7 Sep 2022 20:51:15 +0000 Subject: [PATCH 713/796] [1/N] Migrate top level java_library targets to BUILD.bazel ### Problem / Solution 1. If a library is at the top of the build graph from Pants' perspective, then all its dependees have already migrated. 2. If the BUILD files only contain a library, then it would have no side effects migrating other targets in the same BUILD file. JIRA Issues: DPB-12362 Differential Revision: https://phabricator.twitter.biz/D972539 --- .../main/java/com/twitter/util/inject/flag/{BUILD => BUILD.bazel} | 0 .../src/test/java/com/twitter/util/inject/{BUILD => BUILD.bazel} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename util-inject/src/main/java/com/twitter/util/inject/flag/{BUILD => BUILD.bazel} (100%) rename util-inject/src/test/java/com/twitter/util/inject/{BUILD => BUILD.bazel} (100%) diff --git a/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD b/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD.bazel similarity index 100% rename from util-inject/src/main/java/com/twitter/util/inject/flag/BUILD rename to util-inject/src/main/java/com/twitter/util/inject/flag/BUILD.bazel diff --git a/util-inject/src/test/java/com/twitter/util/inject/BUILD b/util-inject/src/test/java/com/twitter/util/inject/BUILD.bazel similarity index 100% rename from util-inject/src/test/java/com/twitter/util/inject/BUILD rename to util-inject/src/test/java/com/twitter/util/inject/BUILD.bazel From cb79b788697c77b08a670a65076656bb8e6908b3 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 7 Sep 2022 22:34:43 +0000 Subject: [PATCH 714/796] finagle/util: Small performance improvements Problem / Solution Some small optimizations to eke a little more performance out of our security libs. JIRA Issues: CSL-12260 Differential Revision: https://phabricator.twitter.biz/D968665 --- .../util/security/ProxyX509Certificate.scala | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 util-security/src/main/scala/com/twitter/util/security/ProxyX509Certificate.scala diff --git a/util-security/src/main/scala/com/twitter/util/security/ProxyX509Certificate.scala b/util-security/src/main/scala/com/twitter/util/security/ProxyX509Certificate.scala new file mode 100644 index 0000000000..da5d4e6ba6 --- /dev/null +++ b/util-security/src/main/scala/com/twitter/util/security/ProxyX509Certificate.scala @@ -0,0 +1,39 @@ +package com.twitter.util.security + +import java.security.cert.X509Certificate + +/** + * X509Certificate that transparently proxies everything + */ +class ProxyX509Certificate(underlying: X509Certificate) extends X509Certificate { + def getEncoded(): Array[Byte] = underlying.getEncoded() + def getPublicKey(): java.security.PublicKey = underlying.getPublicKey() + def verify(key: java.security.PublicKey, sigProvider: String): Unit = + underlying.verify(key, sigProvider) + def verify(key: java.security.PublicKey): Unit = underlying.verify(key) + def checkValidity(date: java.util.Date): Unit = underlying.checkValidity(date) + def checkValidity(): Unit = underlying.checkValidity() + def getBasicConstraints(): Int = underlying.getBasicConstraints() + def getIssuerDN(): java.security.Principal = underlying.getIssuerDN() + def getIssuerUniqueID(): Array[Boolean] = underlying.getIssuerUniqueID() + def getKeyUsage(): Array[Boolean] = underlying.getKeyUsage() + def getNotAfter(): java.util.Date = underlying.getNotAfter() + def getNotBefore(): java.util.Date = underlying.getNotBefore() + def getSerialNumber(): java.math.BigInteger = underlying.getSerialNumber() + def getSigAlgName(): String = underlying.getSigAlgName() + def getSigAlgOID(): String = underlying.getSigAlgOID() + def getSigAlgParams(): Array[Byte] = underlying.getSigAlgParams() + def getSignature(): Array[Byte] = underlying.getSignature() + def getSubjectDN(): java.security.Principal = underlying.getSubjectDN() + def getSubjectUniqueID(): Array[Boolean] = underlying.getSubjectUniqueID() + def getTBSCertificate(): Array[Byte] = underlying.getTBSCertificate() + def getVersion(): Int = underlying.getVersion() + + def getCriticalExtensionOIDs(): java.util.Set[String] = underlying.getCriticalExtensionOIDs() + def getExtensionValue(x: String): Array[Byte] = underlying.getExtensionValue(x) + def getNonCriticalExtensionOIDs(): java.util.Set[String] = + underlying.getNonCriticalExtensionOIDs() + def hasUnsupportedCriticalExtension(): Boolean = underlying.hasUnsupportedCriticalExtension() + + override def toString: String = underlying.toString +} From bbb6d161a721dcba4e28c4a4a431d3945bb5409a Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Fri, 9 Sep 2022 13:12:37 +0000 Subject: [PATCH 715/796] [mockito] remove unused mockito deps Problem go/mockito-problem Solution remove unused `3rdparty/jvm/org/scalatestplus:mockito` JIRA Issues: DEAD-3 Differential Revision: https://phabricator.twitter.biz/D974031 --- util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel b/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel index 639d02ca6e..ceb954656e 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel @@ -12,7 +12,6 @@ junit_tests( "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:mockito", "util/util-routing/src/main/scala/com/twitter/util/routing", "util/util-routing/src/test/scala/com/twitter/util/routing/dynamic", "util/util-routing/src/test/scala/com/twitter/util/routing/simple", From 53a1981dde3014241075b34162ecd8a20c24da71 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Fri, 9 Sep 2022 16:09:10 +0000 Subject: [PATCH 716/796] [mockito] clean up 3rdparty/jvm/org/mockito:mockito-core JIRA Issues: DEAD-3 Differential Revision: https://phabricator.twitter.biz/D974046 --- util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel b/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel index ceb954656e..cdb7da747f 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel @@ -9,7 +9,6 @@ junit_tests( ], dependencies = [ "3rdparty/jvm/junit", - "3rdparty/jvm/org/mockito:mockito-core", "3rdparty/jvm/org/scalatest", "3rdparty/jvm/org/scalatestplus:junit", "util/util-routing/src/main/scala/com/twitter/util/routing", From cd576f9cee47474b34442f6c4d51f8b43f0ed437 Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Mon, 12 Sep 2022 22:35:43 +0000 Subject: [PATCH 717/796] [util] Submit continuations separately to the scheduler in Promise Problem: We submit a Promise's entire WaitQueue to the scheduler. Solution: Submit each continuation to the scheduler on its own. This could have subtle changes to the order at which continuations are executed, but we make no guarantees about ordering. This opens us up to further scheduler submission changes that assume continuations are submitted one at a time. JIRA Issues: CSL-12265 Differential Revision: https://phabricator.twitter.biz/D973092 --- .../src/main/scala/com/twitter/util/Promise.scala | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 1d6692caa4..4ce815d37f 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -92,19 +92,14 @@ object Promise { loop(this, WaitQueue.empty) } + @tailrec final def runInScheduler(t: Try[A]): Unit = { if (this ne WaitQueue.Empty) { - Scheduler.submit(() => WaitQueue.this.run(t)) + Scheduler.submit(() => first(t)) + rest.runInScheduler(t) } } - @tailrec - private def run(t: Try[A]): Unit = - if (this ne WaitQueue.Empty) { - first(t) - rest.run(t) - } - final override def toString: String = s"WaitQueue(size=$size)" } From 02a9fc44db3d3d3b32823d9cd51a4720c3f7ed26 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Tue, 13 Sep 2022 11:06:13 +0000 Subject: [PATCH 718/796] [mockito] clean 3rdparty/jvm/org/scalatestplus:mockito* Problem go/mockito-problem Solution Remove unused deps on `3rdparty/jvm/org/scalatestplus:mockito` and `3rdparty/jvm/org/scalatestplus:mockito-1-10` JIRA Issues: DEAD-3 Differential Revision: https://phabricator.twitter.biz/D974965 --- util-test/src/main/scala/com/twitter/util/testing/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/util-test/src/main/scala/com/twitter/util/testing/BUILD b/util-test/src/main/scala/com/twitter/util/testing/BUILD index 806019dbaf..7c7919813c 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/main/scala/com/twitter/util/testing/BUILD @@ -11,7 +11,6 @@ scala_library( dependencies = [ "3rdparty/jvm/org/mockito:mockito-all", "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:mockito-1-10", "util/util-stats/src/main/scala", "util/util-stats/src/main/scala/com/twitter/finagle/stats", ], From fb656b9e57248b52d14106d11ee350fc43146261 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Fri, 16 Sep 2022 01:47:55 +0000 Subject: [PATCH 719/796] [logging] ban new uses of util-logging Problem We want all new users to use slf4j (+logback) but there are ancient util/util-logging libraries available for use which do the wrong thing. Solution prevent new users of util/util-logging. JIRA Issues: DEAD-53, CSL-5387 Differential Revision: https://phabricator.twitter.biz/D977304 --- util-logging/BUILD | 5 ++++- util-logging/src/main/scala/BUILD | 5 ++++- .../src/main/scala/com/twitter/logging/BUILD | 15 ++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/util-logging/BUILD b/util-logging/BUILD index 7271a72ab1..ed171ba770 100644 --- a/util-logging/BUILD +++ b/util-logging/BUILD @@ -1,5 +1,8 @@ target( - tags = ["bazel-compatible"], + tags = [ + "bazel-compatible", + "visibility://codestructure/packages/util-logging:legal", + ], dependencies = [ "util/util-logging/src/main/scala", ], diff --git a/util-logging/src/main/scala/BUILD b/util-logging/src/main/scala/BUILD index 3f66d5dfc9..866e167235 100644 --- a/util-logging/src/main/scala/BUILD +++ b/util-logging/src/main/scala/BUILD @@ -1,5 +1,8 @@ target( - tags = ["bazel-compatible"], + tags = [ + "bazel-compatible", + "visibility://codestructure/packages/util-logging:legal", + ], dependencies = [ "util/util-logging/src/main/scala/com/twitter/logging", ], diff --git a/util-logging/src/main/scala/com/twitter/logging/BUILD b/util-logging/src/main/scala/com/twitter/logging/BUILD index 986f79252f..46262eb7de 100644 --- a/util-logging/src/main/scala/com/twitter/logging/BUILD +++ b/util-logging/src/main/scala/com/twitter/logging/BUILD @@ -1,4 +1,15 @@ +# workaround for https://github.com/bazelbuild/bazel/issues/3864 +alias( + name = "logging", + target = ":internal", + tags = [ + "bazel-compatible", + "visibility://codestructure/packages/util-logging:legal", + ], +) + scala_library( + name = "internal", sources = ["**/*.scala"], compiler_option_sets = ["fatal_warnings"], platform = "java8", @@ -8,7 +19,9 @@ scala_library( repo = artifactory, ), strict_deps = True, - tags = ["bazel-compatible"], + tags = [ + "bazel-compatible", + ], dependencies = [ "util/util-app/src/main/scala", "util/util-core:util-core-util", From 3174c84988964a979433c3f187020d60263c3ac4 Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Fri, 16 Sep 2022 02:02:38 +0000 Subject: [PATCH 720/796] [util-core] Move scheduler submission into Fiber stored in Local Problem: We want to introduce Fibers. Solution: Introduce the default Fiber implementation and hand off all scheduler submission to Fibers. The fibers themselves will be stored in the Local context. For now, there is no behavior change because the default Fiber only submits tasks to the Scheduler. JIRA Issues: CSL-12266 Differential Revision: https://phabricator.twitter.biz/D976677 --- .../scala/com/twitter/util/ConstFuture.scala | 69 +- .../main/scala/com/twitter/util/Fiber.scala | 30 + .../scala/com/twitter/util/FiberTask.scala | 19 + .../main/scala/com/twitter/util/Local.scala | 995 ++++++++++++++++-- .../main/scala/com/twitter/util/Promise.scala | 14 +- .../scala/com/twitter/util/FutureTest.scala | 2 +- .../com/twitter/util/WaitQueueTest.scala | 2 +- 7 files changed, 980 insertions(+), 151 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/util/Fiber.scala create mode 100644 util-core/src/main/scala/com/twitter/util/FiberTask.scala diff --git a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala index 8eeebba0ce..3659208df7 100644 --- a/util-core/src/main/scala/com/twitter/util/ConstFuture.scala +++ b/util-core/src/main/scala/com/twitter/util/ConstFuture.scala @@ -1,6 +1,5 @@ package com.twitter.util -import com.twitter.concurrent.Scheduler import scala.runtime.NonLocalReturnControl /** @@ -19,20 +18,18 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { // with `Promise`. def respond(f: Try[A] => Unit): Future[A] = { val saved = Local.save() - Scheduler.submit(new Runnable { - def run(): Unit = { - val current = Local.save() - if (current ne saved) Local.restore(saved) - - val tracker = saved.resourceTracker - val run = - if (tracker eq None) f - else ResourceTracker.wrapAndMeasureUsage(f, tracker.get) - - try run(result) - catch Monitor.catcher - finally Local.restore(current) - } + saved.fiber.submitTask(() => { + val current = Local.save() + if (current ne saved) Local.restore(saved) + + val tracker = saved.resourceTracker + val run = + if (tracker eq None) f + else ResourceTracker.wrapAndMeasureUsage(f, tracker.get) + + try run(result) + catch Monitor.catcher + finally Local.restore(current) }) this } @@ -57,28 +54,26 @@ class ConstFuture[A](result: Try[A]) extends Future[A] { val p = new Promise[B] // see the note on `respond` for an explanation of why `Scheduler` is used. val saved = Local.save() - Scheduler.submit(new Runnable { - def run(): Unit = { - val current = Local.save() - if (current ne saved) Local.restore(saved) - - val tracker = saved.resourceTracker - val run = - if (tracker eq None) f - else ResourceTracker.wrapAndMeasureUsage(f, tracker.get) - - val computed = - try run(result) - catch { - case e: NonLocalReturnControl[_] => Future.exception(new FutureNonLocalReturnControl(e)) - case scala.util.control.NonFatal(e) => Future.exception(e) - case t: Throwable => - Monitor.handle(t) - throw t - } finally Local.restore(current) - - p.become(computed) - } + saved.fiber.submitTask(() => { + val current = Local.save() + if (current ne saved) Local.restore(saved) + + val tracker = saved.resourceTracker + val run = + if (tracker eq None) f + else ResourceTracker.wrapAndMeasureUsage(f, tracker.get) + + val computed = + try run(result) + catch { + case e: NonLocalReturnControl[_] => Future.exception(new FutureNonLocalReturnControl(e)) + case scala.util.control.NonFatal(e) => Future.exception(e) + case t: Throwable => + Monitor.handle(t) + throw t + } finally Local.restore(current) + + p.become(computed) }) p } diff --git a/util-core/src/main/scala/com/twitter/util/Fiber.scala b/util-core/src/main/scala/com/twitter/util/Fiber.scala new file mode 100644 index 0000000000..dc235c7a76 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/Fiber.scala @@ -0,0 +1,30 @@ +package com.twitter.util + +import com.twitter.concurrent.Scheduler + +/** + * A Fiber receives runnable tasks and submits them to a Scheduler for execution. + */ +private[twitter] abstract class Fiber { + + /** Submit work to the Fiber */ + def submitTask(r: FiberTask): Unit +} + +private[twitter] object Fiber { + + // Global default fiber which solely submits tasks to the global Scheduler + val Global: Fiber = new Fiber { + override def submitTask(r: FiberTask): Unit = { + Scheduler.submit(r) + } + } + + def let[T](fiber: Fiber)(f: => T): T = { + val oldCtx = Local.save() + val newCtx = oldCtx.setFiber(fiber) + Local.restore(newCtx) + try f + finally Local.restore(oldCtx) + } +} diff --git a/util-core/src/main/scala/com/twitter/util/FiberTask.scala b/util-core/src/main/scala/com/twitter/util/FiberTask.scala new file mode 100644 index 0000000000..909a4537fc --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/FiberTask.scala @@ -0,0 +1,19 @@ +package com.twitter.util + +/** + * Work for a `Fiber` to do. + * + * This is indistinguishable from a `Runnable` other than it is an abstract class. + * We do this so that in the case of the JVM not being able to optimize away the + * dynamic invocation we get v-table invocations which are constant time instead + * of i-table invocations which require a linear search through the dynamic types + * interfaces. + */ +private[twitter] abstract class FiberTask extends Runnable { + // From the `Runnable` interface + final def run(): Unit = doRun() + + // make this an abstract class method so we get v-table performance + // when invoked directly. + def doRun(): Unit +} diff --git a/util-core/src/main/scala/com/twitter/util/Local.scala b/util-core/src/main/scala/com/twitter/util/Local.scala index 85e0ebcdc1..98d1f7bd21 100644 --- a/util-core/src/main/scala/com/twitter/util/Local.scala +++ b/util-core/src/main/scala/com/twitter/util/Local.scala @@ -8,10 +8,13 @@ object Local { * key - Local.Key, value - Option[_] */ sealed abstract class Context private ( - final private[util] val resourceTracker: Option[ResourceTracker]) { + final private[util] val resourceTracker: Option[ResourceTracker], + final private[util] val fiber: Fiber) { private[util] def setResourceTracker(tracker: ResourceTracker): Context private[util] def removeResourceTracker(): Context + private[util] def setFiber(f: Fiber): Context + private[util] def get(k: Key): Option[_] private[util] def remove(k: Key): Context private[util] def set(k: Key, v: Some[_]): Context @@ -24,33 +27,40 @@ object Local { */ val empty: Context = EmptyContext - private object EmptyContext extends Context(None) { - def setResourceTracker(tracker: ResourceTracker): Context = new Context0(Some(tracker)) - def removeResourceTracker(): Context = this + private object EmptyContext extends Context(None, Fiber.Global) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context0(Some(tracker), fiber) + def removeResourceTracker(): Context = new Context0(None, fiber) + + def setFiber(f: Fiber): Context = new Context0(resourceTracker, f) def get(k: Key): Option[_] = None def remove(k: Key): Context = this - def set(k: Key, v: Some[_]): Context = new Context1(resourceTracker, k, v) + def set(k: Key, v: Some[_]): Context = new Context1(resourceTracker, fiber, k, v) } - private final class Context0(resourceTracker: Option[ResourceTracker]) - extends Context(resourceTracker) { - def setResourceTracker(tracker: ResourceTracker): Context = new Context0(Some(tracker)) - def removeResourceTracker(): Context = EmptyContext + private final class Context0(resourceTracker: Option[ResourceTracker], fiber: Fiber) + extends Context(resourceTracker, fiber) { + def setResourceTracker(tracker: ResourceTracker): Context = new Context0(Some(tracker), fiber) + def removeResourceTracker(): Context = new Context0(None, fiber) + + def setFiber(f: Fiber): Context = new Context0(resourceTracker, f) def get(k: Key): Option[_] = None def remove(k: Key): Context = this - def set(k: Key, v: Some[_]): Context = new Context1(resourceTracker, k, v) + def set(k: Key, v: Some[_]): Context = new Context1(resourceTracker, fiber, k, v) } private final class Context1( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = - new Context1(Some(tracker), k1, v1) - def removeResourceTracker(): Context = new Context1(None, k1, v1) + new Context1(Some(tracker), fiber, k1, v1) + def removeResourceTracker(): Context = new Context1(None, fiber, k1, v1) + + def setFiber(f: Fiber): Context = new Context1(resourceTracker, f, k1, v1) def get(k: Key): Option[_] = if (k eq k1) v1 else None @@ -59,20 +69,24 @@ object Local { if (k eq k1) EmptyContext else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context1(resourceTracker, k1, v) - else new Context2(resourceTracker, k1, v1, k, v) + if (k eq k1) new Context1(resourceTracker, fiber, k1, v) + else new Context2(resourceTracker, fiber, k1, v1, k, v) } private final class Context2( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, v2: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = - new Context2(Some(tracker), k1, v1, k2, v2) - def removeResourceTracker(): Context = new Context2(None, k1, v1, k2, v2) + new Context2(Some(tracker), fiber, k1, v1, k2, v2) + def removeResourceTracker(): Context = new Context2(None, fiber, k1, v1, k2, v2) + + def setFiber(f: Fiber): Context = + new Context2(resourceTracker, f, k1, v1, k2, v2) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -80,14 +94,14 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context1(resourceTracker, k2, v2) - else if (k eq k2) new Context1(resourceTracker, k1, v1) + if (k eq k1) new Context1(resourceTracker, fiber, k2, v2) + else if (k eq k2) new Context1(resourceTracker, fiber, k1, v1) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context2(resourceTracker, k1, v, k2, v2) - else if (k eq k2) new Context2(resourceTracker, k1, v1, k2, v) - else new Context3(resourceTracker, k1, v1, k2, v2, k, v) + if (k eq k1) new Context2(resourceTracker, fiber, k1, v, k2, v2) + else if (k eq k2) new Context2(resourceTracker, fiber, k1, v1, k2, v) + else new Context3(resourceTracker, fiber, k1, v1, k2, v2, k, v) } // Script for generating the ContextN @@ -97,12 +111,17 @@ object Local { def mkContext(num: Int): String = { s"""private final class Context$num( resourceTracker: Option[ResourceTracker], + fiber: Fiber, ${fieldList(num)} - ) extends Context(resourceTracker) { + ) extends Context(resourceTracker, fiber) { ${mkSetResourceTracker(num)} ${mkRemoveResourceTracker(num)} + ${mkSetFiber(num)} + + ${mkRemoveFiber(num)} + ${mkGet(num)} ${mkRemove(num)} @@ -120,11 +139,15 @@ object Local { } def mkSetResourceTracker(num: Int): String = { - s"def setResourceTracker(tracker: ResourceTracker): Context = new Context$num(Some(tracker), ${fieldList(num)})" + s"def setResourceTracker(tracker: ResourceTracker): Context = new Context$num(Some(tracker), fiber, ${fieldList(num)})" } def mkRemoveResourceTracker(num: Int): String = { - s"def removeResourceTracker(): Context = new Context$num(None, ${fieldList(num)})" + s"def removeResourceTracker(): Context = new Context$num(None, fiber, ${fieldList(num)})" + } + + def mkSetFiber(num: Int): String = { + s"def setFiber(f: Fiber): Context = new Context$num(resourceTracker, f, ${fieldList(num)})" } def mkGet(num: Int): String = { @@ -140,7 +163,7 @@ object Local { def newCtx(sum: Int, current: Int): String = { (1.to(current - 1) ++: (current + 1).to(sum)).map { i => s"k$i, v$i" - }.mkString("(resourceTracker, ", ", ", ")") + }.mkString("(resourceTracker, fiber, ", ", ", ")") } val content = (1.to(num)).map { i => s"k$i) new Context${num - 1}${newCtx(num, i)}" @@ -155,7 +178,7 @@ object Local { if (i == current && last) s"k, v" else if (i == current) s"k$i, v" else s"k$i, v$i" - }.mkString("(resourceTracker, ", ", ", ")") + }.mkString("(resourceTracker, fiber, ", ", ", ")") } val content = (1.to(num)).map { i => s"k$i) new Context$num${newCtx(num, i, false)}" @@ -170,15 +193,17 @@ object Local { */ private final class Context3( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, v2: Some[_], k3: Key, v3: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context3( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -187,7 +212,17 @@ object Local { v3: Some[_]) def removeResourceTracker(): Context = - new Context3(None, k1: Key, v1: Some[_], k2: Key, v2: Some[_], k3: Key, v3: Some[_]) + new Context3(None, fiber, k1: Key, v1: Some[_], k2: Key, v2: Some[_], k3: Key, v3: Some[_]) + + def setFiber(f: Fiber): Context = new Context3( + resourceTracker, + f, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_]) def get(k: Key): Option[_] = if (k eq k1) v1 @@ -196,20 +231,21 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context2(resourceTracker, k2, v2, k3, v3) - else if (k eq k2) new Context2(resourceTracker, k1, v1, k3, v3) - else if (k eq k3) new Context2(resourceTracker, k1, v1, k2, v2) + if (k eq k1) new Context2(resourceTracker, fiber, k2, v2, k3, v3) + else if (k eq k2) new Context2(resourceTracker, fiber, k1, v1, k3, v3) + else if (k eq k3) new Context2(resourceTracker, fiber, k1, v1, k2, v2) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context3(resourceTracker, k1, v, k2, v2, k3, v3) - else if (k eq k2) new Context3(resourceTracker, k1, v1, k2, v, k3, v3) - else if (k eq k3) new Context3(resourceTracker, k1, v1, k2, v2, k3, v) - else new Context4(resourceTracker, k1, v1, k2, v2, k3, v3, k, v) + if (k eq k1) new Context3(resourceTracker, fiber, k1, v, k2, v2, k3, v3) + else if (k eq k2) new Context3(resourceTracker, fiber, k1, v1, k2, v, k3, v3) + else if (k eq k3) new Context3(resourceTracker, fiber, k1, v1, k2, v2, k3, v) + else new Context4(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k, v) } private final class Context4( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -218,9 +254,10 @@ object Local { v3: Some[_], k4: Key, v4: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context4( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -232,6 +269,19 @@ object Local { def removeResourceTracker(): Context = new Context4( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_]) + + def setFiber(f: Fiber): Context = new Context4( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -249,22 +299,23 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context3(resourceTracker, k2, v2, k3, v3, k4, v4) - else if (k eq k2) new Context3(resourceTracker, k1, v1, k3, v3, k4, v4) - else if (k eq k3) new Context3(resourceTracker, k1, v1, k2, v2, k4, v4) - else if (k eq k4) new Context3(resourceTracker, k1, v1, k2, v2, k3, v3) + if (k eq k1) new Context3(resourceTracker, fiber, k2, v2, k3, v3, k4, v4) + else if (k eq k2) new Context3(resourceTracker, fiber, k1, v1, k3, v3, k4, v4) + else if (k eq k3) new Context3(resourceTracker, fiber, k1, v1, k2, v2, k4, v4) + else if (k eq k4) new Context3(resourceTracker, fiber, k1, v1, k2, v2, k3, v3) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context4(resourceTracker, k1, v, k2, v2, k3, v3, k4, v4) - else if (k eq k2) new Context4(resourceTracker, k1, v1, k2, v, k3, v3, k4, v4) - else if (k eq k3) new Context4(resourceTracker, k1, v1, k2, v2, k3, v, k4, v4) - else if (k eq k4) new Context4(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v) - else new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k, v) + if (k eq k1) new Context4(resourceTracker, fiber, k1, v, k2, v2, k3, v3, k4, v4) + else if (k eq k2) new Context4(resourceTracker, fiber, k1, v1, k2, v, k3, v3, k4, v4) + else if (k eq k3) new Context4(resourceTracker, fiber, k1, v1, k2, v2, k3, v, k4, v4) + else if (k eq k4) new Context4(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v) + else new Context5(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k, v) } private final class Context5( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -275,9 +326,10 @@ object Local { v4: Some[_], k5: Key, v5: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context5( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -291,6 +343,21 @@ object Local { def removeResourceTracker(): Context = new Context5( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_]) + + def setFiber(f: Fiber): Context = new Context5( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -311,24 +378,29 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context4(resourceTracker, k2, v2, k3, v3, k4, v4, k5, v5) - else if (k eq k2) new Context4(resourceTracker, k1, v1, k3, v3, k4, v4, k5, v5) - else if (k eq k3) new Context4(resourceTracker, k1, v1, k2, v2, k4, v4, k5, v5) - else if (k eq k4) new Context4(resourceTracker, k1, v1, k2, v2, k3, v3, k5, v5) - else if (k eq k5) new Context4(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4) + if (k eq k1) new Context4(resourceTracker, fiber, k2, v2, k3, v3, k4, v4, k5, v5) + else if (k eq k2) new Context4(resourceTracker, fiber, k1, v1, k3, v3, k4, v4, k5, v5) + else if (k eq k3) new Context4(resourceTracker, fiber, k1, v1, k2, v2, k4, v4, k5, v5) + else if (k eq k4) new Context4(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k5, v5) + else if (k eq k5) new Context4(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context5(resourceTracker, k1, v, k2, v2, k3, v3, k4, v4, k5, v5) - else if (k eq k2) new Context5(resourceTracker, k1, v1, k2, v, k3, v3, k4, v4, k5, v5) - else if (k eq k3) new Context5(resourceTracker, k1, v1, k2, v2, k3, v, k4, v4, k5, v5) - else if (k eq k4) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v, k5, v5) - else if (k eq k5) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v) - else new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k, v) + if (k eq k1) new Context5(resourceTracker, fiber, k1, v, k2, v2, k3, v3, k4, v4, k5, v5) + else if (k eq k2) + new Context5(resourceTracker, fiber, k1, v1, k2, v, k3, v3, k4, v4, k5, v5) + else if (k eq k3) + new Context5(resourceTracker, fiber, k1, v1, k2, v2, k3, v, k4, v4, k5, v5) + else if (k eq k4) + new Context5(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v, k5, v5) + else if (k eq k5) + new Context5(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k5, v) + else new Context6(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k, v) } private final class Context6( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -341,9 +413,10 @@ object Local { v5: Some[_], k6: Key, v6: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context6( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -359,6 +432,23 @@ object Local { def removeResourceTracker(): Context = new Context6( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_]) + + def setFiber(f: Fiber): Context = new Context6( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -382,31 +472,39 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context5(resourceTracker, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) - else if (k eq k2) new Context5(resourceTracker, k1, v1, k3, v3, k4, v4, k5, v5, k6, v6) - else if (k eq k3) new Context5(resourceTracker, k1, v1, k2, v2, k4, v4, k5, v5, k6, v6) - else if (k eq k4) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k5, v5, k6, v6) - else if (k eq k5) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k6, v6) - else if (k eq k6) new Context5(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5) + if (k eq k1) new Context5(resourceTracker, fiber, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) + else if (k eq k2) + new Context5(resourceTracker, fiber, k1, v1, k3, v3, k4, v4, k5, v5, k6, v6) + else if (k eq k3) + new Context5(resourceTracker, fiber, k1, v1, k2, v2, k4, v4, k5, v5, k6, v6) + else if (k eq k4) + new Context5(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k5, v5, k6, v6) + else if (k eq k5) + new Context5(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k6, v6) + else if (k eq k6) + new Context5(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5) else this def set(k: Key, v: Some[_]): Context = - if (k eq k1) new Context6(resourceTracker, k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) + if (k eq k1) + new Context6(resourceTracker, fiber, k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) else if (k eq k2) - new Context6(resourceTracker, k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6) + new Context6(resourceTracker, fiber, k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6) else if (k eq k3) - new Context6(resourceTracker, k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6) + new Context6(resourceTracker, fiber, k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6) else if (k eq k4) - new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6) + new Context6(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6) else if (k eq k5) - new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6) + new Context6(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6) else if (k eq k6) - new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v) - else new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k, v) + new Context6(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v) + else + new Context7(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k, v) } private final class Context7( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -421,9 +519,10 @@ object Local { v6: Some[_], k7: Key, v7: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context7( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -441,6 +540,25 @@ object Local { def removeResourceTracker(): Context = new Context7( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_]) + + def setFiber(f: Fiber): Context = new Context7( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -467,39 +585,153 @@ object Local { else None def remove(k: Key): Context = - if (k eq k1) new Context6(resourceTracker, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + if (k eq k1) + new Context6(resourceTracker, fiber, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) else if (k eq k2) - new Context6(resourceTracker, k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + new Context6(resourceTracker, fiber, k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) else if (k eq k3) - new Context6(resourceTracker, k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7) + new Context6(resourceTracker, fiber, k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7) else if (k eq k4) - new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7) + new Context6(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7) else if (k eq k5) - new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7) + new Context6(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7) else if (k eq k6) - new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7) + new Context6(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7) else if (k eq k7) - new Context6(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) + new Context6(resourceTracker, fiber, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6) else this def set(k: Key, v: Some[_]): Context = if (k eq k1) - new Context7(resourceTracker, k1, v, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + new Context7( + resourceTracker, + fiber, + k1, + v, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7) else if (k eq k2) - new Context7(resourceTracker, k1, v1, k2, v, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7) else if (k eq k3) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v, k4, v4, k5, v5, k6, v6, k7, v7) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7) else if (k eq k4) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v, k5, v5, k6, v6, k7, v7) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v, + k5, + v5, + k6, + v6, + k7, + v7) else if (k eq k5) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v, k6, v6, k7, v7) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v, + k6, + v6, + k7, + v7) else if (k eq k6) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v, k7, v7) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v, + k7, + v7) else if (k eq k7) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v) else new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -520,6 +752,7 @@ object Local { private final class Context8( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -536,9 +769,10 @@ object Local { v7: Some[_], k8: Key, v8: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context8( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -558,6 +792,27 @@ object Local { def removeResourceTracker(): Context = new Context8( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_]) + + def setFiber(f: Fiber): Context = new Context8( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -588,27 +843,156 @@ object Local { def remove(k: Key): Context = if (k eq k1) - new Context7(resourceTracker, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + new Context7( + resourceTracker, + fiber, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k2) - new Context7(resourceTracker, k1, v1, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k3) - new Context7(resourceTracker, k1, v1, k2, v2, k4, v4, k5, v5, k6, v6, k7, v7, k8, v8) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k4) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k5, v5, k6, v6, k7, v7, k8, v8) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v3, + k5, + v5, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k5) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k6, v6, k7, v7, k8, v8) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k6, + v6, + k7, + v7, + k8, + v8) else if (k eq k6) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k7, v7, k8, v8) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k7, + v7, + k8, + v8) else if (k eq k7) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k8, v8) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k8, + v8) else if (k eq k8) - new Context7(resourceTracker, k1, v1, k2, v2, k3, v3, k4, v4, k5, v5, k6, v6, k7, v7) + new Context7( + resourceTracker, + fiber, + k1, + v1, + k2, + v2, + k3, + v3, + k4, + v4, + k5, + v5, + k6, + v6, + k7, + v7) else this def set(k: Key, v: Some[_]): Context = if (k eq k1) new Context8( resourceTracker, + fiber, k1, v, k2, @@ -628,6 +1012,7 @@ object Local { else if (k eq k2) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -647,6 +1032,7 @@ object Local { else if (k eq k3) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -666,6 +1052,7 @@ object Local { else if (k eq k4) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -685,6 +1072,7 @@ object Local { else if (k eq k5) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -704,6 +1092,7 @@ object Local { else if (k eq k6) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -723,6 +1112,7 @@ object Local { else if (k eq k7) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -742,6 +1132,7 @@ object Local { else if (k eq k8) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -761,6 +1152,7 @@ object Local { else new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -783,6 +1175,7 @@ object Local { private final class Context9( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -801,9 +1194,10 @@ object Local { v8: Some[_], k9: Key, v9: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context9( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -825,6 +1219,29 @@ object Local { def removeResourceTracker(): Context = new Context9( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_]) + + def setFiber(f: Fiber): Context = new Context9( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -860,6 +1277,7 @@ object Local { if (k eq k1) new Context8( resourceTracker, + fiber, k2, v2, k3, @@ -879,6 +1297,7 @@ object Local { else if (k eq k2) new Context8( resourceTracker, + fiber, k1, v1, k3, @@ -898,6 +1317,7 @@ object Local { else if (k eq k3) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -917,6 +1337,7 @@ object Local { else if (k eq k4) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -936,6 +1357,7 @@ object Local { else if (k eq k5) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -955,6 +1377,7 @@ object Local { else if (k eq k6) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -974,6 +1397,7 @@ object Local { else if (k eq k7) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -993,6 +1417,7 @@ object Local { else if (k eq k8) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -1012,6 +1437,7 @@ object Local { else if (k eq k9) new Context8( resourceTracker, + fiber, k1, v1, k2, @@ -1034,6 +1460,7 @@ object Local { if (k eq k1) new Context9( resourceTracker, + fiber, k1, v, k2, @@ -1055,6 +1482,7 @@ object Local { else if (k eq k2) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1076,6 +1504,7 @@ object Local { else if (k eq k3) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1097,6 +1526,7 @@ object Local { else if (k eq k4) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1118,6 +1548,7 @@ object Local { else if (k eq k5) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1139,6 +1570,7 @@ object Local { else if (k eq k6) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1160,6 +1592,7 @@ object Local { else if (k eq k7) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1181,6 +1614,7 @@ object Local { else if (k eq k8) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1202,6 +1636,7 @@ object Local { else if (k eq k9) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1223,6 +1658,7 @@ object Local { else new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1247,6 +1683,7 @@ object Local { private final class Context10( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -1267,9 +1704,10 @@ object Local { v9: Some[_], k10: Key, v10: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context10( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -1293,6 +1731,31 @@ object Local { def removeResourceTracker(): Context = new Context10( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_]) + + def setFiber(f: Fiber): Context = new Context10( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -1331,6 +1794,7 @@ object Local { if (k eq k1) new Context9( resourceTracker, + fiber, k2, v2, k3, @@ -1352,6 +1816,7 @@ object Local { else if (k eq k2) new Context9( resourceTracker, + fiber, k1, v1, k3, @@ -1373,6 +1838,7 @@ object Local { else if (k eq k3) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1394,6 +1860,7 @@ object Local { else if (k eq k4) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1415,6 +1882,7 @@ object Local { else if (k eq k5) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1436,6 +1904,7 @@ object Local { else if (k eq k6) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1457,6 +1926,7 @@ object Local { else if (k eq k7) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1478,6 +1948,7 @@ object Local { else if (k eq k8) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1499,6 +1970,7 @@ object Local { else if (k eq k9) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1520,6 +1992,7 @@ object Local { else if (k eq k10) new Context9( resourceTracker, + fiber, k1, v1, k2, @@ -1544,6 +2017,7 @@ object Local { if (k eq k1) new Context10( resourceTracker, + fiber, k1, v, k2, @@ -1567,6 +2041,7 @@ object Local { else if (k eq k2) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1590,6 +2065,7 @@ object Local { else if (k eq k3) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1613,6 +2089,7 @@ object Local { else if (k eq k4) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1636,6 +2113,7 @@ object Local { else if (k eq k5) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1659,6 +2137,7 @@ object Local { else if (k eq k6) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1682,6 +2161,7 @@ object Local { else if (k eq k7) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1705,6 +2185,7 @@ object Local { else if (k eq k8) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1728,6 +2209,7 @@ object Local { else if (k eq k9) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1751,6 +2233,7 @@ object Local { else if (k eq k10) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1774,6 +2257,7 @@ object Local { else new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -1800,6 +2284,7 @@ object Local { private final class Context11( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -1822,9 +2307,10 @@ object Local { v10: Some[_], k11: Key, v11: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context11( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -1850,6 +2336,33 @@ object Local { def removeResourceTracker(): Context = new Context11( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_]) + + def setFiber(f: Fiber): Context = new Context11( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -1891,6 +2404,7 @@ object Local { if (k eq k1) new Context10( resourceTracker, + fiber, k2, v2, k3, @@ -1914,6 +2428,7 @@ object Local { else if (k eq k2) new Context10( resourceTracker, + fiber, k1, v1, k3, @@ -1937,6 +2452,7 @@ object Local { else if (k eq k3) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1960,6 +2476,7 @@ object Local { else if (k eq k4) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -1983,6 +2500,7 @@ object Local { else if (k eq k5) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -2006,6 +2524,7 @@ object Local { else if (k eq k6) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -2029,6 +2548,7 @@ object Local { else if (k eq k7) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -2052,6 +2572,7 @@ object Local { else if (k eq k8) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -2075,6 +2596,7 @@ object Local { else if (k eq k9) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -2098,6 +2620,7 @@ object Local { else if (k eq k10) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -2121,6 +2644,7 @@ object Local { else if (k eq k11) new Context10( resourceTracker, + fiber, k1, v1, k2, @@ -2147,6 +2671,7 @@ object Local { if (k eq k1) new Context11( resourceTracker, + fiber, k1, v, k2, @@ -2172,6 +2697,7 @@ object Local { else if (k eq k2) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2197,6 +2723,7 @@ object Local { else if (k eq k3) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2222,6 +2749,7 @@ object Local { else if (k eq k4) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2247,6 +2775,7 @@ object Local { else if (k eq k5) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2272,6 +2801,7 @@ object Local { else if (k eq k6) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2297,6 +2827,7 @@ object Local { else if (k eq k7) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2322,6 +2853,7 @@ object Local { else if (k eq k8) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2347,6 +2879,7 @@ object Local { else if (k eq k9) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2372,6 +2905,7 @@ object Local { else if (k eq k10) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2397,6 +2931,7 @@ object Local { else if (k eq k11) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2422,6 +2957,7 @@ object Local { else new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -2450,6 +2986,7 @@ object Local { private final class Context12( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -2474,9 +3011,10 @@ object Local { v11: Some[_], k12: Key, v12: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context12( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -2504,6 +3042,35 @@ object Local { def removeResourceTracker(): Context = new Context12( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_]) + + def setFiber(f: Fiber): Context = new Context12( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -2548,6 +3115,7 @@ object Local { if (k eq k1) new Context11( resourceTracker, + fiber, k2, v2, k3, @@ -2573,6 +3141,7 @@ object Local { else if (k eq k2) new Context11( resourceTracker, + fiber, k1, v1, k3, @@ -2598,6 +3167,7 @@ object Local { else if (k eq k3) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2623,6 +3193,7 @@ object Local { else if (k eq k4) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2648,6 +3219,7 @@ object Local { else if (k eq k5) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2673,6 +3245,7 @@ object Local { else if (k eq k6) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2698,6 +3271,7 @@ object Local { else if (k eq k7) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2723,6 +3297,7 @@ object Local { else if (k eq k8) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2748,6 +3323,7 @@ object Local { else if (k eq k9) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2773,6 +3349,7 @@ object Local { else if (k eq k10) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2798,6 +3375,7 @@ object Local { else if (k eq k11) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2823,6 +3401,7 @@ object Local { else if (k eq k12) new Context11( resourceTracker, + fiber, k1, v1, k2, @@ -2851,6 +3430,7 @@ object Local { if (k eq k1) new Context12( resourceTracker, + fiber, k1, v, k2, @@ -2878,6 +3458,7 @@ object Local { else if (k eq k2) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -2905,6 +3486,7 @@ object Local { else if (k eq k3) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -2932,6 +3514,7 @@ object Local { else if (k eq k4) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -2959,6 +3542,7 @@ object Local { else if (k eq k5) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -2986,6 +3570,7 @@ object Local { else if (k eq k6) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3013,6 +3598,7 @@ object Local { else if (k eq k7) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3040,6 +3626,7 @@ object Local { else if (k eq k8) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3067,6 +3654,7 @@ object Local { else if (k eq k9) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3094,6 +3682,7 @@ object Local { else if (k eq k10) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3121,6 +3710,7 @@ object Local { else if (k eq k11) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3148,6 +3738,7 @@ object Local { else if (k eq k12) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3175,6 +3766,7 @@ object Local { else new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3205,6 +3797,7 @@ object Local { private final class Context13( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -3231,9 +3824,10 @@ object Local { v12: Some[_], k13: Key, v13: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context13( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -3263,6 +3857,37 @@ object Local { def removeResourceTracker(): Context = new Context13( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_]) + + def setFiber(f: Fiber): Context = new Context13( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -3310,6 +3935,7 @@ object Local { if (k eq k1) new Context12( resourceTracker, + fiber, k2, v2, k3, @@ -3337,6 +3963,7 @@ object Local { else if (k eq k2) new Context12( resourceTracker, + fiber, k1, v1, k3, @@ -3364,6 +3991,7 @@ object Local { else if (k eq k3) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3391,6 +4019,7 @@ object Local { else if (k eq k4) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3418,6 +4047,7 @@ object Local { else if (k eq k5) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3445,6 +4075,7 @@ object Local { else if (k eq k6) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3472,6 +4103,7 @@ object Local { else if (k eq k7) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3499,6 +4131,7 @@ object Local { else if (k eq k8) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3526,6 +4159,7 @@ object Local { else if (k eq k9) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3553,6 +4187,7 @@ object Local { else if (k eq k10) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3580,6 +4215,7 @@ object Local { else if (k eq k11) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3607,6 +4243,7 @@ object Local { else if (k eq k12) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3634,6 +4271,7 @@ object Local { else if (k eq k13) new Context12( resourceTracker, + fiber, k1, v1, k2, @@ -3664,6 +4302,7 @@ object Local { if (k eq k1) new Context13( resourceTracker, + fiber, k1, v, k2, @@ -3693,6 +4332,7 @@ object Local { else if (k eq k2) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3722,6 +4362,7 @@ object Local { else if (k eq k3) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3751,6 +4392,7 @@ object Local { else if (k eq k4) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3780,6 +4422,7 @@ object Local { else if (k eq k5) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3809,6 +4452,7 @@ object Local { else if (k eq k6) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3838,6 +4482,7 @@ object Local { else if (k eq k7) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3867,6 +4512,7 @@ object Local { else if (k eq k8) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3896,6 +4542,7 @@ object Local { else if (k eq k9) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3925,6 +4572,7 @@ object Local { else if (k eq k10) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3954,6 +4602,7 @@ object Local { else if (k eq k11) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -3983,6 +4632,7 @@ object Local { else if (k eq k12) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4012,6 +4662,7 @@ object Local { else if (k eq k13) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4041,6 +4692,7 @@ object Local { else new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4073,6 +4725,7 @@ object Local { private final class Context14( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -4101,9 +4754,10 @@ object Local { v13: Some[_], k14: Key, v14: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context14( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -4135,6 +4789,39 @@ object Local { def removeResourceTracker(): Context = new Context14( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_], + k14: Key, + v14: Some[_]) + + def setFiber(f: Fiber): Context = new Context14( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -4185,6 +4872,7 @@ object Local { if (k eq k1) new Context13( resourceTracker, + fiber, k2, v2, k3, @@ -4214,6 +4902,7 @@ object Local { else if (k eq k2) new Context13( resourceTracker, + fiber, k1, v1, k3, @@ -4243,6 +4932,7 @@ object Local { else if (k eq k3) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4272,6 +4962,7 @@ object Local { else if (k eq k4) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4301,6 +4992,7 @@ object Local { else if (k eq k5) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4330,6 +5022,7 @@ object Local { else if (k eq k6) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4359,6 +5052,7 @@ object Local { else if (k eq k7) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4388,6 +5082,7 @@ object Local { else if (k eq k8) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4417,6 +5112,7 @@ object Local { else if (k eq k9) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4446,6 +5142,7 @@ object Local { else if (k eq k10) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4475,6 +5172,7 @@ object Local { else if (k eq k11) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4504,6 +5202,7 @@ object Local { else if (k eq k12) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4533,6 +5232,7 @@ object Local { else if (k eq k13) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4562,6 +5262,7 @@ object Local { else if (k eq k14) new Context13( resourceTracker, + fiber, k1, v1, k2, @@ -4594,6 +5295,7 @@ object Local { if (k eq k1) new Context14( resourceTracker, + fiber, k1, v, k2, @@ -4625,6 +5327,7 @@ object Local { else if (k eq k2) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4656,6 +5359,7 @@ object Local { else if (k eq k3) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4687,6 +5391,7 @@ object Local { else if (k eq k4) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4718,6 +5423,7 @@ object Local { else if (k eq k5) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4749,6 +5455,7 @@ object Local { else if (k eq k6) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4780,6 +5487,7 @@ object Local { else if (k eq k7) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4811,6 +5519,7 @@ object Local { else if (k eq k8) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4842,6 +5551,7 @@ object Local { else if (k eq k9) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4873,6 +5583,7 @@ object Local { else if (k eq k10) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4904,6 +5615,7 @@ object Local { else if (k eq k11) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4935,6 +5647,7 @@ object Local { else if (k eq k12) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4966,6 +5679,7 @@ object Local { else if (k eq k13) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -4997,6 +5711,7 @@ object Local { else if (k eq k14) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5028,6 +5743,7 @@ object Local { else new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5062,6 +5778,7 @@ object Local { private final class Context15( resourceTracker: Option[ResourceTracker], + fiber: Fiber, k1: Key, v1: Some[_], k2: Key, @@ -5092,9 +5809,10 @@ object Local { v14: Some[_], k15: Key, v15: Some[_]) - extends Context(resourceTracker) { + extends Context(resourceTracker, fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new Context15( Some(tracker), + fiber, k1: Key, v1: Some[_], k2: Key, @@ -5128,6 +5846,41 @@ object Local { def removeResourceTracker(): Context = new Context15( None, + fiber, + k1: Key, + v1: Some[_], + k2: Key, + v2: Some[_], + k3: Key, + v3: Some[_], + k4: Key, + v4: Some[_], + k5: Key, + v5: Some[_], + k6: Key, + v6: Some[_], + k7: Key, + v7: Some[_], + k8: Key, + v8: Some[_], + k9: Key, + v9: Some[_], + k10: Key, + v10: Some[_], + k11: Key, + v11: Some[_], + k12: Key, + v12: Some[_], + k13: Key, + v13: Some[_], + k14: Key, + v14: Some[_], + k15: Key, + v15: Some[_]) + + def setFiber(f: Fiber): Context = new Context15( + resourceTracker, + f, k1: Key, v1: Some[_], k2: Key, @@ -5181,6 +5934,7 @@ object Local { if (k eq k1) new Context14( resourceTracker, + fiber, k2, v2, k3, @@ -5212,6 +5966,7 @@ object Local { else if (k eq k2) new Context14( resourceTracker, + fiber, k1, v1, k3, @@ -5243,6 +5998,7 @@ object Local { else if (k eq k3) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5274,6 +6030,7 @@ object Local { else if (k eq k4) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5305,6 +6062,7 @@ object Local { else if (k eq k5) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5336,6 +6094,7 @@ object Local { else if (k eq k6) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5367,6 +6126,7 @@ object Local { else if (k eq k7) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5398,6 +6158,7 @@ object Local { else if (k eq k8) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5429,6 +6190,7 @@ object Local { else if (k eq k9) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5460,6 +6222,7 @@ object Local { else if (k eq k10) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5491,6 +6254,7 @@ object Local { else if (k eq k11) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5522,6 +6286,7 @@ object Local { else if (k eq k12) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5553,6 +6318,7 @@ object Local { else if (k eq k13) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5584,6 +6350,7 @@ object Local { else if (k eq k14) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5615,6 +6382,7 @@ object Local { else if (k eq k15) new Context14( resourceTracker, + fiber, k1, v1, k2, @@ -5649,6 +6417,7 @@ object Local { if (k eq k1) new Context15( resourceTracker, + fiber, k1, v, k2, @@ -5682,6 +6451,7 @@ object Local { else if (k eq k2) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5715,6 +6485,7 @@ object Local { else if (k eq k3) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5748,6 +6519,7 @@ object Local { else if (k eq k4) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5781,6 +6553,7 @@ object Local { else if (k eq k5) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5814,6 +6587,7 @@ object Local { else if (k eq k6) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5847,6 +6621,7 @@ object Local { else if (k eq k7) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5880,6 +6655,7 @@ object Local { else if (k eq k8) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5913,6 +6689,7 @@ object Local { else if (k eq k9) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5946,6 +6723,7 @@ object Local { else if (k eq k10) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -5979,6 +6757,7 @@ object Local { else if (k eq k11) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -6012,6 +6791,7 @@ object Local { else if (k eq k12) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -6045,6 +6825,7 @@ object Local { else if (k eq k13) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -6078,6 +6859,7 @@ object Local { else if (k eq k14) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -6111,6 +6893,7 @@ object Local { else if (k eq k15) new Context15( resourceTracker, + fiber, k1, v1, k2, @@ -6145,12 +6928,14 @@ object Local { } private final class ContextN(kN: Key, vN: Some[_], rest: Context) - extends Context(rest.resourceTracker) { + extends Context(rest.resourceTracker, rest.fiber) { def setResourceTracker(tracker: ResourceTracker): Context = new ContextN(kN, vN, rest.setResourceTracker(tracker)) def removeResourceTracker(): Context = new ContextN(kN, vN, rest.removeResourceTracker()) + def setFiber(f: Fiber): Context = new ContextN(kN, vN, rest.setFiber(f)) + def get(k: Key): Option[_] = if (k eq kN) vN else rest.get(k) diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index 4ce815d37f..d2a9aae995 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -95,7 +95,7 @@ object Promise { @tailrec final def runInScheduler(t: Try[A]): Unit = { if (this ne WaitQueue.Empty) { - Scheduler.submit(() => first(t)) + this.first.saved.fiber.submitTask(() => first(t)) rest.runInScheduler(t) } } @@ -138,7 +138,7 @@ object Promise { * it will make "Linked" and "Waiting" state cases ambiguous. This, however, * may change following the further performance improvements. */ - private[util] abstract class K[-A] extends WaitQueue[A] { + private[util] abstract class K[-A](val saved: Local.Context) extends WaitQueue[A] { final def first: K[A] = this final def rest: WaitQueue[A] = WaitQueue.empty def apply(r: Try[A]): Unit @@ -168,7 +168,7 @@ object Promise { // It's not possible (yet) to embed K[A] into Promise because // Promise[A] (Linked) and WaitQueue (Waiting) states become ambiguous. - private[this] val k = new K[A] { + private[this] val k = new K[A](Local.Context.empty) { // This is only called after the underlying has been successfully satisfied def apply(result: Try[A]): Unit = self.update(result) } @@ -212,7 +212,7 @@ object Promise { * @param k the closure to invoke in the saved context, with the * provided result */ - private final class Monitored[A](saved: Local.Context, k: Try[A] => Unit) extends K[A] { + private final class Monitored[A](saved: Local.Context, k: Try[A] => Unit) extends K[A](saved) { def apply(result: Try[A]): Unit = { val current = Local.save() @@ -224,7 +224,7 @@ object Promise { } } - private abstract class Transformer[A, B](saved: Local.Context) extends K[A] { + private abstract class Transformer[A](saved: Local.Context) extends K[A](saved) { protected[this] def k(r: Try[A]): Unit @@ -252,7 +252,7 @@ object Promise { saved: Local.Context, f: Try[A] => Future[B], promise: Promise[B]) - extends Transformer[A, B](saved) { + extends Transformer[A](saved) { protected[this] def k(r: Try[A]): Unit = // The promise can be fulfilled only by the transformer, so it's safe to use `become` here @@ -269,7 +269,7 @@ object Promise { saved: Local.Context, f: Try[A] => Try[B], promise: Promise[B]) - extends Transformer[A, B](saved) { + extends Transformer[A](saved) { protected[this] def k(r: Try[A]): Unit = { // The promise can be fulfilled only by the transformer, so it's safe to use `update` here diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index b8c19a9b14..4b8b4837ca 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -985,7 +985,7 @@ class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenProp val inner1 = new Promise[String] val inner2 = new Promise[String] val f: Future[String] = Future.monitored { inner2.ensure(()); inner1 } - val s: String = "." * 1024 + val s: String = "." * 1024 * 10 val sSize: Long = ObjectSizeCalculator.getObjectSize(s) inner1.setValue(s) val inner2Size: Long = ObjectSizeCalculator.getObjectSize(inner2) diff --git a/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala b/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala index afb75749a1..f4b5a7474d 100644 --- a/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WaitQueueTest.scala @@ -6,7 +6,7 @@ import org.scalatest.funsuite.AnyFunSuite class WaitQueueTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { - private class EmptyK[A] extends Promise.K[A] { + private class EmptyK[A] extends Promise.K[A](Local.Context.empty) { protected[util] def depth: Short = 0 def apply(ta: Try[A]): Unit = () } From 7999056a95bc64b64661e49f7b90eea45a9778f3 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Fri, 16 Sep 2022 17:41:21 +0000 Subject: [PATCH 721/796] [logging] remove unused refs to util/util-logging [1/?] Problem util-logging brings in slf4j-jdk14 implementation which is undesirable. See go/state-of-logging amongst other things. Solution remove unused deps to make it easier to remove the real ones JIRA Issues: DEAD-53 Differential Revision: https://phabricator.twitter.biz/D977311 --- util-jvm/src/test/scala/BUILD | 1 - util-routing/src/main/scala/com/twitter/util/routing/BUILD | 5 ++++- .../src/test/java/com/twitter/util/routing/BUILD.bazel | 1 - .../src/test/scala/com/twitter/util/routing/BUILD.bazel | 1 + .../test/scala/com/twitter/util/routing/simple/BUILD.bazel | 2 -- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/util-jvm/src/test/scala/BUILD b/util-jvm/src/test/scala/BUILD index 48b7cefce7..955928957e 100644 --- a/util-jvm/src/test/scala/BUILD +++ b/util-jvm/src/test/scala/BUILD @@ -6,7 +6,6 @@ COMMON_DEPS = [ "util/util-app/src/main/scala", "util/util-core:util-core-util", "util/util-jvm/src/main/scala/com/twitter/jvm", - "util/util-logging/src/main/scala", ] junit_tests( diff --git a/util-routing/src/main/scala/com/twitter/util/routing/BUILD b/util-routing/src/main/scala/com/twitter/util/routing/BUILD index 137bafba10..7cddac8122 100644 --- a/util-routing/src/main/scala/com/twitter/util/routing/BUILD +++ b/util-routing/src/main/scala/com/twitter/util/routing/BUILD @@ -12,5 +12,8 @@ scala_library( "util/util-core:util-core-util", "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", ], - exports = ["util/util-slf4j-api/src/main/scala/com/twitter/util/logging"], + exports = [ + "util/util-core:util-core-util", + "util/util-slf4j-api/src/main/scala/com/twitter/util/logging", + ], ) diff --git a/util-routing/src/test/java/com/twitter/util/routing/BUILD.bazel b/util-routing/src/test/java/com/twitter/util/routing/BUILD.bazel index 878f921bb0..b680dc96fc 100644 --- a/util-routing/src/test/java/com/twitter/util/routing/BUILD.bazel +++ b/util-routing/src/test/java/com/twitter/util/routing/BUILD.bazel @@ -14,7 +14,6 @@ junit_tests( "3rdparty/jvm/org/scala-lang:scala-library", "3rdparty/jvm/org/scalatestplus:junit", "util/util-core:util-core-util", - "util/util-logging/src/main/scala", "util/util-routing/src/main/scala/com/twitter/util/routing", ], ) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel b/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel index cdb7da747f..37a861211c 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel +++ b/util-routing/src/test/scala/com/twitter/util/routing/BUILD.bazel @@ -14,5 +14,6 @@ junit_tests( "util/util-routing/src/main/scala/com/twitter/util/routing", "util/util-routing/src/test/scala/com/twitter/util/routing/dynamic", "util/util-routing/src/test/scala/com/twitter/util/routing/simple", + "util/util-core/src/main/scala/com/twitter/conversions", ], ) diff --git a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD.bazel b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD.bazel index 621e2a799e..31190bd87e 100644 --- a/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD.bazel +++ b/util-routing/src/test/scala/com/twitter/util/routing/simple/BUILD.bazel @@ -6,10 +6,8 @@ scala_library( tags = ["bazel-compatible"], dependencies = [ "util/util-core:util-core-util", - "util/util-logging/src/main/scala/com/twitter/logging", "util/util-routing/src/main/scala/com/twitter/util/routing", ], exports = [ - "util/util-logging/src/main/scala/com/twitter/logging", ], ) From 9dc260a297e7a0dba5390139730364125982f306 Mon Sep 17 00:00:00 2001 From: Steve Niemitz Date: Fri, 16 Sep 2022 23:30:43 +0000 Subject: [PATCH 722/796] [util] Add "Utf8Bytes" class to represent UTF-8 encoded bytes Problem There is currently no good way of representing a series of UTF-8 encoded bytes other than simply a Buf. A simple way to represent them would be useful for serializing/deserializing data but lazily encoding/decoding strings to bytes. Solution Introduce `Utf8Bytes` Differential Revision: https://phabricator.twitter.biz/D978198 --- .../main/scala/com/twitter/io/Utf8Bytes.scala | 77 +++++++++++++++++++ .../scala/com/twitter/io/Utf8BytesTest.scala | 76 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 util-core/src/main/scala/com/twitter/io/Utf8Bytes.scala create mode 100644 util-core/src/test/scala/com/twitter/io/Utf8BytesTest.scala diff --git a/util-core/src/main/scala/com/twitter/io/Utf8Bytes.scala b/util-core/src/main/scala/com/twitter/io/Utf8Bytes.scala new file mode 100644 index 0000000000..060f7afb67 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/io/Utf8Bytes.scala @@ -0,0 +1,77 @@ +package com.twitter.io + +import java.nio.charset.StandardCharsets + +object Utf8Bytes { + + /** + * Create a [[Utf8Bytes]] instance from a byte array. The array passed in is copied to ensure it can not be later + * mutated. + * + * @note No validation is done on the passed bytes to ensure that they are valid UTF-8. If invalid bytes are present + * they will be converted to the default UTF-8 replacement string. + * @note As an optimization, if it's known that the byte[] will never be mutated once passed to the [[Utf8Bytes]] + * instance, use `Utf8Bytes(Buf.ByteArray.Owned(bytes))` to create an instance without copying the byte[]. + */ + def apply(bytes: Array[Byte]): Utf8Bytes = apply(Buf.ByteArray.Shared(bytes)) + + /** + * Create a [[Utf8Bytes]] instance from a [[String]]. The string is first encoded into UTF-8 bytes. + */ + def apply(str: String): Utf8Bytes = apply(Buf.Utf8(str)) + + /** + * Create a [[Utf8Bytes]] instance from a [[Buf]]. + * + * @note No validation is done on the passed bytes to ensure that they are valid UTF-8. If invalid bytes are present + * they will be converted to the default UTF-8 replacement string. + * @note This is the most efficient way to create an instance. It neither has to encode a string to bytes nor do a + * defensive copy, since [[Buf]] instances are already immutable. + */ + def apply(buf: Buf): Utf8Bytes = new Utf8Bytes(buf) + + val empty: Utf8Bytes = apply(Buf.Empty) +} + +/** + * An immutable representation of a UTF-8 encoded string. The representation is lazily converted to a [[String]] when + * operations require it. + * + * @note This class is thread-safe. + */ +final class Utf8Bytes private ( + private[this] val buf: Buf) + extends CharSequence { + + private[this] var str: String = _ + + /** + * The underlying byte content of this instance. + */ + def asBuf: Buf = buf + + override def toString: String = { + // multiple threads may race to set `str`, however doing so is safe + if (str == null) { + str = Buf.decodeString(buf, StandardCharsets.UTF_8) + } + str + } + + override def equals(obj: Any): Boolean = + obj match { + case other: AnyRef if this.eq(other) => true + case other: Utf8Bytes => buf == other.asBuf + case _ => false + } + + override def hashCode(): Int = buf.hashCode + + // note, all methods below must operate on chars, not bytes, so they must operate on the string representation + // rather than the byte representation + override def length(): Int = toString.length + + override def charAt(index: Int): Char = toString.charAt(index) + + override def subSequence(start: Int, end: Int): CharSequence = toString.subSequence(start, end) +} diff --git a/util-core/src/test/scala/com/twitter/io/Utf8BytesTest.scala b/util-core/src/test/scala/com/twitter/io/Utf8BytesTest.scala new file mode 100644 index 0000000000..726a4a50cb --- /dev/null +++ b/util-core/src/test/scala/com/twitter/io/Utf8BytesTest.scala @@ -0,0 +1,76 @@ +package com.twitter.io + +import org.scalatest.funsuite.AnyFunSuite +import java.nio.charset.StandardCharsets + +class Utf8BytesTest extends AnyFunSuite { + test("Utf8 created from a byte[] is immutable") { + val arr = "this is a test".getBytes(StandardCharsets.UTF_8) + val utf8 = Utf8Bytes(arr) + arr(0) = 'd' + + assert(utf8.toString == "this is a test") + } + + test("from string") { + val utf8 = Utf8Bytes("test string") + assert(utf8.toString == "test string") + } + + test("from Buf") { + val utf8 = Utf8Bytes(Buf.Utf8("test string")) + assert(utf8.toString == "test string") + } + + test("byte length and length accurate when using multi-byte characters") { + val utf8 = Utf8Bytes("\u93E1") + assert(utf8.length() == 1) + assert(utf8.asBuf.length == 3) + } + + test("simple subsequence") { + val utf8 = Utf8Bytes("test string") + val slice = utf8.subSequence(5, 11) + assert(slice == "string") + } + + test("subsequence with multi-byte characters") { + val utf8 = Utf8Bytes("\u93E1 string") + val slice = utf8.subSequence(2, 8) + assert(slice == "string") + } + + test("simple charAt") { + val utf8 = Utf8Bytes("abcdef") + assert(utf8.charAt(4) == 'e') + } + + test("charAt with multi-byte characters") { + val utf8 = Utf8Bytes("\u93E1abcdef") + assert(utf8.charAt(4) == 'd') + } + + test("asBuf round trip") { + val buf = Buf.Utf8("test string") + val utf8 = Utf8Bytes(buf) + assert(utf8.asBuf == buf) + } + + test("invalid utf8 bytes") { + val buf = Buf.ByteArray.Owned(Array(0x80.toByte)) + val utf8 = Utf8Bytes(buf) + assert(utf8.toString == "\uFFFD") + } + + test("equality and hash code") { + val a = Utf8Bytes("test string") + val b = Utf8Bytes("test string") + val c = Utf8Bytes("test string 2") + + assert(a.hashCode() == b.hashCode()) + + assert(a == a) + assert(a == b) + assert(a != c) + } +} From dc649448fc1382f6c243d476c6efd294dc2d7203 Mon Sep 17 00:00:00 2001 From: Steve Niemitz Date: Wed, 21 Sep 2022 05:30:34 +0000 Subject: [PATCH 723/796] util-core: Expose the raw byte[] and start on Buf.ByteArray Problem: There is no efficient way to access the underyling byte[], start, and end fields on Buf.ByteArray. Buf.ByteArray.Owned.unapply requires multiple allocations, making it less desirable for high-performance code. Solution: Expose unsafeArrayOffset as public property for start, add unsafeRawByteArray to allow access to the underlying byte[]. Differential Revision: https://phabricator.twitter.biz/D980110 --- .../src/main/scala/com/twitter/io/Buf.scala | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index eee08d65cd..f942c6b09d 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -1,8 +1,10 @@ package com.twitter.io -import com.twitter.io.Buf.{IndexedThree, IndexedTwo} +import com.twitter.io.Buf.IndexedThree +import com.twitter.io.Buf.IndexedTwo import java.nio.ReadOnlyBufferException -import java.nio.charset.{Charset, StandardCharsets => JChar} +import java.nio.charset.Charset +import java.nio.charset.{StandardCharsets => JChar} import scala.annotation.switch import scala.collection.immutable.VectorBuilder @@ -323,7 +325,10 @@ object Buf { checkSliceArgs(from, until) this } - protected def unsafeByteArrayBuf: Option[Buf.ByteArray] = None + protected val unsafeByteArrayBuf: Option[Buf.ByteArray] = Some( + new ByteArray(Array.emptyByteArray, 0, 0)) + override def unsafeByteArray: Array[Byte] = Array.emptyByteArray + def get(index: Int): Byte = throw new IndexOutOfBoundsException(s"Index out of bounds: $index") def process(from: Int, until: Int, processor: Processor): Int = { @@ -611,6 +616,17 @@ object Buf { private[Buf] val end: Int) extends Buf { + /** + * The backing byte[]. The array is only valid from [unsafeArrayOffset, unsafeArrayOffset + length), any data + * outside of those bounds is undefined. + */ + def unsafeRawByteArray: Array[Byte] = bytes + + /** + * The start offset of the data in the array returned by `unsafeRawByteArray`. + */ + def unsafeArrayOffset: Int = begin + def get(index: Int): Byte = { val off = begin + index if (off >= end) From c990851334b937102c3233c26c20541242fd0ce9 Mon Sep 17 00:00:00 2001 From: Moses Nakamura Date: Wed, 21 Sep 2022 18:14:49 +0000 Subject: [PATCH 724/796] finagle-core: Extract long to hex logic from SpanId for reuse Problem Java's long to hex implementation is a little slow. Finagle provides an implementation, but it's hidden in SpanID, even though it may be useful in other contexts. Solution Add it as a method on com.twitter.finagle.Showable, so that the logic for converting from long to hex doesn't need to be duplicated. JIRA Issues: CSL-12321 Differential Revision: https://phabricator.twitter.biz/D976930 --- .../twitter/conversions/U64Benchmark.scala | 33 +++++++++++++++++++ .../com/twitter/conversions/U64Ops.scala | 27 ++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 util-benchmark/src/main/scala/com/twitter/conversions/U64Benchmark.scala diff --git a/util-benchmark/src/main/scala/com/twitter/conversions/U64Benchmark.scala b/util-benchmark/src/main/scala/com/twitter/conversions/U64Benchmark.scala new file mode 100644 index 0000000000..e3244cb9c0 --- /dev/null +++ b/util-benchmark/src/main/scala/com/twitter/conversions/U64Benchmark.scala @@ -0,0 +1,33 @@ +package com.twitter.conversions + +import com.twitter.conversions.U64Ops._ +import com.twitter.util.StdBenchAnnotations +import org.openjdk.jmh.annotations._ + +// ./bazel run //util/util-benchmark/src/main/scala:jmh -- 'U64Benchmark' +@State(Scope.Benchmark) +class U64Benchmark extends StdBenchAnnotations { + private[this] val rng = new scala.util.Random(42) + private[this] val hexes = Seq.fill(97)(rng.nextLong()) :+ 0L :+ Long.MaxValue :+ Long.MinValue + private[this] val longToHex: Long => String = { long: Long => long.toU64HexString } + private[this] val javaLongToHex: Long => String = (java.lang.Long.toHexString _) + private[this] val leadingZerosJavaLongToHex: Long => String = { long: Long => + "%16x".format(long) + } + + @Benchmark + def u64Hexes: Seq[String] = { + hexes.map(longToHex) + } + + @Benchmark + def javaHexes: Seq[String] = { + hexes.map(javaLongToHex) + } + + @Benchmark + def leadingZerosJavaHexes: Seq[String] = { + hexes.map(leadingZerosJavaLongToHex) + } + +} diff --git a/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala index 75599ffe57..8ff3586440 100644 --- a/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala +++ b/util-core/src/main/scala/com/twitter/conversions/U64Ops.scala @@ -14,11 +14,36 @@ object U64Ops { def toU64Long: Long = java.lang.Long.parseUnsignedLong(self, 16) } + private[this] val lookupTable: Array[Char] = + (('0' to '9') ++ ('a' to 'f')).toArray + + private[this] def hex(l: Long): String = { + val chs = new Array[Char](16) + chs(0) = lookupTable((l >> 60 & 0xf).toInt) + chs(1) = lookupTable((l >> 56 & 0xf).toInt) + chs(2) = lookupTable((l >> 52 & 0xf).toInt) + chs(3) = lookupTable((l >> 48 & 0xf).toInt) + chs(4) = lookupTable((l >> 44 & 0xf).toInt) + chs(5) = lookupTable((l >> 40 & 0xf).toInt) + chs(6) = lookupTable((l >> 36 & 0xf).toInt) + chs(7) = lookupTable((l >> 32 & 0xf).toInt) + chs(8) = lookupTable((l >> 28 & 0xf).toInt) + chs(9) = lookupTable((l >> 24 & 0xf).toInt) + chs(10) = lookupTable((l >> 20 & 0xf).toInt) + chs(11) = lookupTable((l >> 16 & 0xf).toInt) + chs(12) = lookupTable((l >> 12 & 0xf).toInt) + chs(13) = lookupTable((l >> 8 & 0xf).toInt) + chs(14) = lookupTable((l >> 4 & 0xf).toInt) + chs(15) = lookupTable((l & 0xf).toInt) + new String(chs, 0, 16) + } + /** * Converts this unsigned 64-bit long value into a 16-character HEX string. */ implicit class RichU64Long(val self: Long) extends AnyVal { - def toU64HexString: String = "%016x".format(self) + + def toU64HexString: String = hex(self) } /** From 9dc282ab7937efddf6426cba1d3ab9441330f16f Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Fri, 23 Sep 2022 22:19:56 +0000 Subject: [PATCH 725/796] [util-core, finagle-core] Set new fiber per request that comes into the server Problem / Solution: Insert a new filter into the server stack that creates and sets a new Fiber in the Local context for each request as it comes in. This fiber will handle scheduler submission for the given request. Also create a new Fiber implementation that caches the current Scheduler from fiber creation time. This scheduler will only be saved and used for the lifetime of the fiber, which is the duration of the request in this case. JIRA Issues: CSL-12267 Differential Revision: https://phabricator.twitter.biz/D977958 --- util-core/src/main/scala/com/twitter/util/Fiber.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/util-core/src/main/scala/com/twitter/util/Fiber.scala b/util-core/src/main/scala/com/twitter/util/Fiber.scala index dc235c7a76..aaf53b7d8d 100644 --- a/util-core/src/main/scala/com/twitter/util/Fiber.scala +++ b/util-core/src/main/scala/com/twitter/util/Fiber.scala @@ -20,6 +20,16 @@ private[twitter] object Fiber { } } + // Create fiber that captures the current Scheduler to avoid the volatile lookup on each + // submission + def newCachedSchedulerFiber(): Fiber = new Fiber { + // Just cache it so we don't have to keep looking at the volatile var each time. + private val scheduler = Scheduler() + def submitTask(r: FiberTask): Unit = { + scheduler.submit(r) + } + } + def let[T](fiber: Fiber)(f: => T): T = { val oldCtx = Local.save() val newCtx = oldCtx.setFiber(fiber) From 5560502e0e6d0aa4c928411f66128dd13d331afa Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Tue, 27 Sep 2022 21:16:52 +0000 Subject: [PATCH 726/796] util-stats: Add notion of an undetermined identity type Problem We want to be able to exclude some metrics from being dimensional because they will become redundant. However, we don't have a way to express that right now. Solution Add an `NonDeterminate` state for our metric identity. This becomes the default state. As metric names this flag can be refined to express that a metric name is only valid hierarchically. JIRA Issues: CSL-12091 Differential Revision: https://phabricator.twitter.biz/D964827 --- .../twitter/finagle/stats/MetricBuilder.scala | 67 +++++++++++++++++-- .../stats/InMemoryStatsReceiverTest.scala | 6 +- .../finagle/stats/StatsReceiverTest.scala | 7 +- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 9160ecf854..ee53b88056 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -5,6 +5,7 @@ import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType import com.twitter.finagle.stats.MetricBuilder.Identity +import com.twitter.finagle.stats.MetricBuilder.IdentityType import com.twitter.finagle.stats.MetricBuilder.MetricType import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter import scala.annotation.varargs @@ -191,6 +192,60 @@ object MetricBuilder { def toPrometheusString: String = "counter" } + /** Representation of how an [[Identity]]s name is defined */ + sealed abstract class IdentityType private () { + + /** + * Bias this [[IdentityType]] to the specified type. + * + * @param to in the case that this is a `NonDeterminate` identity type, what the result should be. + */ + final def bias(to: IdentityType.ResolvedIdentityType): IdentityType.ResolvedIdentityType = + this match { + case IdentityType.NonDeterminate => to + case IdentityType.Full => IdentityType.Full + case IdentityType.HierarchicalOnly => IdentityType.HierarchicalOnly + } + } + + object IdentityType { + + /** + * Resolves an [[IdentityType]] to the default [[ResolvedIdentityType]] + * + * A centralized place to resolve the ambiguity of the [[NonDeterminate]] identity type. + * We often need to know if a metric has dimensional support or not such as when emitting + * a metric. This function offers a central place where that decision is made so that in + * the future it can be changed in a central location or even put behind a flag. + * + * The current behavior is to default to [[IdentityType.HierarchicalOnly]] + */ + private[twitter] def toResolvedIdentityType(identityType: IdentityType): ResolvedIdentityType = + identityType.bias(IdentityType.HierarchicalOnly) + + /** An [[IdentityType]] that cannot be [[NonDeterminate]] */ + sealed abstract class ResolvedIdentityType extends IdentityType + + /** + * An [[IdentityType]] which has a name that is currently valid both dimensionally and + * hierarchically, but leaves it up to the [[StatsReceiver]] to decide whether to represent + * it with one or both names. + */ + case object NonDeterminate extends IdentityType + + /** + * An [[IdentityType]] that has means the associated metric [[Identity]] has been + * explicitly marked to not be valid dimensionally + */ + case object HierarchicalOnly extends ResolvedIdentityType + + /** + * An [[IdentityType]] which means that the associated metric [[Identity]] is valid + * both dimensionally and hierarchically + */ + case object Full extends ResolvedIdentityType + } + /** * Experimental way to represent both hierarchical and dimensional identity information. * @@ -200,11 +255,11 @@ object MetricBuilder { hierarchicalName: Seq[String], dimensionalName: Seq[String], labels: Map[String, String] = Map.empty, - hierarchicalOnly: Boolean = true) { + identityType: IdentityType = IdentityType.NonDeterminate) { override def toString: String = { val hname = hierarchicalName.mkString(metadataScopeSeparator()) val dname = dimensionalName.mkString(DimensionalNameScopeSeparator) - s"Identity($hname, $dname, $labels, $hierarchicalOnly)" + s"Identity($hname, $dname, $labels, $identityType)" } } } @@ -331,13 +386,13 @@ class MetricBuilder private ( } private[twitter] def withHierarchicalOnly: MetricBuilder = { - if (identity.hierarchicalOnly) this - else copy(identity = identity.copy(hierarchicalOnly = true)) + if (identity.identityType == IdentityType.HierarchicalOnly) this + else copy(identity = identity.copy(identityType = IdentityType.HierarchicalOnly)) } private[twitter] def withDimensionalSupport: MetricBuilder = { - if (identity.hierarchicalOnly) copy(identity = identity.copy(hierarchicalOnly = false)) - else this + if (identity.identityType == IdentityType.Full) this + else copy(identity = identity.copy(identityType = IdentityType.Full)) } @varargs diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala index 33cdb12893..be58e5d891 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/InMemoryStatsReceiverTest.scala @@ -362,13 +362,13 @@ class InMemoryStatsReceiverTest extends AnyFunSuite with Eventually with Integra assert(parts.length == 3) assert( parts( - 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, default, None, Identity(coolGauge, coolGauge, Map(), true), List(), None, default, Vector(), GaugeType)") + 0) == "coolGauge MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, default, None, Identity(coolGauge, coolGauge, Map(), NonDeterminate), List(), None, default, Vector(), GaugeType)") assert( parts( - 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, default, None, Identity(rad/histo, histo, Map(), true), List(), None, default, Vector(), HistogramType)") + 1) == "rad/histo MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, default, None, Identity(rad/histo, histo, Map(), NonDeterminate), List(), None, default, Vector(), HistogramType)") assert( parts( - 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, default, None, Identity(sweet/counter, sweet_counter, Map(), true), List(), None, default, Vector(), CounterType)") + 2) == "sweet/counter MetricBuilder(false, No description provided, Unspecified, NoRoleSpecified, default, None, Identity(sweet/counter, sweet_counter, Map(), NonDeterminate), List(), None, default, Vector(), CounterType)") } finally { ps.close() } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index dfd7ecaea7..399178de82 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -5,6 +5,7 @@ import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.MetricBuilder.HistogramType +import com.twitter.finagle.stats.MetricBuilder.IdentityType import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter import com.twitter.util.Await import com.twitter.util.Future @@ -121,7 +122,7 @@ class StatsReceiverTest extends AnyFunSuite { assert(mb.identity.labels.isEmpty) assert(mb.identity.hierarchicalName == Seq("cool", "numbers")) assert(mb.identity.dimensionalName == Seq("numbers")) - assert(mb.identity.hierarchicalOnly) + assert(mb.identity.identityType == IdentityType.NonDeterminate) } test("StatsReceiver.hierarchicalScope: composes with label correctly") { @@ -137,7 +138,7 @@ class StatsReceiverTest extends AnyFunSuite { assert(mb.identity.hierarchicalName == Seq("clnt", "backend", "requests")) assert(mb.identity.dimensionalName == Seq("requests")) assert(mb.identity.labels == Map("clnt" -> "backend")) - assert(mb.identity.hierarchicalOnly) + assert(mb.identity.identityType == IdentityType.NonDeterminate) } test("StatsReceiver.dimensionalScope: composes with label correctly") { @@ -154,7 +155,7 @@ class StatsReceiverTest extends AnyFunSuite { assert(mb.identity.hierarchicalName == Seq("bar", "requests")) assert(mb.identity.dimensionalName == Seq("foo", "baz", "requests")) assert(mb.identity.labels == Map("clnt" -> "backend")) - assert(mb.identity.hierarchicalOnly) + assert(mb.identity.identityType == IdentityType.NonDeterminate) } test("StatsReceiver.scope: prefix stats by a scope string") { From f57ef78de310ccf3bd65aab854f8faed695cdaeb Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 28 Sep 2022 20:21:13 +0000 Subject: [PATCH 727/796] util: Fix flaky FuturePool ResourceTracker test Problem The ResourceTracker test in FuturePoolTest is flaky, with the tracker occasionally registering an extra update. This is caused by the `Await.result` call in the FuturePool thread occasionally adding a continuation to the Promise, which also has the resource tracker present, to tell when the task is complete. Solution Switch to using a CountdownLatch to (potentially) pause the FuturePool thread. JIRA Issues: CSL-12352 Differential Revision: https://phabricator.twitter.biz/D984397 --- .../scala/com/twitter/util/FuturePoolTest.scala | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala index d1faf6abf8..6c13d0712d 100644 --- a/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FuturePoolTest.scala @@ -17,7 +17,10 @@ class FuturePoolTest extends AnyFunSuite with Eventually { implicit override val patienceConfig: PatienceConfig = PatienceConfig(timeout = scaled(Span(15, Seconds)), interval = scaled(Span(5, Millis))) - class MockTracker(var updates: Long = 0) extends ResourceTracker { + class MockTracker extends ResourceTracker { + + @volatile var updates: Long = 0 + override def addCpuTime(time: Long): Unit = { updates = updates + 1 } @@ -29,12 +32,16 @@ class FuturePoolTest extends AnyFunSuite with Eventually { val pool = FuturePool(executor) val mockTracker = new MockTracker - val source = new Promise[Int] + val latch = new CountDownLatch(1) val result = ResourceTracker.let(mockTracker) { - pool(Await.result(source)) // simulate blocking call + pool { + // Perform a (potentially) blocking call. + latch.await() + } } - source.setValue(1); Await.result(result) + latch.countDown() + Await.result(result) assert(mockTracker.updates == 1) } } From 272d3cb1ffa9a3dc35b95ea30c606b9c0feee864 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 5 Oct 2022 20:02:00 +0000 Subject: [PATCH 728/796] util-core: Add experimental WorkQueueFiber Problem We want to experiment with a fiber that has an internal work queue but is not tied to any execution model. Solution Create that fiber. JIRA Issues: CSL-12269 Differential Revision: https://phabricator.twitter.biz/D988164 --- .../com/twitter/util/WorkQueueFiber.scala | 241 ++++++++++++ .../com/twitter/util/WorkQueueFiberTest.scala | 347 ++++++++++++++++++ 2 files changed, 588 insertions(+) create mode 100644 util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala create mode 100644 util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala diff --git a/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala new file mode 100644 index 0000000000..57bfb3c126 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala @@ -0,0 +1,241 @@ +package com.twitter.util + +import java.util.ArrayDeque +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicBoolean +import scala.annotation.tailrec + +/** + * An experimental [[Fiber]] implementation that has its own internal work queue + * + * This `Fiber` abstraction is intended to allow work associated with the fiber to have + * a common execution context while not defining the specific thread model that will execute + * this `Fiber`s work queue. + * + * The work queue is actually two work queues: one that accepts tasks submitted by this thread + * and one that is thread safe. This is to optimize for the common case where work can immediate + * finish other work and immediately trigger another task. This lets us avoid the overhead + * of using a thread safe queue in a very common case. + * + * @note that priority is given to the work submitted by this thread. + * + * @param fiberMetrics telemetry used for tracking the execution of this fiber, and likely + * other fibers as well. + */ +private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.FiberMetrics) + extends Fiber { + + fiberMetrics.fiberCreated() + + // The thread safe queue that we can submit work to + private[this] val multiThreadedQueue = new ConcurrentLinkedQueue[FiberTask] + + // Flag used to ensure that at most one thread is processing this fibers work queue + // at any given time. + private[this] val executing = new AtomicBoolean(false) + + // State used for the non-thread safe queue. This is a micro opt to make + // it so we can quickly trampoline tasks, potentially without allocating. + private[this] var r0, r1, r2: FiberTask = null + private[this] val rs = new ArrayDeque[FiberTask](0 /* effectively 8 due to impl */ ) + + // This is thread safe. Why? + // * We only care if the value stored is *this* thread. + // * A thread will only ever set the field to itself when beginning execution + // and unsets the field on exiting the submission loop. + // + // So if we see a value that is equal to `Thread.currentThread`, it must + // be *this* thread that set it, and if it is *this* thread that set it, the access has been + // single threaded all along. + private[this] var executingThread: Thread = null + + private[this] val workRunnable: Runnable = { () => startWork() } + + /** + * Submit the work to an actual thread for execution + * + * @note that we require this method call to be thread safe such that the memory updates + * from the thread that calls this method will be visible to the thread that begins + * the execution of `r`. This is generally true of thread-safe execution of `Runnable`s. + */ + protected def schedulerSubmit(r: Runnable): Unit + + final def submitTask(r: FiberTask): Unit = { + fiberMetrics.taskSubmissionIncrement() + // Check if anybody is running this already before getting a reference to the current thread. + if ((executingThread ne null) && (executingThread eq Thread.currentThread())) { + fiberMetrics.threadLocalSubmitIncrement() + threadLocalSubmit(r) + } else multiThreadedSubmit(r) + } + + /** + * If necessary, drain any thread specific state and resubmit the work queue. + * + * @note This is in service of blocking calls which are not desirable in async code + * but sometimes you gotta do what you gotta do. It should be noted that in the + * absence of `flush()` calls this fiber can guarantee that tasks are executed + * serially, eg one task is fully completed before the next is begun. However, + * a `flush()` call will necessarily shift the work potentially to a separate + * thread allowing all the code after a `flush()` call to potentially run parallel + * with other work submitted to the fiber. + */ + final def flush(): Unit = { + if (Thread.currentThread() eq executingThread) { + fiberMetrics.flushIncrement() + // We need to move all our work into a new `schedulerSubmit` call. That means moving all + // our thread local work into the thread-safe queue and resubmitting this fiber if there + // are still tasks queued. + while (localHasNext) { + multiThreadedQueue.add(localNext()) + } + + // Set this thread to non-executing and then try to re-submit ourselves, if necessary. + // Someone else might re-submit first, which is fine. + // + // Note: that we _must_ de-schedule ourselves in the case that the work we're waiting + // on belongs to this fiber but is completed later, eg, it wasn't already scheduled. + executingThread = null + executing.set(false) + + if (!multiThreadedQueue.isEmpty) { + // There is fiber work that needs to be executed. Attempt to reschedule. + + // TODO: if we end up using the ThreadLocalScheduler as our underlying execution engine + // we're going to need to be able to flush it as well. Special consideration will need + // to be taken for CPU usage tracking if that is the case. + // We can work all this in the future. + tryScheduleFiber() + } + } + } + + private[this] def threadLocalSubmit(r: FiberTask): Unit = { + if (r0 == null) r0 = r + else if (r1 == null) r1 = r + else if (r2 == null) r2 = r + else rs.addLast(r) + } + + private[this] def multiThreadedSubmit(r: FiberTask): Unit = { + multiThreadedQueue.add(r) + tryScheduleFiber() + } + + private[this] def tryScheduleFiber(): Unit = { + if (!executing.getAndSet(true)) { + // Lucky us! We're the ones that submit to the scheduler. + fiberMetrics.schedulerSubmissionIncrement() + schedulerSubmit(workRunnable) + } + } + + /** Start executing the work queue. + * + * Invoking this will execute all the tasks in both the local and multi-threaded work queues. + */ + private[this] def startWork(): Unit = { + + // This check serves two purposes. First, it guarantees that all updates to the non-thread + // safe state are visible to this thread because every time we exit the `startWork` loop we + // end by setting this state to false, and this thread reading it is a happens-before. We + // should also get that through the `schedulerSubmit` calls but this leaves no doubt. + // If it shows up as a performance bottleneck then we can consider removing it. + // + // Second, failure to see the expected state represents a very serious error: if executing + // is not true then something has gone catastrophically wrong. + if (!executing.get) { + throw new IllegalStateException("Fiber entered startWork loop with execution flag false") + } + + val currentThread = Thread.currentThread() + executingThread = currentThread + + // A thread just starting work should find two things: that the thread local queue is empty + // and that the multi-threaded queue is non-empty. + assert(!localHasNext) + assert(!multiThreadedQueue.isEmpty) + + // Expects that the first thing to do is execute a thread local task. + @tailrec def go(n: FiberTask): Unit = { + n.doRun() + // If we're not the executing thread anymore it is because a task has "flushed" us. + // If we've been flushed then we just need to exit now. The `flush()` method will + // take care of the state management. + if (executingThread eq currentThread) { + // Get our next task, if it exists. + var nn = localNext() + if (nn == null) { + nn = multiThreadedQueue.poll() + } + + if (nn != null) go(nn) + else { + // We found our local queue and multi-threaded queues both empty. We now go about + // exiting this fiber submission. + + // We set `executingThread` to null before unsetting our executing flag to ensure + // that it is always null when entering this method. Otherwise we might race between + // unsetting the flag, thread pause, another thread submits to the scheduler, + // and finally this thread gets around to unsetting. + executingThread = null + executing.set(false) + + // Now we need to double check that someone didn't submit a task and not submit + // it to the scheduler because this loop was running. + if (!multiThreadedQueue.isEmpty && !executing.getAndSet(true)) { + executingThread = currentThread + go(multiThreadedQueue.poll()) + } + } + } + } + + go(multiThreadedQueue.poll()) + } + + @inline private[this] def localHasNext: Boolean = r0 != null + + @inline private[this] def localNext(): FiberTask = { + // via moderately silly benchmarking, the + // queue unrolling gives us a ~50% speedup + // over pure Queue usage for common + // situations. + + val r = r0 + r0 = r1 + r1 = r2 + if (r2 != null) { + // If r2 was null then rs must be empty and no need to consult the + // heavy(ish) data structure. + r2 = rs.poll() + } + r + } +} + +private object WorkQueueFiber { + + /** Telemetry used to monitor [[WorkQueueFiber]]s */ + abstract class FiberMetrics { + + /** Called when a new [[WorkQueueFiber]] is created */ + def fiberCreated(): Unit + + /** Called when a new task is submitted to a [[WorkQueueFiber]] via the thread-local queue */ + def threadLocalSubmitIncrement(): Unit + + /** Called when a task is submitted to a [[WorkQueueFiber]] */ + def taskSubmissionIncrement(): Unit + + /** Called when a [[WorkQueueFiber]] is submitted for execution */ + def schedulerSubmissionIncrement(): Unit + + /** + * Called when a [[Fiber]] performs a flush + * + * @note not invoked if the flush() call is a no-op + */ + def flushIncrement(): Unit + } +} diff --git a/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala b/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala new file mode 100644 index 0000000000..f69b3457b1 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala @@ -0,0 +1,347 @@ +package com.twitter.util + +import com.twitter.conversions.DurationOps.richDurationFromInt +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicInteger +import org.scalatest.funsuite.AnyFunSuite +import scala.collection.mutable.ListBuffer + +class WorkQueueFiberTest extends AnyFunSuite { + + private def await[T](t: Awaitable[T]): T = Await.result(t, 10.seconds) + + private class TestMetrics extends WorkQueueFiber.FiberMetrics { + + val fiberFlushed, fibersCreated, threadLocalSubmits, taskSubmissions, schedulerSubmissions = + new AtomicInteger() + + def flushIncrement(): Unit = fiberFlushed.incrementAndGet() + + def fiberCreated(): Unit = fibersCreated.incrementAndGet() + def threadLocalSubmitIncrement(): Unit = threadLocalSubmits.incrementAndGet() + def taskSubmissionIncrement(): Unit = taskSubmissions.incrementAndGet() + def schedulerSubmissionIncrement(): Unit = schedulerSubmissions.incrementAndGet() + } + + private class TestFiber(metrics: TestMetrics) extends WorkQueueFiber(metrics) { + + var runnable: Runnable = null + + def executePendingRunnable(): Unit = { + assert(runnable != null) + val r = runnable + runnable = null + r.run() + } + + protected def schedulerSubmit(r: Runnable): Unit = { + assert(runnable == null) + runnable = r + } + } + + test("fiber creations are tracked") { + val N = 5 + val metrics = new TestMetrics + for (_ <- 0 until N) { + new TestFiber(metrics) + } + assert(metrics.fibersCreated.get == N) + } + + test("submit multiple tasks while fiber not running") { + val metrics = new TestMetrics + val f = new TestFiber(metrics) + + assert(metrics.fibersCreated.get == 1) + + val executed = new AtomicInteger() + + val N = 5 + for (_ <- 0 until N) { + f.submitTask { () => + executed.incrementAndGet() + } + } + + assert(f.runnable != null) + assert(metrics.threadLocalSubmits.get == 0) + assert(metrics.taskSubmissions.get == N) + assert(metrics.schedulerSubmissions.get == 1) + assert(executed.get == 0) + + f.executePendingRunnable() + assert(executed.get == N) + } + + test("submit a thread local task") { + val metrics = new TestMetrics + val f = new TestFiber(metrics) + + assert(metrics.fibersCreated.get == 1) + + val executed = new AtomicInteger() + val submissionOrder = new ListBuffer[Int] + + f.submitTask { () => + executed.incrementAndGet() + + // This one should be submitted thread-locally. + f.submitTask { () => + executed.incrementAndGet() + submissionOrder += 1 + } + + // fiber task 1 should complete all the way through before we execute the sub-task + submissionOrder += 0 + } + + assert(f.runnable != null) + assert(metrics.threadLocalSubmits.get == 0) + assert(metrics.taskSubmissions.get == 1) + assert(metrics.schedulerSubmissions.get == 1) + assert(executed.get == 0) + + f.executePendingRunnable() + assert(metrics.threadLocalSubmits.get == 1) + assert(metrics.taskSubmissions.get == 2) + assert(executed.get == 2) + assert(submissionOrder.result() == List(0, 1)) + + // Now throw in one more to make sure we don't get a thread-local submit since the fiber + // is clearly not executing. + f.submitTask { () => + executed.incrementAndGet() + submissionOrder += 2 + } + + assert(metrics.threadLocalSubmits.get == 1) + assert(metrics.taskSubmissions.get == 3) + assert(metrics.schedulerSubmissions.get == 2) + f.executePendingRunnable() + + assert(executed.get == 3) + assert(submissionOrder.result() == List(0, 1, 2)) + } + + test("submit a task while a separate thread is executing the fiber") { + val metrics = new TestMetrics + val f = new TestFiber(metrics) + + assert(metrics.fibersCreated.get == 1) + + val taskAwait, taskExecuting = new CountDownLatch(1) + + f.submitTask { () => + taskExecuting.countDown() + taskAwait.await() + } + + assert(f.runnable != null) + assert(metrics.threadLocalSubmits.get == 0) + assert(metrics.taskSubmissions.get == 1) + assert(metrics.schedulerSubmissions.get == 1) + + // Start the fiber in a separate thread + val t = new Thread({ () => + f.executePendingRunnable() + }) + t.start() + + taskExecuting.await() + + val executingThread = new Promise[Thread]() + f.submitTask { () => + executingThread.setValue(Thread.currentThread) + } + + assert(metrics.schedulerSubmissions.get == 1) + assert(metrics.taskSubmissions.get == 2) + assert(metrics.threadLocalSubmits.get == 0) + assert(!executingThread.isDefined) + + taskAwait.countDown() + assert(await(executingThread) == t) + } + + test("flushing re-submits to the scheduler") { + val metrics = new TestMetrics + val f = new TestFiber(metrics) + + val latch1, latch2, latch3, flushed = new CountDownLatch(1) + + // This will eventually be the test thread so we don't need it to be volatile. + var threadLocalSubmissionsExecutionThread: Thread = null + var lastTaskDone = false + f.submitTask { () => + latch1.countDown() + latch2.await() + f.submitTask { () => + threadLocalSubmissionsExecutionThread = Thread.currentThread() + } + + // Now flush and continue to suspend the thread. + assert(metrics.fiberFlushed.get == 0) + f.flush() + assert(metrics.fiberFlushed.get == 1) + flushed.countDown() + latch3.await() + + // schedule one more task that should not be a thread local submit because flush + // voided that. + f.submitTask { () => + lastTaskDone = true + } + } + + assert(metrics.taskSubmissions.get == 1) + + val t = new Thread({ () => + f.executePendingRunnable() + }) + t.start() + + latch1.await() + + // We'll run this one in the test thread but it's going to be submitted as a non-thread local + var done = false + f.submitTask { () => + done = true + } + + // Release thread t and then stall t right after the flush + latch2.countDown() + flushed.await() + + assert(metrics.taskSubmissions.get == 3) + assert(metrics.threadLocalSubmits.get == 1) + assert(metrics.schedulerSubmissions.get == 2) + assert(threadLocalSubmissionsExecutionThread == null) + assert(!done) + + // We should now have 2 tasks awaiting whoever executes the next submission. Now run it. + f.executePendingRunnable() + assert(threadLocalSubmissionsExecutionThread == Thread.currentThread()) + assert(done) + + // Now let our thread submit it's final task which shouldn't be a thread local task even + // though it's submitted from within a FiberTask. + latch3.countDown() + t.join() + + assert(metrics.taskSubmissions.get == 4) + assert(metrics.threadLocalSubmits.get == 1) + assert(metrics.schedulerSubmissions.get == 3) + assert(!lastTaskDone) + + f.executePendingRunnable() + assert(lastTaskDone) + } + + test("flushing to a recursive scheduler") { + val metrics = new TestMetrics + val f = new WorkQueueFiber(metrics) { + override protected def schedulerSubmit(r: Runnable): Unit = { + r.run() + } + } + + f.submitTask { () => + assert(metrics.taskSubmissions.get == 1) + assert(metrics.schedulerSubmissions.get == 1) + assert(metrics.threadLocalSubmits.get == 0) + + var threadLocalSubmit = false + f.submitTask { () => + threadLocalSubmit = true + } + assert(metrics.taskSubmissions.get == 2) + assert(metrics.schedulerSubmissions.get == 1) + assert(metrics.threadLocalSubmits.get == 1) + + assert(!threadLocalSubmit) + assert(metrics.fiberFlushed.get == 0) + f.flush() + assert(metrics.fiberFlushed.get == 1) + assert(threadLocalSubmit) + assert(metrics.taskSubmissions.get == 2) + assert(metrics.schedulerSubmissions.get == 2) + assert(metrics.threadLocalSubmits.get == 1) + + // After the flush() call, even recursive calls should act like an external submission. + var shouldExecuteNow = false + f.submitTask { () => + shouldExecuteNow = true + } + assert(shouldExecuteNow) + assert(metrics.taskSubmissions.get == 3) + assert(metrics.schedulerSubmissions.get == 3) + assert(metrics.threadLocalSubmits.get == 1) + } + } + + test("thread-local flush with empty work queues still de-schedules itself") { + val metrics = new TestMetrics + val f = new TestFiber(metrics) + + var executed = false + f.submitTask { () => + assert(metrics.taskSubmissions.get == 1) + assert(metrics.schedulerSubmissions.get == 1) + assert(metrics.threadLocalSubmits.get == 0) + assert(metrics.fiberFlushed.get == 0) + + f.flush() + + // At this point we should be as if we're not actually executing in the fiber. + + assert(metrics.taskSubmissions.get == 1) + assert(metrics.schedulerSubmissions.get == 1) + assert(metrics.threadLocalSubmits.get == 0) + assert(metrics.fiberFlushed.get == 1) + + // This will get added to the end of the queue and we'll run it without + // needing a resubmit. + f.submitTask { () => + executed = true + } + + assert(!executed) + assert(metrics.taskSubmissions.get == 2) + assert(metrics.schedulerSubmissions.get == 2) + assert(metrics.threadLocalSubmits.get == 0) + assert(metrics.fiberFlushed.get == 1) + } + + f.executePendingRunnable() + // Still not executed. Even though the submission was done inside a FiberTask + // the flush() had caused the task to no longer see itself as the executor of + // the fiber. + assert(!executed) + + f.executePendingRunnable() + assert(executed) + } + + test("a flush outside the executing code is a no-op") { + val metrics = new TestMetrics + val f = new TestFiber(metrics) + + var executed = false + f.submitTask { () => + executed = true + } + + assert(metrics.taskSubmissions.get == 1) + assert(metrics.schedulerSubmissions.get == 1) + assert(metrics.threadLocalSubmits.get == 0) + assert(metrics.fiberFlushed.get == 0) + + f.flush() + assert(metrics.fiberFlushed.get == 0) + assert(!executed) + + f.executePendingRunnable() + assert(executed) + } +} From b65c8296c131afca6869a43a4bfa259a146fdf27 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 7 Oct 2022 14:07:48 +0000 Subject: [PATCH 729/796] util-hashing: migrate tests off of WordSpec Problem: Our recommendation for testing is to use AnyFunSuite, but our own testing uses AnyWordSpec Solution: Migrate JIRA Issues: CSL-9463 Differential Revision: https://phabricator.twitter.biz/D990567 --- .../ConsistentHashingDistributorTest.scala | 185 +++++++++--------- .../com/twitter/hashing/DiagnosticsTest.scala | 60 +++--- .../com/twitter/hashing/KeyHasherTest.scala | 41 ++-- 3 files changed, 141 insertions(+), 145 deletions(-) diff --git a/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala index bed6f207f3..8a2f2bc833 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/ConsistentHashingDistributorTest.scala @@ -3,117 +3,118 @@ package com.twitter.hashing import org.scalacheck.Gen import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable -import java.io.{BufferedReader, InputStreamReader} -import java.nio.{ByteBuffer, ByteOrder} +import java.io.BufferedReader +import java.io.InputStreamReader +import java.nio.ByteBuffer +import java.nio.ByteOrder import java.security.MessageDigest -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class ConsistentHashingDistributorTest extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { - "KetamaDistributor" should { - val nodes = Seq( - HashNode("10.0.1.1", 600, 1), - HashNode("10.0.1.2", 300, 2), - HashNode("10.0.1.3", 200, 3), - HashNode("10.0.1.4", 350, 4), - HashNode("10.0.1.5", 1000, 5), - HashNode("10.0.1.6", 800, 6), - HashNode("10.0.1.7", 950, 7), - HashNode("10.0.1.8", 100, 8) - ) +class ConsistentHashingDistributorTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { + val nodes = Seq( + HashNode("10.0.1.1", 600, 1), + HashNode("10.0.1.2", 300, 2), + HashNode("10.0.1.3", 200, 3), + HashNode("10.0.1.4", 350, 4), + HashNode("10.0.1.5", 1000, 5), + HashNode("10.0.1.6", 800, 6), + HashNode("10.0.1.7", 950, 7), + HashNode("10.0.1.8", 100, 8) + ) - // 160 is the hard coded value for libmemcached, which was this input data is from - val ketamaDistributor = new ConsistentHashingDistributor(nodes, 160) - val ketamaDistributorInoldLibMemcachedVersionComplianceMode = - new ConsistentHashingDistributor(nodes, 160, true) + // 160 is the hard coded value for libmemcached, which was this input data is from + val ketamaDistributor = new ConsistentHashingDistributor(nodes, 160) + val ketamaDistributorInoldLibMemcachedVersionComplianceMode = + new ConsistentHashingDistributor(nodes, 160, true) - "pick the correct node with ketama hash function" in { - // Test from Smile's KetamaNodeLocatorSpec.scala + test("KetamaDistributor should pick the correct node with ketama hash function") { + // Test from Smile's KetamaNodeLocatorSpec.scala - // Load known good results (key, hash(?), continuum ceiling(?), IP) - val stream = getClass.getClassLoader.getResourceAsStream("ketama_results") - val reader = new BufferedReader(new InputStreamReader(stream)) - val expected = new mutable.ListBuffer[Array[String]] - var line: String = reader.readLine() - while (line != null) { - val segments = line.split(" ") - assert(segments.length == 4) - expected += segments - line = reader.readLine() - } - assert(expected.size == 99) + // Load known good results (key, hash(?), continuum ceiling(?), IP) + val stream = getClass.getClassLoader.getResourceAsStream("ketama_results") + val reader = new BufferedReader(new InputStreamReader(stream)) + val expected = new mutable.ListBuffer[Array[String]] + var line: String = reader.readLine() + while (line != null) { + val segments = line.split(" ") + assert(segments.length == 4) + expected += segments + line = reader.readLine() + } + assert(expected.size == 99) - // Test that ketamaClient.clientOf(key) == expected IP - val handleToIp = nodes.map { n => n.handle -> n.identifier }.toMap - for (testcase <- expected) { - val hash = KeyHasher.KETAMA.hashKey(testcase(0).getBytes) + // Test that ketamaClient.clientOf(key) == expected IP + val handleToIp = nodes.map { n => n.handle -> n.identifier }.toMap + for (testcase <- expected) { + val hash = KeyHasher.KETAMA.hashKey(testcase(0).getBytes) - val handle = ketamaDistributor.nodeForHash(hash) - val handle2 = ketamaDistributorInoldLibMemcachedVersionComplianceMode.nodeForHash(hash) - val resultIp = handleToIp(handle) - val resultIp2 = handleToIp(handle2) - assert(testcase(3) == resultIp) - assert(testcase(3) == resultIp2) - } + val handle = ketamaDistributor.nodeForHash(hash) + val handle2 = ketamaDistributorInoldLibMemcachedVersionComplianceMode.nodeForHash(hash) + val resultIp = handleToIp(handle) + val resultIp2 = handleToIp(handle2) + assert(testcase(3) == resultIp) + assert(testcase(3) == resultIp2) } + } - "pick the correct node with 64-bit hash values" in { - val knownGoodValues = Map( - -166124121512512L -> 5, - 8796093022208L -> 3, - 4312515125124L -> 2, - -8192481414141L -> 1, - -9515121512312L -> 5 - ) + test("KetamaDistributor should pick the correct node with 64-bit hash values") { + val knownGoodValues = Map( + -166124121512512L -> 5, + 8796093022208L -> 3, + 4312515125124L -> 2, + -8192481414141L -> 1, + -9515121512312L -> 5 + ) - knownGoodValues foreach { - case (key, node) => - val handle = ketamaDistributor.nodeForHash(key) - assert(handle == node) - } + knownGoodValues foreach { + case (key, node) => + val handle = ketamaDistributor.nodeForHash(key) + assert(handle == node) } + } - "hashInt" in { - val ketama = new ConsistentHashingDistributor[Unit](Seq.empty, 0, false) - forAll(Gen.chooseNum(0, Int.MaxValue)) { i => - val md = MessageDigest.getInstance("MD5") - ketama.hashInt(i, md) - val array = md.digest() - assert(array.toSeq == md.digest(i.toString.getBytes("UTF-8")).toSeq) - } + test("KetamaDistributor should hashInt") { + val ketama = new ConsistentHashingDistributor[Unit](Seq.empty, 0, false) + forAll(Gen.chooseNum(0, Int.MaxValue)) { i => + val md = MessageDigest.getInstance("MD5") + ketama.hashInt(i, md) + val array = md.digest() + assert(array.toSeq == md.digest(i.toString.getBytes("UTF-8")).toSeq) } + } - "byteArrayToLE" in { - val ketama = new ConsistentHashingDistributor[Unit](Seq.empty, 0, false) - forAll { (s: String) => - val ba = MessageDigest.getInstance("MD5").digest(s.getBytes("UTF-8")) - val bb = ByteBuffer.wrap(ba).order(ByteOrder.LITTLE_ENDIAN) + test("KetamaDistributor should byteArrayToLE") { + val ketama = new ConsistentHashingDistributor[Unit](Seq.empty, 0, false) + forAll { (s: String) => + val ba = MessageDigest.getInstance("MD5").digest(s.getBytes("UTF-8")) + val bb = ByteBuffer.wrap(ba).order(ByteOrder.LITTLE_ENDIAN) - assert(ketama.byteArrayToLE(ba, 0) == bb.getInt(0)) - assert(ketama.byteArrayToLE(ba, 4) == bb.getInt(4)) - assert(ketama.byteArrayToLE(ba, 8) == bb.getInt(8)) - assert(ketama.byteArrayToLE(ba, 12) == bb.getInt(12)) - } + assert(ketama.byteArrayToLE(ba, 0) == bb.getInt(0)) + assert(ketama.byteArrayToLE(ba, 4) == bb.getInt(4)) + assert(ketama.byteArrayToLE(ba, 8) == bb.getInt(8)) + assert(ketama.byteArrayToLE(ba, 12) == bb.getInt(12)) } + } - "partitionIdForHash is the same as entryForHash 1st tuple element" in { - val keyHasher = KeyHasher.MURMUR3 + test( + "KetamaDistributor should partitionIdForHash is the same as entryForHash 1st tuple element") { + val keyHasher = KeyHasher.MURMUR3 - { - // Special case this unicode string since this test has failed - // on it in the past. - val s = "ퟲ狂✟풻棇埲蟂덧➀缘็滀佳黟韕숻᩿볯箿䜐㫌홻ᖛ磌ᡎ油" - val hash = keyHasher.hashKey(s.getBytes("UTF-8")) - val (pid1, _) = ketamaDistributor.entryForHash(hash) - val pid2 = ketamaDistributor.partitionIdForHash(hash) - assert(pid1 == pid2) - } + { + // Special case this unicode string since this test has failed + // on it in the past. + val s = "ퟲ狂✟풻棇埲蟂덧➀缘็滀佳黟韕숻᩿볯箿䜐㫌홻ᖛ磌ᡎ油" + val hash = keyHasher.hashKey(s.getBytes("UTF-8")) + val (pid1, _) = ketamaDistributor.entryForHash(hash) + val pid2 = ketamaDistributor.partitionIdForHash(hash) + assert(pid1 == pid2) + } - forAll { (s: String) => - val hash = keyHasher.hashKey(s.getBytes("UTF-8")) - val (pid1, _) = ketamaDistributor.entryForHash(hash) - val pid2 = ketamaDistributor.partitionIdForHash(hash) - assert(pid1 == pid2) - } + forAll { (s: String) => + val hash = keyHasher.hashKey(s.getBytes("UTF-8")) + val (pid1, _) = ketamaDistributor.entryForHash(hash) + val pid2 = ketamaDistributor.partitionIdForHash(hash) + assert(pid1 == pid2) } } } diff --git a/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala index 2c679128a6..8360613ab2 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/DiagnosticsTest.scala @@ -1,39 +1,37 @@ package com.twitter.hashing -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class DiagnosticsTest extends AnyWordSpec { - "Diagnostics" should { - "print distribution" in { - val hosts = 1 until 500 map { "10.1.1." + _ + ":11211:4" } +class DiagnosticsTest extends AnyFunSuite { + test("Diagnostics should print distribution") { + val hosts = 1 until 500 map { "10.1.1." + _ + ":11211:4" } - val nodes = hosts.map { s => - val Array(host, port, weight) = s.split(":") - val identifier = host + ":" + port - HashNode(identifier, weight.toInt, identifier) - } + val nodes = hosts.map { s => + val Array(host, port, weight) = s.split(":") + val identifier = host + ":" + port + HashNode(identifier, weight.toInt, identifier) + } - val hashFunctions = List( - "FNV1_32" -> KeyHasher.FNV1_32, - "FNV1A_32" -> KeyHasher.FNV1A_32, - "FNV1_64" -> KeyHasher.FNV1_64, - "FNV1A_64" -> KeyHasher.FNV1A_64, - "CRC32-ITU" -> KeyHasher.CRC32_ITU, - "HSIEH" -> KeyHasher.HSIEH, - "JENKINS" -> KeyHasher.JENKINS, - "MURMUR3" -> KeyHasher.MURMUR3 - ) + val hashFunctions = List( + "FNV1_32" -> KeyHasher.FNV1_32, + "FNV1A_32" -> KeyHasher.FNV1A_32, + "FNV1_64" -> KeyHasher.FNV1_64, + "FNV1A_64" -> KeyHasher.FNV1A_64, + "CRC32-ITU" -> KeyHasher.CRC32_ITU, + "HSIEH" -> KeyHasher.HSIEH, + "JENKINS" -> KeyHasher.JENKINS, + "MURMUR3" -> KeyHasher.MURMUR3 + ) - val keys = (1 until 1000000).map(_.toString).toList - // Comment this out unless you're running it results - // hashFunctions foreach { case (s, h) => - // val distributor = new KetamaDistributor(nodes, 160) - // val tester = new DistributionTester(distributor) - // val start = Time.now - // val dev = tester.distributionDeviation(keys map { s => h.hashKey(s.getBytes) }) - // val duration = (Time.now - start).inMilliseconds - // println("%s\n distribution: %.5f\n duration: %dms\n".format(s, dev, duration)) - // } - } + val keys = (1 until 1000000).map(_.toString).toList + // Comment this out unless you're running it results + // hashFunctions foreach { case (s, h) => + // val distributor = new KetamaDistributor(nodes, 160) + // val tester = new DistributionTester(distributor) + // val start = Time.now + // val dev = tester.distributionDeviation(keys map { s => h.hashKey(s.getBytes) }) + // val duration = (Time.now - start).inMilliseconds + // println("%s\n distribution: %.5f\n duration: %dms\n".format(s, dev, duration)) + // } } } diff --git a/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala b/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala index 2aff9d868d..57569ae3e4 100644 --- a/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala +++ b/util-hashing/src/test/scala/com/twitter/hashing/KeyHasherTest.scala @@ -4,9 +4,9 @@ import com.twitter.io.TempFile import java.util.Base64 import scala.collection.mutable.ListBuffer import java.nio.charset.StandardCharsets.UTF_8 -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class KeyHasherTest extends AnyWordSpec { +class KeyHasherTest extends AnyFunSuite { def readResource(name: String) = { var lines = new ListBuffer[String]() val src = scala.io.Source.fromFile(TempFile.fromResourcePath(getClass, "/" + name)) @@ -28,30 +28,27 @@ class KeyHasherTest extends AnyWordSpec { } } - "KeyHasher" should { - "correctly hash fnv1_32" in { - testHasher("fnv1_32", KeyHasher.FNV1_32) - } - - "correctly hash fnv1_64" in { - testHasher("fnv1_64", KeyHasher.FNV1_64) - } + test("KeyHasher should correctly hash fnv1_32") { + testHasher("fnv1_32", KeyHasher.FNV1_32) + } - "correctly hash fnv1a_32" in { - testHasher("fnv1a_32", KeyHasher.FNV1A_32) - } + test("KeyHasher should correctly hash fnv1_64") { + testHasher("fnv1_64", KeyHasher.FNV1_64) + } - "correctly hash fnv1a_64" in { - testHasher("fnv1a_64", KeyHasher.FNV1A_64) - } + test("KeyHasher should correctly hash fnv1a_32") { + testHasher("fnv1a_32", KeyHasher.FNV1A_32) + } - "correctly hash jenkins" in { - testHasher("jenkins", KeyHasher.JENKINS) - } + test("KeyHasher should correctly hash fnv1a_64") { + testHasher("fnv1a_64", KeyHasher.FNV1A_64) + } - "correctly hash crc32 itu" in { - testHasher("crc32", KeyHasher.CRC32_ITU) - } + test("KeyHasher should correctly hash jenkins") { + testHasher("jenkins", KeyHasher.JENKINS) + } + test("KeyHasher should correctly hash crc32 itu") { + testHasher("crc32", KeyHasher.CRC32_ITU) } } From 8bcdb1f7a66e10dd0a7058e939ecbdbe42135b8e Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 7 Oct 2022 14:09:39 +0000 Subject: [PATCH 730/796] util-jvm: migrate tests off of WordSpec Problem: Our recommendation for testing is to use AnyFunSuite, but our own testing uses AnyWordSpec Solution: Migrate JIRA Issues: CSL-9463 Differential Revision: https://phabricator.twitter.biz/D989994 --- .../test/scala/com/twitter/jvm/JvmTest.scala | 278 +++++++++--------- 1 file changed, 137 insertions(+), 141 deletions(-) diff --git a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala index 59e9b27cc8..02c1125c84 100644 --- a/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala +++ b/util-jvm/src/test/scala/com/twitter/jvm/JvmTest.scala @@ -2,160 +2,156 @@ package com.twitter.jvm import com.twitter.conversions.DurationOps._ import com.twitter.util.Time -import java.util.logging.{Level, Logger} +import java.util.logging.Level +import java.util.logging.Logger import org.mockito.ArgumentMatchers.contains -import org.mockito.Mockito.{times, verify, when} +import org.mockito.Mockito.times +import org.mockito.Mockito.verify +import org.mockito.Mockito.when import org.scalatestplus.mockito.MockitoSugar import scala.collection.mutable -import org.scalatest.wordspec.AnyWordSpec - -class JvmTest extends AnyWordSpec with MockitoSugar { - "Jvm" should { - class JvmHelper(suppliedLogger: Option[Logger] = None) extends Jvm { - protected override def logger: Logger = suppliedLogger match { - case Some(l) => l - case None => super.logger - } - - @volatile private[this] var currentSnap: Snapshot = - Snapshot(Time.epoch, Heap(0, 0, Seq()), Seq()) - - val opts = new Opts { - def compileThresh = None - } - def snapCounters = Map() - def setSnap(snap: Snapshot): Unit = { - currentSnap = snap - } - - override val executor: MockScheduledExecutorService = new MockScheduledExecutorService - - def snap = currentSnap - - def pushGc(gc: Gc): Unit = { - val gcs = snap.lastGcs filter (_.name != gc.name) - setSnap(snap.copy(lastGcs = gc +: gcs)) - } - - def forceGc() = () - def edenPool = NilJvm.edenPool - def metaspaceUsage = NilJvm.metaspaceUsage - def safepoint = NilJvm.safepoint - def applicationTime = NilJvm.applicationTime - def tenuringThreshold = NilJvm.tenuringThreshold +import org.scalatest.funsuite.AnyFunSuite + +class JvmTest extends AnyFunSuite with MockitoSugar { + class JvmHelper(suppliedLogger: Option[Logger] = None) extends Jvm { + protected override def logger: Logger = suppliedLogger match { + case Some(l) => l + case None => super.logger + } + + @volatile private[this] var currentSnap: Snapshot = + Snapshot(Time.epoch, Heap(0, 0, Seq()), Seq()) + + val opts = new Opts { + def compileThresh = None + } + def snapCounters = Map() + def setSnap(snap: Snapshot): Unit = { + currentSnap = snap } - "ProcessId" should { - val supported = Seq("Mac OS X", "Linux") + override val executor: MockScheduledExecutorService = new MockScheduledExecutorService - val osIsSupported = - Option(System.getProperty("os.name")).exists(supported.contains) + def snap = currentSnap - if (osIsSupported) { - "define process id on supported platforms" in { - assert(Jvm.ProcessId.isDefined) - } - } + def pushGc(gc: Gc): Unit = { + val gcs = snap.lastGcs filter (_.name != gc.name) + setSnap(snap.copy(lastGcs = gc +: gcs)) } - "foreachGc" should { - "Capture interleaving GCs with different names" in { - val jvm = new JvmHelper() - val b = mutable.Buffer[Gc]() - assert(jvm.executor.schedules == List()) - jvm foreachGc { b += _ } - assert(jvm.executor.schedules.size == 1) - val r = jvm.executor.schedules.head._1 - r.run() - assert(b == List()) - val gc = Gc(0, "pcopy", Time.now, 1.millisecond) - jvm.pushGc(gc) - r.run() - assert(b.size == 1) - assert(b(0) == gc) - r.run() - assert(b.size == 1) - val gc1 = gc.copy(name = "CMS") - jvm.pushGc(gc1) - r.run() - assert(b.size == 2) - assert(b(1) == gc1) - r.run() - r.run() - r.run() - assert(b.size == 2) - jvm.pushGc(gc1.copy(count = 1)) - jvm.pushGc(gc.copy(count = 1)) - r.run() - assert(b.size == 4) - assert(b(2) == gc.copy(count = 1)) - assert(b(3) == gc1.copy(count = 1)) - } - - "Complain when sampling rate is too low, every 30 minutes" in Time.withCurrentTimeFrozen { - tc => - val logger = mock[Logger] - when(logger.isLoggable(Level.FINE)).thenReturn(true) - val jvm = new JvmHelper(Some(logger)) - - jvm.foreachGc(_ => () /*ignore*/ ) - assert(jvm.executor.schedules.size == 1) - val r = jvm.executor.schedules.head._1 - val gc = Gc(0, "pcopy", Time.now, 1.millisecond) - r.run() - jvm.pushGc(gc) - r.run() - jvm.pushGc(gc.copy(count = 2)) - r.run() - verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) - jvm.pushGc(gc.copy(count = 10)) - verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) - r.run() - tc.advance(29.minutes) - r.run() - verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) - tc.advance(2.minutes) - jvm.pushGc(gc.copy(count = 12)) - r.run() - verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) - verify(logger, times(1)).fine(contains("Missed 8 collections for pcopy due to sampling")) - } + def forceGc() = () + def edenPool = NilJvm.edenPool + def metaspaceUsage = NilJvm.metaspaceUsage + def safepoint = NilJvm.safepoint + def applicationTime = NilJvm.applicationTime + def tenuringThreshold = NilJvm.tenuringThreshold + } + + test("ProcessId should define processid on supported platforms") { + val supported = Seq("Mac OS X", "Linux") + + val osIsSupported = + Option(System.getProperty("os.name")).exists(supported.contains) + + if (osIsSupported) { + assert(Jvm.ProcessId.isDefined) } + } + + test("foreachGc should capture interleaving GCs with different names") { + val jvm = new JvmHelper() + val b = mutable.Buffer[Gc]() + assert(jvm.executor.schedules == List()) + jvm foreachGc { b += _ } + assert(jvm.executor.schedules.size == 1) + val r = jvm.executor.schedules.head._1 + r.run() + assert(b == List()) + val gc = Gc(0, "pcopy", Time.now, 1.millisecond) + jvm.pushGc(gc) + r.run() + assert(b.size == 1) + assert(b(0) == gc) + r.run() + assert(b.size == 1) + val gc1 = gc.copy(name = "CMS") + jvm.pushGc(gc1) + r.run() + assert(b.size == 2) + assert(b(1) == gc1) + r.run() + r.run() + r.run() + assert(b.size == 2) + jvm.pushGc(gc1.copy(count = 1)) + jvm.pushGc(gc.copy(count = 1)) + r.run() + assert(b.size == 4) + assert(b(2) == gc.copy(count = 1)) + assert(b(3) == gc1.copy(count = 1)) + } - "monitorsGcs" should { - "queries gcs in range, in reverse chronological order" in Time.withCurrentTimeFrozen { tc => - val jvm = new JvmHelper() - val query = jvm.monitorGcs(10.seconds) - assert(jvm.executor.schedules.size == 1) - val r = jvm.executor.schedules.head._1 - val gc0 = Gc(0, "pcopy", Time.now, 1.millisecond) - val gc1 = Gc(1, "CMS", Time.now, 1.millisecond) - jvm.pushGc(gc1) - jvm.pushGc(gc0) - r.run() - tc.advance(9.seconds) - val gc2 = Gc(1, "pcopy", Time.now, 2.milliseconds) - jvm.pushGc(gc2) - r.run() - assert(query(10.seconds.ago) == Seq(gc2, gc1, gc0)) - tc.advance(2.seconds) - val gc3 = Gc(2, "CMS", Time.now, 1.second) - jvm.pushGc(gc3) - r.run() - assert(query(10.seconds.ago) == Seq(gc3, gc2)) - } + test("foreachGc should complain when sampling rate is too low, every 30 minutes") { + Time.withCurrentTimeFrozen { tc => + val logger = mock[Logger] + when(logger.isLoggable(Level.FINE)).thenReturn(true) + val jvm = new JvmHelper(Some(logger)) + + jvm.foreachGc(_ => () /*ignore*/ ) + assert(jvm.executor.schedules.size == 1) + val r = jvm.executor.schedules.head._1 + val gc = Gc(0, "pcopy", Time.now, 1.millisecond) + r.run() + jvm.pushGc(gc) + r.run() + jvm.pushGc(gc.copy(count = 2)) + r.run() + verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) + jvm.pushGc(gc.copy(count = 10)) + verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) + r.run() + tc.advance(29.minutes) + r.run() + verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) + tc.advance(2.minutes) + jvm.pushGc(gc.copy(count = 12)) + r.run() + verify(logger, times(1)).fine(contains("Missed 1 collections for pcopy due to sampling")) + verify(logger, times(1)).fine(contains("Missed 8 collections for pcopy due to sampling")) } + } - "safepoint" should { - "show an increase in total time spent after a gc" in { - val j = Jvm() - val preGc = j.safepoint - val totalTimePreGc = preGc.totalTimeMillis - System.gc() - val postGc = j.safepoint - val totalTimePostGc = postGc.totalTimeMillis - assert(totalTimePostGc > totalTimePreGc) - } + test("monitorsGcs queries gcs in range, in reverse chronological order") { + Time.withCurrentTimeFrozen { tc => + val jvm = new JvmHelper() + val query = jvm.monitorGcs(10.seconds) + assert(jvm.executor.schedules.size == 1) + val r = jvm.executor.schedules.head._1 + val gc0 = Gc(0, "pcopy", Time.now, 1.millisecond) + val gc1 = Gc(1, "CMS", Time.now, 1.millisecond) + jvm.pushGc(gc1) + jvm.pushGc(gc0) + r.run() + tc.advance(9.seconds) + val gc2 = Gc(1, "pcopy", Time.now, 2.milliseconds) + jvm.pushGc(gc2) + r.run() + assert(query(10.seconds.ago) == Seq(gc2, gc1, gc0)) + tc.advance(2.seconds) + val gc3 = Gc(2, "CMS", Time.now, 1.second) + jvm.pushGc(gc3) + r.run() + assert(query(10.seconds.ago) == Seq(gc3, gc2)) } } + + test("safepoint should show an increase in total time spent after a gc") { + val j = Jvm() + val preGc = j.safepoint + val totalTimePreGc = preGc.totalTimeMillis + System.gc() + val postGc = j.safepoint + val totalTimePostGc = postGc.totalTimeMillis + assert(totalTimePostGc > totalTimePreGc) + } } From 92528c15281099ad5b3c1f5a87949e30034fd005 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 7 Oct 2022 14:11:45 +0000 Subject: [PATCH 731/796] util-core: migrate tests off of WordSpec Problem: Our recommendation for testing is to use AnyFunSuite, but our own testing uses AnyWordSpec Solution: Migrate JIRA Issues: CSL-9463 Differential Revision: https://phabricator.twitter.biz/D991007 --- .../com/twitter/concurrent/BrokerTest.scala | 204 +- .../com/twitter/concurrent/OfferTest.scala | 586 ++- .../twitter/concurrent/SerializedTest.scala | 56 +- .../twitter/concurrent/SpoolSourceTest.scala | 177 +- .../com/twitter/concurrent/SpoolTest.scala | 723 ++-- .../scala/com/twitter/concurrent/TxTest.scala | 116 +- .../test/scala/com/twitter/io/FilesTest.scala | 9 +- .../scala/com/twitter/io/StreamIOTest.scala | 33 +- .../com/twitter/io/TempDirectoryTest.scala | 26 +- .../scala/com/twitter/io/TempFileTest.scala | 48 +- .../scala/com/twitter/util/DurationTest.scala | 479 ++- .../com/twitter/util/FutureOfferTest.scala | 35 +- .../scala/com/twitter/util/FutureTest.scala | 3130 ++++++++--------- .../twitter/util/LastWriteWinsQueueTest.scala | 36 +- .../scala/com/twitter/util/NetUtilTest.scala | 257 +- .../scala/com/twitter/util/PoolTest.scala | 106 +- .../com/twitter/util/StateMachineTest.scala | 52 +- .../scala/com/twitter/util/TimeTest.scala | 1150 +++--- .../twitter/util/TwitterDateFormatTest.scala | 61 +- 19 files changed, 3572 insertions(+), 3712 deletions(-) diff --git a/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala b/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala index 34ea24cb50..00a7c674fd 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/BrokerTest.scala @@ -1,116 +1,114 @@ package com.twitter.concurrent -import com.twitter.util.{Await, Return, ObjectSizeCalculator} -import org.scalatest.wordspec.AnyWordSpec - -class BrokerTest extends AnyWordSpec { - "Broker" should { - "send data (send, recv)" in { - val br = new Broker[Int] - val sendF = br.send(123).sync() - assert(sendF.isDefined == false) - val recvF = br.recv.sync() - assert(recvF.isDefined == true) - assert(Await.result(recvF) == 123) - assert(sendF.isDefined == true) - } +import com.twitter.util.Await +import com.twitter.util.ObjectSizeCalculator +import com.twitter.util.Return +import org.scalatest.funsuite.AnyFunSuite + +class BrokerTest extends AnyFunSuite { + test("Broker should send data (send, recv)") { + val br = new Broker[Int] + val sendF = br.send(123).sync() + assert(sendF.isDefined == false) + val recvF = br.recv.sync() + assert(recvF.isDefined == true) + assert(Await.result(recvF) == 123) + assert(sendF.isDefined == true) + } - "send data (recv, send)" in { - val br = new Broker[Int] - val recvF = br.recv.sync() - assert(recvF.isDefined == false) - val sendF = br.send(123).sync() - assert(sendF.isDefined == true) - assert(recvF.isDefined == true) + test("Broker should send data (recv, send)") { + val br = new Broker[Int] + val recvF = br.recv.sync() + assert(recvF.isDefined == false) + val sendF = br.send(123).sync() + assert(sendF.isDefined == true) + assert(recvF.isDefined == true) - assert(Await.result(recvF) == 123) - } + assert(Await.result(recvF) == 123) + } - "queue receivers (recv, recv, send, send)" in { - val br = new Broker[Int] - val r0, r1 = br.recv.sync() - assert(r0.isDefined == false) - assert(r1.isDefined == false) - val s = br.send(123) - assert(s.sync().poll == Some(Return.Unit)) - assert(r0.poll == Some(Return(123))) - assert(r1.isDefined == false) - assert(s.sync().poll == Some(Return.Unit)) - assert(r1.poll == Some(Return(123))) - assert(s.sync().isDefined == false) - } + test("Broker should queue receivers (recv, recv, send, send)") { + val br = new Broker[Int] + val r0, r1 = br.recv.sync() + assert(r0.isDefined == false) + assert(r1.isDefined == false) + val s = br.send(123) + assert(s.sync().poll == Some(Return.Unit)) + assert(r0.poll == Some(Return(123))) + assert(r1.isDefined == false) + assert(s.sync().poll == Some(Return.Unit)) + assert(r1.poll == Some(Return(123))) + assert(s.sync().isDefined == false) + } - "queue senders (send, send, recv, recv)" in { - val br = new Broker[Int] - val s0, s1 = br.send(123).sync() - assert(s0.isDefined == false) - assert(s1.isDefined == false) - val r = br.recv - assert(r.sync().poll == Some(Return(123))) - assert(s0.poll == Some(Return.Unit)) - assert(s1.isDefined == false) - assert(r.sync().poll == Some(Return(123))) - assert(s1.poll == Some(Return.Unit)) - assert(r.sync().isDefined == false) - } + test("Broker should queue senders (send, send, recv, recv)") { + val br = new Broker[Int] + val s0, s1 = br.send(123).sync() + assert(s0.isDefined == false) + assert(s1.isDefined == false) + val r = br.recv + assert(r.sync().poll == Some(Return(123))) + assert(s0.poll == Some(Return.Unit)) + assert(s1.isDefined == false) + assert(r.sync().poll == Some(Return(123))) + assert(s1.poll == Some(Return.Unit)) + assert(r.sync().isDefined == false) + } - "interrupts" should { - "removes queued receiver" in { - val br = new Broker[Int] - val recvF = br.recv.sync() - recvF.raise(new Exception) - assert(br.send(123).sync().poll == None) - assert(recvF.poll == None) - } - - "removes queued sender" in { - val br = new Broker[Int] - val sendF = br.send(123).sync() - sendF.raise(new Exception) - assert(br.recv.sync().poll == None) - assert(sendF.poll == None) - } - - "doesn't result in space leaks" in { - val br = new Broker[Int] - - assert(Offer.select(Offer.const(1), br.recv).poll == Some(Return(1))) - val initial = ObjectSizeCalculator.getObjectSize(br) - - for (_ <- 0 until 1000) { - assert(Offer.select(Offer.const(1), br.recv).poll == Some(Return(1))) - assert(ObjectSizeCalculator.getObjectSize(br) == initial) - } - } - - "works with orElse" in { - val b0, b1 = new Broker[Int] - - val o = b0.recv orElse b1.recv - val f = o.sync() - assert(f.isDefined == false) - - val sendf0 = b0.send(12).sync() - assert(sendf0.isDefined == false) - val sendf1 = b1.send(32).sync() - assert(sendf1.isDefined == true) - assert(f.poll == Some(Return(32))) - - assert(o.sync().poll == Some(Return(12))) - assert(sendf0.poll == Some(Return.Unit)) - } - } + test("Broker interrupts should remove queued receiver") { + val br = new Broker[Int] + val recvF = br.recv.sync() + recvF.raise(new Exception) + assert(br.send(123).sync().poll == None) + assert(recvF.poll == None) + } - "integrate" in { - val br = new Broker[Int] - val offer = Offer.choose(br.recv, Offer.const(999)) - assert(offer.sync().poll == Some(Return(999))) + test("Broker interrupts should removes queued sender") { + val br = new Broker[Int] + val sendF = br.send(123).sync() + sendF.raise(new Exception) + assert(br.recv.sync().poll == None) + assert(sendF.poll == None) + } - val item = br.recv.sync() - assert(item.isDefined == false) + test("Broker interrupts should doesn't result in space leaks") { + val br = new Broker[Int] - assert(br.send(123).sync().poll == Some(Return.Unit)) - assert(item.poll == Some(Return(123))) + assert(Offer.select(Offer.const(1), br.recv).poll == Some(Return(1))) + val initial = ObjectSizeCalculator.getObjectSize(br) + + for (_ <- 0 until 1000) { + assert(Offer.select(Offer.const(1), br.recv).poll == Some(Return(1))) + assert(ObjectSizeCalculator.getObjectSize(br) == initial) } } + + test("Broker interrupts should works with orElse") { + val b0, b1 = new Broker[Int] + + val o = b0.recv orElse b1.recv + val f = o.sync() + assert(f.isDefined == false) + + val sendf0 = b0.send(12).sync() + assert(sendf0.isDefined == false) + val sendf1 = b1.send(32).sync() + assert(sendf1.isDefined == true) + assert(f.poll == Some(Return(32))) + + assert(o.sync().poll == Some(Return(12))) + assert(sendf0.poll == Some(Return.Unit)) + } + + test("Broker integrate") { + val br = new Broker[Int] + val offer = Offer.choose(br.recv, Offer.const(999)) + assert(offer.sync().poll == Some(Return(999))) + + val item = br.recv.sync() + assert(item.isDefined == false) + + assert(br.send(123).sync().poll == Some(Return.Unit)) + assert(item.poll == Some(Return(123))) + } } diff --git a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala index 2808dad379..1ad41d6755 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/OfferTest.scala @@ -1,10 +1,15 @@ package com.twitter.concurrent import com.twitter.conversions.DurationOps._ -import com.twitter.util.{Await, Future, MockTimer, Promise, Return, Time} +import com.twitter.util.Await +import com.twitter.util.Future +import com.twitter.util.MockTimer +import com.twitter.util.Promise +import com.twitter.util.Return +import com.twitter.util.Time import org.mockito.Mockito._ +import org.scalatest.funsuite.AnyFunSuite import org.scalatestplus.mockito.MockitoSugar -import org.scalatest.wordspec.AnyWordSpec import scala.collection.compat.immutable.LazyList import scala.util.Random @@ -19,322 +24,303 @@ class SimpleOffer[T](var futures: LazyList[Future[Tx[T]]]) extends Offer[T] { } } -class OfferTest extends AnyWordSpec with MockitoSugar { +class OfferSpecHelper { + val pendingTxs = 0 until 3 map { _ => new Promise[Tx[Int]] } + val offers = pendingTxs map { tx => spy(new SimpleOffer(tx)) } + val offer = Offer.choose(offers: _*) +} - def await[A](f: Future[A]): A = Await.result(f, 2.seconds) +class TxReadyHelper extends OfferSpecHelper with MockitoSugar { + val tx1 = mock[Tx[Int]] + pendingTxs(1).setValue(tx1) +} - import Tx.{Commit, Abort} +class AllTxsReadyHelper extends OfferSpecHelper with MockitoSugar { + val txs = for (p <- pendingTxs) yield { + val tx = mock[Tx[Int]] + p.setValue(tx) + tx + } +} - "Offer.map" should { - "apply f in after Tx.ack()" in { - // mockito can't spy on anonymous classes. - val tx = mock[Tx[Int]] - val result = Future.value(Commit(123)) - when(tx.ack()).thenReturn(result) +class OfferOrElseHelper { + val txp = new Promise[Tx[Int]] + val e0 = spy(new SimpleOffer(txp)) +} - assert(tx.ack() == (result)) - val offer = spy(new SimpleOffer(tx)) +class SyncIntegrationHelper { + val tx2 = new Promise[Tx[Int]] + val e0 = spy( + new SimpleOffer( + Future.value(Tx.aborted: Tx[Int]) #:: (tx2: Future[Tx[Int]]) #:: LazyList.empty + ) + ) + val offer = e0 orElse Offer.const(123) +} - val mapped = offer map { i => (i - 100).toString } +class OfferTest extends AnyFunSuite with MockitoSugar { - val f = mapped.prepare() flatMap { tx => tx.ack() } + def await[A](f: Future[A]): A = Await.result(f, 2.seconds) - assert(await(f) == Commit("23")) - } - } + import Tx.Commit + import Tx.Abort - "Offer.choose" should { - class OfferSpecHelper { - val pendingTxs = 0 until 3 map { _ => new Promise[Tx[Int]] } - val offers = pendingTxs map { tx => spy(new SimpleOffer(tx)) } - val offer = Offer.choose(offers: _*) - } + test("Offer.map should apply f in after Tx.ack()") { + // mockito can't spy on anonymous classes. + val tx = mock[Tx[Int]] + val result = Future.value(Commit(123)) + when(tx.ack()).thenReturn(result) - "when a tx is already ready" should { - class TxReadyHelper extends OfferSpecHelper { - val tx1 = mock[Tx[Int]] - pendingTxs(1).setValue(tx1) - } + assert(tx.ack() == (result)) + val offer = spy(new SimpleOffer(tx)) - "prepare() prepares all" in { - val h = new TxReadyHelper - import h._ + val mapped = offer map { i => (i - 100).toString } - offers foreach { of => verify(of, never()).prepare() } - assert(offer.prepare().isDefined == true) - offers foreach { of => verify(of).prepare() } - } + val f = mapped.prepare() flatMap { tx => tx.ack() } - "select it" in { - val h = new TxReadyHelper - import h._ + assert(await(f) == Commit("23")) + } - assert(await(offer.prepare()) == tx1) - verify(tx1, never()).ack() - verify(tx1, never()).nack() - } + test("Offer.choose when a tx is already ready: prepare() prepares all") { + val h = new TxReadyHelper + import h._ - "nack losers" in { - val h = new TxReadyHelper - import h._ - - offer.prepare() - val tx = mock[Tx[Int]] - for (i <- Seq(0, 2)) { - pendingTxs(i).setValue(tx) - } - verify(tx, times(2)).nack() - verify(tx, never()).ack() - } - } + offers foreach { of => verify(of, never()).prepare() } + assert(offer.prepare().isDefined == true) + offers foreach { of => verify(of).prepare() } + } - "when a tx is ready after prepare()" should { - "select it" in { - val h = new OfferSpecHelper - import h._ + test("Offer.choose when a tx is already ready should select it") { + val h = new TxReadyHelper + import h._ - val tx = offer.prepare() - assert(tx.isDefined == false) - val tx0 = mock[Tx[Int]] - pendingTxs(0).setValue(tx0) + assert(await(offer.prepare()) == tx1) + verify(tx1, never()).ack() + verify(tx1, never()).nack() + } - assert(await(tx) == tx0) - } + test("Offer.choose when a tx is already ready should nack losers") { + val h = new TxReadyHelper + import h.offer + import h.pendingTxs - "nack losers" in { - val h = new OfferSpecHelper - import h._ - - offer.prepare() - pendingTxs(0).setValue(mock[Tx[Int]]) - for (i <- Seq(1, 2)) { - val tx = mock[Tx[Int]] - pendingTxs(i).setValue(tx) - verify(tx).nack() - verify(tx, never()).ack() - } - } + offer.prepare() + val tx = mock[Tx[Int]] + for (i <- Seq(0, 2)) { + pendingTxs(i).setValue(tx) } + verify(tx, times(2)).nack() + verify(tx, never()).ack() + } - "when all txs are ready" should { - class AllTxsReadyHelper extends OfferSpecHelper { - val txs = for (p <- pendingTxs) yield { - val tx = mock[Tx[Int]] - p.setValue(tx) - tx - } - } - - "shuffle winner" in Time.withTimeAt(Time.epoch) { tc => - val h = new AllTxsReadyHelper - import h._ + test("Offer.choose when a tx is ready after prepare() should select it") { + val h = new OfferSpecHelper + import h._ - val shuffledOffer = Offer.choose(Some(new Random(Time.now.inNanoseconds)), offers) - val histo = new Array[Int](3) - for (_ <- 0 until 1000) { - for (tx <- shuffledOffer.prepare()) - histo(txs.indexOf(tx)) += 1 - } + val tx = offer.prepare() + assert(tx.isDefined == false) + val tx0 = mock[Tx[Int]] + pendingTxs(0).setValue(tx0) - assert(histo(0) == 311) - assert(histo(1) == 346) - assert(histo(2) == 343) - } + assert(await(tx) == tx0) + } - "nack losers" in { - val h = new AllTxsReadyHelper - import h._ + test("Offer.choose when a tx is ready after prepare() should nack losers") { + val h = new OfferSpecHelper + import h._ - val tx = await(offer.prepare()) - for (loser <- txs if loser ne tx) verify(loser).nack() - } + offer.prepare() + pendingTxs(0).setValue(mock[Tx[Int]]) + for (i <- Seq(1, 2)) { + val tx = mock[Tx[Int]] + pendingTxs(i).setValue(tx) + verify(tx).nack() + verify(tx, never()).ack() } + } - "work with 0 offers" in { - val h = new OfferSpecHelper + test("Offer.choose when all txs are ready should shuffle winner") { + Time.withTimeAt(Time.epoch) { tc => + val h = new AllTxsReadyHelper + import h._ + + val shuffledOffer = Offer.choose(Some(new Random(Time.now.inNanoseconds)), offers) + val histo = new Array[Int](3) + for (_ <- 0 until 1000) { + for (tx <- shuffledOffer.prepare()) + histo(txs.indexOf(tx)) += 1 + } - val of = Offer.choose() - assert(of.sync().poll == None) + assert(histo(0) == 311) + assert(histo(1) == 346) + assert(histo(2) == 343) } } - "Offer.sync" should { - "when Tx.prepare is immediately available" should { - "when it commits" in { - val txp = new Promise[Tx[Int]] - val offer = spy(new SimpleOffer(txp)) - val tx = mock[Tx[Int]] - val result = Future.value(Commit(123)) - when(tx.ack()).thenReturn(result) - - assert(tx.ack() == (result)) - txp.setValue(tx) - - assert(await(offer.sync()) == 123) - verify(tx, times(2)).ack() - verify(tx, never()).nack() - } + test("Offer.choose when all txs are ready should nack losers") { + val h = new AllTxsReadyHelper + import h._ - "retry when it aborts" in { - val txps = new Promise[Tx[Int]] #:: new Promise[Tx[Int]] #:: LazyList.empty - val offer = spy(new SimpleOffer(txps)) - val badTx = mock[Tx[Int]] - val result = Future.value(Abort) - when(badTx.ack()).thenReturn(result) + val tx = await(offer.prepare()) + for (loser <- txs if loser ne tx) verify(loser).nack() + } - assert(badTx.ack() == (result)) - txps(0).setValue(badTx) + test("Offer.choose should work with 0 offers") { + val h = new OfferSpecHelper - val syncd = offer.sync() + val of = Offer.choose() + assert(of.sync().poll == None) + } - assert(syncd.poll == None) - verify(badTx, times(2)).ack() - verify(offer, times(2)).prepare() + test("Offer.sync when Tx.prepare is immediately available: when it commits") { + val txp = new Promise[Tx[Int]] + val offer = spy(new SimpleOffer(txp)) + val tx = mock[Tx[Int]] + val result = Future.value(Commit(123)) + when(tx.ack()).thenReturn(result) - val okTx = mock[Tx[Int]] - val okResult = Future.value(Commit(333)) - when(okTx.ack()).thenReturn(okResult) + assert(tx.ack() == (result)) + txp.setValue(tx) - assert(okTx.ack() == (okResult)) - txps(1).setValue(okTx) + assert(await(offer.sync()) == 123) + verify(tx, times(2)).ack() + verify(tx, never()).nack() + } - assert(syncd.poll == Some(Return(333))) + test("Offer.sync when Tx.prepare is immediately available: retry when it aborts") { + val txps = new Promise[Tx[Int]] #:: new Promise[Tx[Int]] #:: LazyList.empty + val offer = spy(new SimpleOffer(txps)) + val badTx = mock[Tx[Int]] + val result = Future.value(Abort) + when(badTx.ack()).thenReturn(result) - verify(okTx, times(2)).ack() - verify(offer, times(2)).prepare() - } - } + assert(badTx.ack() == (result)) + txps(0).setValue(badTx) - "when Tx.prepare is delayed" should { - "when it commits" in { - val tx = mock[Tx[Int]] - val result = Future.value(Commit(123)) - when(tx.ack()).thenReturn(result) - assert(tx.ack() == (result)) - val offer = spy(new SimpleOffer(tx)) - - assert(await(offer.sync()) == 123) - verify(tx, times(2)).ack() - verify(tx, never()).nack() - verify(offer).prepare() - } - } - } + val syncd = offer.sync() - "Offer.const" should { - "always provide the same result" in { - val offer = Offer.const(123) + assert(syncd.poll == None) + verify(badTx, times(2)).ack() + verify(offer, times(2)).prepare() - assert(offer.sync().poll == Some(Return(123))) - assert(offer.sync().poll == Some(Return(123))) - } + val okTx = mock[Tx[Int]] + val okResult = Future.value(Commit(333)) + when(okTx.ack()).thenReturn(okResult) - "evaluate argument for each prepare()" in { - var i = 0 - val offer = Offer.const { i = i + 1; i } - assert(offer.sync().poll == Some(Return(1))) - assert(offer.sync().poll == Some(Return(2))) - } - } + assert(okTx.ack() == (okResult)) + txps(1).setValue(okTx) - "Offer.orElse" should { - "with const orElse" should { - class OfferOrElseHelper { - val txp = new Promise[Tx[Int]] - val e0 = spy(new SimpleOffer(txp)) - } + assert(syncd.poll == Some(Return(333))) - "prepare orElse event when prepare isn't immediately available" in { - val h = new OfferOrElseHelper - import h._ - - val e1 = Offer.const(123) - val offer = e0 orElse e1 - assert(offer.sync().poll == Some(Return(123))) - verify(e0).prepare() - val tx = mock[Tx[Int]] - txp.setValue(tx) - verify(tx).nack() - verify(tx, never()).ack() - } + verify(okTx, times(2)).ack() + verify(offer, times(2)).prepare() + } - "not prepare orElse event when the result is immediately available" in { - val h = new OfferOrElseHelper - import h._ - - val e1 = spy(new SimpleOffer(LazyList.empty)) - val offer = e0 orElse e1 - val tx = mock[Tx[Int]] - val result = Future.value(Commit(321)) - when(tx.ack()).thenReturn(result) - - assert(tx.ack() == (result)) - txp.setValue(tx) - assert(offer.sync().poll == Some(Return(321))) - verify(e0).prepare() - verify(tx, times(2)).ack() - verify(tx, never()).nack() - verify(e1, never).prepare() - } - } + test("Offer.sync when Tx.prepare is delayed: when it commits") { + val tx = mock[Tx[Int]] + val result = Future.value(Commit(123)) + when(tx.ack()).thenReturn(result) + assert(tx.ack() == (result)) + val offer = spy(new SimpleOffer(tx)) + + assert(await(offer.sync()) == 123) + verify(tx, times(2)).ack() + verify(tx, never()).nack() + verify(offer).prepare() + } - "sync integration: when first transaction aborts" should { - class SyncIntegrationHelper { - val tx2 = new Promise[Tx[Int]] - val e0 = spy( - new SimpleOffer( - Future.value(Tx.aborted: Tx[Int]) #:: (tx2: Future[Tx[Int]]) #:: LazyList.empty - ) - ) - val offer = e0 orElse Offer.const(123) - } + test("Offer.const should always provide the same result") { + val offer = Offer.const(123) - "select first again if tx2 is ready" in { - val h = new SyncIntegrationHelper - import h._ + assert(offer.sync().poll == Some(Return(123))) + assert(offer.sync().poll == Some(Return(123))) + } - val tx = mock[Tx[Int]] - val result = Future.value(Commit(321)) - when(tx.ack()).thenReturn(result) + test("Offer.const should evaluate argument for each prepare()") { + var i = 0 + val offer = Offer.const { i = i + 1; i } + assert(offer.sync().poll == Some(Return(1))) + assert(offer.sync().poll == Some(Return(2))) + } - assert(tx.ack() == (result)) - tx2.setValue(tx) + test( + "Offer.orElse with const orElse should prepare orElse event when prepare isn't immediately available") { + val h = new OfferOrElseHelper + import h._ + + val e1 = Offer.const(123) + val offer = e0 orElse e1 + assert(offer.sync().poll == Some(Return(123))) + verify(e0).prepare() + val tx = mock[Tx[Int]] + txp.setValue(tx) + verify(tx).nack() + verify(tx, never()).ack() + } - assert(offer.sync().poll == Some(Return(321))) - verify(e0, times(2)).prepare() - verify(tx, times(2)).ack() - verify(tx, never()).nack() - } + test( + "Offer.orElse with const orElse should not prepare orElse event when the result is immediately available") { + val h = new OfferOrElseHelper + import h._ + + val e1 = spy(new SimpleOffer(LazyList.empty)) + val offer = e0 orElse e1 + val tx = mock[Tx[Int]] + val result = Future.value(Commit(321)) + when(tx.ack()).thenReturn(result) + + assert(tx.ack() == (result)) + txp.setValue(tx) + assert(offer.sync().poll == Some(Return(321))) + verify(e0).prepare() + verify(tx, times(2)).ack() + verify(tx, never()).nack() + verify(e1, never).prepare() + } + test( + "Offer.orElse sync integration: when first transaction aborts should select first again if tx2 is ready") { + val h = new SyncIntegrationHelper + import h._ + + val tx = mock[Tx[Int]] + val result = Future.value(Commit(321)) + when(tx.ack()).thenReturn(result) + + assert(tx.ack() == (result)) + tx2.setValue(tx) + + assert(offer.sync().poll == Some(Return(321))) + verify(e0, times(2)).prepare() + verify(tx, times(2)).ack() + verify(tx, never()).nack() + } - "select alternative if not ready the second time" in { - val h = new SyncIntegrationHelper - import h._ + test( + "Offer.orElse sync integration: when first transaction aborts should select alternative if not ready the second time") { + val h = new SyncIntegrationHelper + import h._ - assert(offer.sync().poll == Some(Return(123))) - verify(e0, times(2)).prepare() + assert(offer.sync().poll == Some(Return(123))) + verify(e0, times(2)).prepare() - val tx = mock[Tx[Int]] - tx2.setValue(tx) - verify(tx).nack() - } - } + val tx = mock[Tx[Int]] + tx2.setValue(tx) + verify(tx).nack() } - "Offer.foreach" should { - "synchronize on offers forever" in { - val b = new Broker[Int] - var count = 0 - b.recv foreach { _ => count += 1 } - assert(count == 0) - assert(b.send(1).sync().isDefined == true) - assert(count == 1) - assert(b.send(1).sync().isDefined == true) - assert(count == 2) - } + test("Offer.foreach should synchronize on offers forever") { + val b = new Broker[Int] + var count = 0 + b.recv foreach { _ => count += 1 } + assert(count == 0) + assert(b.send(1).sync().isDefined == true) + assert(count == 1) + assert(b.send(1).sync().isDefined == true) + assert(count == 2) } - "Offer.timeout" should { - "be available after timeout (prepare)" in Time.withTimeAt(Time.epoch) { tc => + test("Offer.timeout should be available after timeout (prepare)") { + Time.withTimeAt(Time.epoch) { tc => implicit val timer = new MockTimer val e = Offer.timeout(10.seconds) assert(e.prepare().isDefined == false) @@ -345,8 +331,10 @@ class OfferTest extends AnyWordSpec with MockitoSugar { timer.tick() assert(e.prepare().isDefined == true) } + } - "cancel timer tasks when losing" in Time.withTimeAt(Time.epoch) { tc => + test("Offer.timeout should cancel timer tasks when losing") { + Time.withTimeAt(Time.epoch) { tc => implicit val timer = new MockTimer val e10 = Offer.timeout(10.seconds) map { _ => 10 } val e5 = Offer.timeout(5.seconds) map { _ => 5 } @@ -365,47 +353,43 @@ class OfferTest extends AnyWordSpec with MockitoSugar { } } - "Offer.prioritize" should { - "prioritize high priority offer" in { - val offers = 0 until 10 map { i => - val tx = mock[Tx[Int]] - val result = Future.value(Commit(i)) - when(tx.ack()).thenReturn(result) - new SimpleOffer(Future.value(tx)) - } - val chosenOffer = Offer.prioritize(offers: _*) - val of = chosenOffer.sync() - assert(of.isDefined == true) - assert(Await.result(of) == 0) + test("Offer.prioritize should prioritize high priority offer") { + val offers = 0 until 10 map { i => + val tx = mock[Tx[Int]] + val result = Future.value(Commit(i)) + when(tx.ack()).thenReturn(result) + new SimpleOffer(Future.value(tx)) } + val chosenOffer = Offer.prioritize(offers: _*) + val of = chosenOffer.sync() + assert(of.isDefined == true) + assert(Await.result(of) == 0) } - "Integration" should { - "select across multiple brokers" in { - val b0 = new Broker[Int] - val b1 = new Broker[String] - - val o = Offer.choose( - b0.send(123) const { "put!" }, - b1.recv - ) - - val f = o.sync() - assert(f.isDefined == false) - assert(b1.send("hey").sync().isDefined == true) - assert(f.isDefined == true) - assert(Await.result(f) == "hey") - - val gf = b0.recv.sync() - assert(gf.isDefined == false) - val of = o.sync() - assert(of.isDefined == true) - assert(Await.result(of) == "put!") - assert(gf.isDefined == true) - assert(Await.result(gf) == 123) - - // syncing again fails. - assert(o.sync().isDefined == false) - } + test("Integration should select across multiple brokers") { + val b0 = new Broker[Int] + val b1 = new Broker[String] + + val o = Offer.choose( + b0.send(123) const { "put!" }, + b1.recv + ) + + val f = o.sync() + assert(f.isDefined == false) + assert(b1.send("hey").sync().isDefined == true) + assert(f.isDefined == true) + assert(Await.result(f) == "hey") + + val gf = b0.recv.sync() + assert(gf.isDefined == false) + val of = o.sync() + assert(of.isDefined == true) + assert(Await.result(of) == "put!") + assert(gf.isDefined == true) + assert(Await.result(gf) == 123) + + // syncing again fails. + assert(o.sync().isDefined == false) } } diff --git a/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala index 578129fb82..2fb27928c8 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SerializedTest.scala @@ -1,43 +1,41 @@ package com.twitter.concurrent import java.util.concurrent.CountDownLatch -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class SerializedTest extends AnyWordSpec with Serialized { - "Serialized" should { - "runs blocks, one at a time, in the order received" in { - val t1CallsSerializedFirst = new CountDownLatch(1) - val t1FinishesWork = new CountDownLatch(1) - val orderOfExecution = new collection.mutable.ListBuffer[Thread] +class SerializedTest extends AnyFunSuite with Serialized { + test("Serialized runs blocks, one at a time, in the order received") { + val t1CallsSerializedFirst = new CountDownLatch(1) + val t1FinishesWork = new CountDownLatch(1) + val orderOfExecution = new collection.mutable.ListBuffer[Thread] - val t1 = new Thread { - override def run: Unit = { - serialized { - t1CallsSerializedFirst.countDown() - t1FinishesWork.await() - orderOfExecution += this - () - } + val t1 = new Thread { + override def run: Unit = { + serialized { + t1CallsSerializedFirst.countDown() + t1FinishesWork.await() + orderOfExecution += this + () } } + } - val t2 = new Thread { - override def run: Unit = { - t1CallsSerializedFirst.await() - serialized { - orderOfExecution += this - () - } - t1FinishesWork.countDown() + val t2 = new Thread { + override def run: Unit = { + t1CallsSerializedFirst.await() + serialized { + orderOfExecution += this + () } + t1FinishesWork.countDown() } + } - t1.start() - t2.start() - t1.join() - t2.join() + t1.start() + t2.start() + t1.join() + t2.join() - assert(orderOfExecution.toList == List(t1, t2)) - } + assert(orderOfExecution.toList == List(t1, t2)) } } diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala index 792835f201..70b29f584a 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolSourceTest.scala @@ -1,110 +1,109 @@ package com.twitter.concurrent -import com.twitter.util.{Return, Await} +import com.twitter.util.Await +import com.twitter.util.Return import java.util.concurrent.atomic.AtomicInteger -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class SpoolSourceTest extends AnyWordSpec { - "SpoolSource" should { - class SpoolSourceHelper { - val source = new SpoolSource[Int] - } +class SpoolSourceTest extends AnyFunSuite { + class SpoolSourceHelper { + val source = new SpoolSource[Int] + } - "add values to the spool, ignoring values after close" in { - val h = new SpoolSourceHelper - import h._ - - val futureSpool = source() - source.offer(1) - source.offer(2) - source.offer(3) - source.close() - source.offer(4) - source.offer(5) - assert(Await.result(futureSpool flatMap (_.toSeq)) == Seq(1, 2, 3)) - } + test("SpoolSource should add values to the spool, ignoring values after close") { + val h = new SpoolSourceHelper + import h._ + + val futureSpool = source() + source.offer(1) + source.offer(2) + source.offer(3) + source.close() + source.offer(4) + source.offer(5) + assert(Await.result(futureSpool flatMap (_.toSeq)) == Seq(1, 2, 3)) + } - "add values to the spool, ignoring values after offerAndClose" in { - val h = new SpoolSourceHelper - import h._ - - val futureSpool = source() - source.offer(1) - source.offer(2) - source.offerAndClose(3) - source.offer(4) - source.offer(5) - assert(Await.result(futureSpool flatMap (_.toSeq)) == Seq(1, 2, 3)) - assert(Await.result(source.closed.liftToTry) == Return(())) - } + test("SpoolSource should add values to the spool, ignoring values after offerAndClose") { + val h = new SpoolSourceHelper + import h._ + + val futureSpool = source() + source.offer(1) + source.offer(2) + source.offerAndClose(3) + source.offer(4) + source.offer(5) + assert(Await.result(futureSpool flatMap (_.toSeq)) == Seq(1, 2, 3)) + assert(Await.result(source.closed.liftToTry) == Return(())) + } - "return multiple Future Spools that only see values added later" in { - val h = new SpoolSourceHelper - import h._ - - val futureSpool1 = source() - source.offer(1) - val futureSpool2 = source() - source.offer(2) - val futureSpool3 = source() - source.offer(3) - val futureSpool4 = source() - source.close() - assert(Await.result(futureSpool1 flatMap (_.toSeq)) == Seq(1, 2, 3)) - assert(Await.result(futureSpool2 flatMap (_.toSeq)) == Seq(2, 3)) - assert(Await.result(futureSpool3 flatMap (_.toSeq)) == Seq(3)) - assert(Await.result(futureSpool4).isEmpty == true) - } + test("SpoolSource should return multiple Future Spools that only see values added later") { + val h = new SpoolSourceHelper + import h._ + + val futureSpool1 = source() + source.offer(1) + val futureSpool2 = source() + source.offer(2) + val futureSpool3 = source() + source.offer(3) + val futureSpool4 = source() + source.close() + assert(Await.result(futureSpool1 flatMap (_.toSeq)) == Seq(1, 2, 3)) + assert(Await.result(futureSpool2 flatMap (_.toSeq)) == Seq(2, 3)) + assert(Await.result(futureSpool3 flatMap (_.toSeq)) == Seq(3)) + assert(Await.result(futureSpool4).isEmpty == true) + } - "throw exception and close spool when exception is raised" in { - val h = new SpoolSourceHelper - import h._ - - val futureSpool1 = source() - source.offer(1) - source.raise(new Exception("sad panda")) - val futureSpool2 = source() - source.offer(1) - intercept[Exception] { - Await.result(futureSpool1 flatMap (_.toSeq)) - } - assert(Await.result(futureSpool2).isEmpty == true) + test("SpoolSource should throw exception and close spool when exception is raised") { + val h = new SpoolSourceHelper + import h._ + + val futureSpool1 = source() + source.offer(1) + source.raise(new Exception("sad panda")) + val futureSpool2 = source() + source.offer(1) + intercept[Exception] { + Await.result(futureSpool1 flatMap (_.toSeq)) } + assert(Await.result(futureSpool2).isEmpty == true) + } - "invoke registered interrupt handlers" in { - val h = new SpoolSourceHelper + test("SpoolSource should invoke registered interrupt handlers") { + val h = new SpoolSourceHelper - val fired = new AtomicInteger(0) + val fired = new AtomicInteger(0) - val isource = new SpoolSource[Int]({ - case exc => - fired.addAndGet(1) - }) + val isource = new SpoolSource[Int]({ + case exc => + fired.addAndGet(1) + }) - val a = isource() - val b = a.flatMap(_.tail) - val c = b.flatMap(_.tail) - val d = c.flatMap(_.tail) + val a = isource() + val b = a.flatMap(_.tail) + val c = b.flatMap(_.tail) + val d = c.flatMap(_.tail) - assert(fired.get() == 0) + assert(fired.get() == 0) - a.raise(new Exception("sad panda 0")) - assert(fired.get() == 1) + a.raise(new Exception("sad panda 0")) + assert(fired.get() == 1) - isource.offer(1) - b.raise(new Exception("sad panda 1")) - assert(fired.get() == 2) + isource.offer(1) + b.raise(new Exception("sad panda 1")) + assert(fired.get() == 2) - isource.offer(2) - c.raise(new Exception("sad panda 2")) - assert(fired.get() == 3) + isource.offer(2) + c.raise(new Exception("sad panda 2")) + assert(fired.get() == 3) - // ignore exceptions once the source is closed - isource.offerAndClose(3) - d.raise(new Exception("sad panda 3")) - assert(fired.get() == 3) + // ignore exceptions once the source is closed + isource.offerAndClose(3) + d.raise(new Exception("sad panda 3")) + assert(fired.get() == 3) - assert(Await.result(a flatMap (_.toSeq)) == Seq(1, 2, 3)) - } + assert(Await.result(a flatMap (_.toSeq)) == Seq(1, 2, 3)) } } diff --git a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala index 38cc0d2c65..dfa743153a 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/SpoolTest.scala @@ -1,467 +1,464 @@ package com.twitter.concurrent -import com.twitter.concurrent.Spool.{*::, seqToSpool} +import com.twitter.concurrent.Spool.*:: +import com.twitter.concurrent.Spool.seqToSpool import com.twitter.conversions.DurationOps._ -import com.twitter.util.{Await, Future, Promise, Return, Throw} +import com.twitter.util.Await +import com.twitter.util.Future +import com.twitter.util.Promise +import com.twitter.util.Return +import com.twitter.util.Throw import java.io.EOFException import org.scalacheck.Arbitrary +import org.scalatest.funsuite.AnyFunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.collection.mutable.ArrayBuffer -import org.scalatest.wordspec.AnyWordSpec -class SpoolTest extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { +class SpoolTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { def await[A](f: Future[A]): A = Await.result(f, 2.seconds) + val resolvedSpoolFuture = 1 *:: Future.value(2 *:: Future.value(Spool.empty[Int])) + val emptySpool = Spool.empty[Int] + + class SimpleDelayedSpoolHelper { + val p = new Promise[Spool[Int]] + val p1 = new Promise[Spool[Int]] + val p2 = new Promise[Spool[Int]] + val s = 1 *:: p + } - "Empty Spool" should { - val s = Spool.empty[Int] - - "iterate over all elements" in { - val xs = new ArrayBuffer[Int] - s foreach { xs += _ } - assert(xs.size == 0) - } + /** + * Confirms that the given operation does not consume an entire Spool, and then + * returns the resulting Spool and tail check for further validation. + */ + def applyLazily( + f: Spool[Int] => Future[Spool[Int]] + ): (Future[Spool[Int]], Future[Spool[Int]]) = { + val tail = new Promise[Spool[Int]] + + // A spool where only the head is valid. + def spool: Spool[Int] = 0 *:: { tail.setException(new Exception); tail } + + // create, apply, poll + val s = f(spool) + assert(!tail.isDefined) + (s, tail) + } - "map" in { - assert((s map { _ * 2 }) == Spool.empty[Int]) - } + test("Empty Spool should iterate over all elements") { + val xs = new ArrayBuffer[Int] + emptySpool foreach { xs += _ } + assert(xs.size == 0) + } - "mapFuture" in { - val mapFuture = s mapFuture { Future.value(_) } - assert(mapFuture.poll == Some(Return(s))) - } + test("Empty Spool should map") { + assert((emptySpool map { _ * 2 }) == Spool.empty[Int]) + } - "deconstruct" in { - assert(s match { - case x *:: _ => false - case _ => true - }) - } + test("Empty Spool should mapFuture") { + val mapFuture = emptySpool mapFuture { Future.value(_) } + assert(mapFuture.poll == Some(Return(emptySpool))) + } - "append via ++" in { - assert((s ++ Spool.empty[Int]) == Spool.empty[Int]) - assert((Spool.empty[Int] ++ s) == Spool.empty[Int]) + test("Empty Spool should deconstruct") { + assert(emptySpool match { + case x *:: _ => false + case _ => true + }) + } - val s2 = s ++ (3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) - assert(await(s2.toSeq) == Seq(3, 4)) - } + test("Empty Spool should append via ++") { + assert((emptySpool ++ Spool.empty[Int]) == Spool.empty[Int]) + assert((Spool.empty[Int] ++ emptySpool) == Spool.empty[Int]) - "append via ++ with Future rhs" in { - assert(await(s ++ Future(Spool.empty[Int])) == Spool.empty[Int]) - assert(await(Spool.empty[Int] ++ Future(s)) == Spool.empty[Int]) + val s2 = emptySpool ++ (3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) + assert(await(s2.toSeq) == Seq(3, 4)) + } - val s2 = s ++ Future(3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) - assert(await(s2 flatMap (_.toSeq)) == Seq(3, 4)) - } + test("Empty Spool should append via ++ with Future rhs") { + assert(await(emptySpool ++ Future(Spool.empty[Int])) == Spool.empty[Int]) + assert(await(Spool.empty[Int] ++ Future(emptySpool)) == Spool.empty[Int]) - "flatMap" in { - val f = (x: Int) => - Future(x.toString *:: Future.value((x * 2).toString *:: Future.value(Spool.empty[String]))) - assert(await(s flatMap f) == Spool.empty[Int]) - } + val s2 = emptySpool ++ Future(3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) + assert(await(s2 flatMap (_.toSeq)) == Seq(3, 4)) + } - "fold left" in { - val fold = s.foldLeft(0) { (x, y) => x + y } - assert(await(fold) == 0) - } + test("Empty Spool should flatMap") { + val f = (x: Int) => + Future(x.toString *:: Future.value((x * 2).toString *:: Future.value(Spool.empty[String]))) + assert(await(emptySpool flatMap f) == Spool.empty[Int]) + } - "reduce left" in { - val fold = s.reduceLeft { (x, y) => x + y } - intercept[UnsupportedOperationException] { - await(fold) - } - } + test("Empty Spool should fold left") { + val fold = emptySpool.foldLeft(0) { (x, y) => x + y } + assert(await(fold) == 0) + } - "zip with empty" in { - val result = s.zip(Spool.empty[Int]) - assert(await(result.toSeq) == Nil) + test("Empty Spool should reduce left") { + val fold = emptySpool.reduceLeft { (x, y) => x + y } + intercept[UnsupportedOperationException] { + await(fold) } + } - "zip with non-empty" in { - val result = s.zip(Seq(1, 2, 3).toSpool) - assert(await(result.toSeq) == Nil) - } + test("Empty Spool should zip with empty") { + val result = emptySpool.zip(Spool.empty[Int]) + assert(await(result.toSeq) == Nil) + } - "take" in { - assert(s.take(10) == Spool.empty[Int]) - } + test("Empty Spool should zip with non-empty") { + val result = emptySpool.zip(Seq(1, 2, 3).toSpool) + assert(await(result.toSeq) == Nil) } - "Simple resolved Spool" should { - val s = 1 *:: Future.value(2 *:: Future.value(Spool.empty[Int])) + test("Empty Spool should take") { + assert(emptySpool.take(10) == Spool.empty[Int]) + } - "iterate over all elements" in { - val xs = new ArrayBuffer[Int] - s foreach { xs += _ } - assert(xs.toSeq == Seq(1, 2)) - } + test("Simple resolved spool should iterate over all elements") { + val xs = new ArrayBuffer[Int] + resolvedSpoolFuture foreach { xs += _ } + assert(xs.toSeq == Seq(1, 2)) + } - "buffer to a sequence" in { - assert(await(s.toSeq) == Seq(1, 2)) - } + test("Simple resolved spool should buffer to a sequence") { + assert(await(resolvedSpoolFuture.toSeq) == Seq(1, 2)) + } - "map" in { - assert(await(s.map { _ * 2 }.toSeq) == Seq(2, 4)) - } + test("Simple resolved spool should map") { + assert(await(resolvedSpoolFuture.map { _ * 2 }.toSeq) == Seq(2, 4)) + } - "mapFuture" in { - val f = s.mapFuture { Future.value(_) }.flatMap { _.toSeq }.poll - assert(f == Some(Return(Seq(1, 2)))) - } + test("Simple resolved spool should mapFuture") { + val f = resolvedSpoolFuture.mapFuture { Future.value(_) }.flatMap { _.toSeq }.poll + assert(f == Some(Return(Seq(1, 2)))) + } - "deconstruct" in { - assert(s match { - case x *:: rest => - assert(x == 1) - await(rest) match { - case y *:: rest if y == 2 => await(rest).isEmpty - case _ => fail("expected 3 elements in spool") - } - case _ => fail("expected 3 elements in spool") - }) - } + test("Simple resolved spool should deconstruct") { + assert(resolvedSpoolFuture match { + case x *:: rest => + assert(x == 1) + await(rest) match { + case y *:: rest if y == 2 => await(rest).isEmpty + case _ => fail("expected 3 elements in spool") + } + case _ => fail("expected 3 elements in spool") + }) + } - "append via ++" in { - assert(await((s ++ Spool.empty[Int]).toSeq) == Seq(1, 2)) - assert(await((Spool.empty[Int] ++ s).toSeq) == Seq(1, 2)) + test("Simple resolved spool should append via ++") { + assert(await((resolvedSpoolFuture ++ Spool.empty[Int]).toSeq) == Seq(1, 2)) + assert(await((Spool.empty[Int] ++ resolvedSpoolFuture).toSeq) == Seq(1, 2)) - val s2 = s ++ (3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) - assert(await(s2.toSeq) == Seq(1, 2, 3, 4)) - } + val s2 = resolvedSpoolFuture ++ (3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) + assert(await(s2.toSeq) == Seq(1, 2, 3, 4)) + } - "append via ++ with Future rhs" in { - assert(await(s ++ Future.value(Spool.empty[Int]) flatMap (_.toSeq)) == Seq(1, 2)) - assert(await(Spool.empty[Int] ++ Future.value(s) flatMap (_.toSeq)) == Seq(1, 2)) + test("Simple resolved spool should append via ++ with Future rhs") { + assert( + await(resolvedSpoolFuture ++ Future.value(Spool.empty[Int]) flatMap (_.toSeq)) == Seq(1, 2)) + assert( + await(Spool.empty[Int] ++ Future.value(resolvedSpoolFuture) flatMap (_.toSeq)) == Seq(1, 2)) - val s2 = s ++ Future.value(3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) - assert(await(s2 flatMap (_.toSeq)) == Seq(1, 2, 3, 4)) - } + val s2 = + resolvedSpoolFuture ++ Future.value(3 *:: Future.value(4 *:: Future.value(Spool.empty[Int]))) + assert(await(s2 flatMap (_.toSeq)) == Seq(1, 2, 3, 4)) + } - "flatMap" in { - val f = (x: Int) => Future(Seq(x.toString, (x * 2).toString).toSpool) - val s2 = s flatMap f - assert(await(s2 flatMap (_.toSeq)) == Seq("1", "2", "2", "4")) - } + test("Simple resolved spool should flatMap") { + val f = (x: Int) => Future(Seq(x.toString, (x * 2).toString).toSpool) + val s2 = resolvedSpoolFuture flatMap f + assert(await(s2 flatMap (_.toSeq)) == Seq("1", "2", "2", "4")) + } - "fold left" in { - val fold = s.foldLeft(0) { (x, y) => x + y } - assert(await(fold) == 3) - } + test("Simple resolved spool should fold left") { + val fold = resolvedSpoolFuture.foldLeft(0) { (x, y) => x + y } + assert(await(fold) == 3) + } - "reduce left" in { - val fold = s.reduceLeft { (x, y) => x + y } - assert(await(fold) == 3) - } + test("Simple resolved spool should reduce left") { + val fold = resolvedSpoolFuture.reduceLeft { (x, y) => x + y } + assert(await(fold) == 3) + } - "zip with empty" in { - val zip = s.zip(Spool.empty[Int]) - assert(await(zip.toSeq) == Nil) - } + test("Simple resolved spool should zip with empty") { + val zip = resolvedSpoolFuture.zip(Spool.empty[Int]) + assert(await(zip.toSeq) == Nil) + } - "zip with same size spool" in { - val zip = s.zip(Seq("a", "b").toSpool) - assert(await(zip.toSeq) == Seq((1, "a"), (2, "b"))) - } + test("Simple resolved spool should zip with same size spool") { + val zip = resolvedSpoolFuture.zip(Seq("a", "b").toSpool) + assert(await(zip.toSeq) == Seq((1, "a"), (2, "b"))) + } - "zip with larger spool" in { - val zip = s.zip(Seq("a", "b", "c", "d").toSpool) - assert(await(zip.toSeq) == Seq((1, "a"), (2, "b"))) - } + test("Simple resolved spool should zip with larger spool") { + val zip = resolvedSpoolFuture.zip(Seq("a", "b", "c", "d").toSpool) + assert(await(zip.toSeq) == Seq((1, "a"), (2, "b"))) + } - "be roundtrippable through toSeq/toSpool" in { - val seq = (0 to 10).toSeq - assert(await(seq.toSpool.toSeq) == seq) - } + test("Simple resolved spool should be roundtrippable through toSeq/toSpool") { + val seq = (0 to 10).toSeq + assert(await(seq.toSpool.toSeq) == seq) + } - "flatten via flatMap of toSpool" in { - val spool = Seq(Seq(1, 2), Seq(3, 4)).toSpool - val seq = await(spool.toSeq) + test("Simple resolved spool should flatten via flatMap of toSpool") { + val spool = Seq(Seq(1, 2), Seq(3, 4)).toSpool + val seq = await(spool.toSeq) - val flatSpool = - spool.flatMap { inner => Future.value(inner.toSpool) } + val flatSpool = + spool.flatMap { inner => Future.value(inner.toSpool) } - assert(await(flatSpool.flatMap(_.toSeq)) == seq.flatten) - } + assert(await(flatSpool.flatMap(_.toSeq)) == seq.flatten) + } - "take" in { - val ls = (1 to 4).toSeq.toSpool - assert(await(ls.take(2).toSeq) == Seq(1, 2)) - assert(await(ls.take(1).toSeq) == Seq(1)) - assert(await(ls.take(0).toSeq) == Seq.empty) - assert(await(ls.take(-2).toSeq) == Seq.empty) - } + test("Simple resolved spool should take") { + val ls = (1 to 4).toSeq.toSpool + assert(await(ls.take(2).toSeq) == Seq(1, 2)) + assert(await(ls.take(1).toSeq) == Seq(1)) + assert(await(ls.take(0).toSeq) == Seq.empty) + assert(await(ls.take(-2).toSeq) == Seq.empty) } - "Simple resolved spool with EOFException" should { + test("Simple resolved spool with EOF iteration on EOFException") { val p = new Promise[Spool[Int]](Throw(new EOFException("sad panda"))) val s = 1 *:: Future.value(2 *:: p) - - "EOF iteration on EOFException" in { - val xs = new ArrayBuffer[Option[Int]] - s foreachElem { xs += _ } - assert(xs.toSeq == Seq(Some(1), Some(2), None)) - } + val xs = new ArrayBuffer[Option[Int]] + s foreachElem { xs += _ } + assert(xs.toSeq == Seq(Some(1), Some(2), None)) } - "Simple resolved spool with error" should { + test("Simple resolved spool with error should return with exception on error") { val p = new Promise[Spool[Int]](Throw(new Exception("sad panda"))) val s = 1 *:: Future.value(2 *:: p) - - "return with exception on error" in { - val xs = new ArrayBuffer[Option[Int]] - s foreachElem { xs += _ } - intercept[Exception] { - await(s.toSeq) - } - } - - "return with exception on error in callback" in { - val xs = new ArrayBuffer[Option[Int]] - val f = s foreach { _ => throw new Exception("sad panda") } - intercept[Exception] { - await(f) - } + val xs = new ArrayBuffer[Option[Int]] + s foreachElem { xs += _ } + intercept[Exception] { + await(s.toSeq) } + } - "return with exception on EOFException in callback" in { - val xs = new ArrayBuffer[Option[Int]] - val f = s foreach { _ => throw new EOFException("sad panda") } - intercept[EOFException] { - await(f) - } + test("Simple resolved spool with error should return with exception on error in callback") { + val p = new Promise[Spool[Int]](Throw(new Exception("sad panda"))) + val s = 1 *:: Future.value(2 *:: p) + val xs = new ArrayBuffer[Option[Int]] + val f = s foreach { _ => throw new Exception("sad panda") } + intercept[Exception] { + await(f) } } - "Simple delayed Spool" should { - class SimpleDelayedSpoolHelper { - val p = new Promise[Spool[Int]] - val p1 = new Promise[Spool[Int]] - val p2 = new Promise[Spool[Int]] - val s = 1 *:: p + test( + "Simple resolved spool with error should return with exception on EOFException in callback") { + val p = new Promise[Spool[Int]](Throw(new Exception("sad panda"))) + val s = 1 *:: Future.value(2 *:: p) + val xs = new ArrayBuffer[Option[Int]] + val f = s foreach { _ => throw new EOFException("sad panda") } + intercept[EOFException] { + await(f) } + } - "iterate as results become available" in { - val h = new SimpleDelayedSpoolHelper - import h._ - - val xs = new ArrayBuffer[Int] - s foreach { xs += _ } - assert(xs.toSeq == Seq(1)) - p() = Return(2 *:: p1) - assert(xs.toSeq == Seq(1, 2)) - p1() = Return(Spool.empty) - assert(xs.toSeq == Seq(1, 2)) - } + test("Simple delayed Spool should iterate as results become available") { + val h = new SimpleDelayedSpoolHelper + import h._ + + val xs = new ArrayBuffer[Int] + s foreach { xs += _ } + assert(xs.toSeq == Seq(1)) + p() = Return(2 *:: p1) + assert(xs.toSeq == Seq(1, 2)) + p1() = Return(Spool.empty) + assert(xs.toSeq == Seq(1, 2)) + } - "EOF iteration on EOFException" in { - val h = new SimpleDelayedSpoolHelper - import h._ + test("Simple delayed Spool should EOF iteration on EOFException") { + val h = new SimpleDelayedSpoolHelper + import h._ - val xs = new ArrayBuffer[Option[Int]] - s foreachElem { xs += _ } - assert(xs.toSeq == Seq(Some(1))) - p() = Throw(new EOFException("sad panda")) - assert(xs.toSeq == Seq(Some(1), None)) - } + val xs = new ArrayBuffer[Option[Int]] + s foreachElem { xs += _ } + assert(xs.toSeq == Seq(Some(1))) + p() = Throw(new EOFException("sad panda")) + assert(xs.toSeq == Seq(Some(1), None)) + } - "return with exception on error" in { - val h = new SimpleDelayedSpoolHelper - import h._ + test("Simple delayed Spool should return with exception on error") { + val h = new SimpleDelayedSpoolHelper + import h._ - val xs = new ArrayBuffer[Option[Int]] - s foreachElem { xs += _ } - assert(xs.toSeq == Seq(Some(1))) - p() = Throw(new Exception("sad panda")) - intercept[Exception] { - await(s.toSeq) - } + val xs = new ArrayBuffer[Option[Int]] + s foreachElem { xs += _ } + assert(xs.toSeq == Seq(Some(1))) + p() = Throw(new Exception("sad panda")) + intercept[Exception] { + await(s.toSeq) } + } - "return with exception on error in callback" in { - val h = new SimpleDelayedSpoolHelper - import h._ + test("Simple delayed Spool should return with exception on error in callback") { + val h = new SimpleDelayedSpoolHelper + import h._ - val xs = new ArrayBuffer[Option[Int]] - val f = s foreach { _ => throw new Exception("sad panda") } - p() = Return(2 *:: p1) - intercept[Exception] { - await(f) - } + val xs = new ArrayBuffer[Option[Int]] + val f = s foreach { _ => throw new Exception("sad panda") } + p() = Return(2 *:: p1) + intercept[Exception] { + await(f) } + } - "return with exception on EOFException in callback" in { - val h = new SimpleDelayedSpoolHelper - import h._ - - val xs = new ArrayBuffer[Option[Int]] - val f = s foreach { _ => throw new EOFException("sad panda") } - p() = Return(2 *:: p1) - intercept[EOFException] { - await(f) - } - } + test("Simple delayed Spool should return with exception on EOFException in callback") { + val h = new SimpleDelayedSpoolHelper + import h._ - "return a buffered seq when complete" in { - val h = new SimpleDelayedSpoolHelper - import h._ - - val f = s.toSeq - assert(f.isDefined == false) - p() = Return(2 *:: p1) - assert(f.isDefined == false) - p1() = Return(Spool.empty) - assert(f.isDefined == true) - assert(await(f) == Seq(1, 2)) + val xs = new ArrayBuffer[Option[Int]] + val f = s foreach { _ => throw new EOFException("sad panda") } + p() = Return(2 *:: p1) + intercept[EOFException] { + await(f) } + } - "deconstruct" in { - val h = new SimpleDelayedSpoolHelper - import h._ - - assert(s match { - case fst *:: rest if fst == 1 && !rest.isDefined => true - case _ => false - }) - } + test("Simple delayed Spool should return a buffered seq when complete") { + val h = new SimpleDelayedSpoolHelper + import h._ + + val f = s.toSeq + assert(f.isDefined == false) + p() = Return(2 *:: p1) + assert(f.isDefined == false) + p1() = Return(Spool.empty) + assert(f.isDefined == true) + assert(await(f) == Seq(1, 2)) + } - "collect" in { - val h = new SimpleDelayedSpoolHelper - import h._ + test("Simple delayed Spool should deconstruct") { + val h = new SimpleDelayedSpoolHelper + import h._ - val f = s collect { - case x if x % 2 == 0 => x * 2 - } + assert(s match { + case fst *:: rest if fst == 1 && !rest.isDefined => true + case _ => false + }) + } - assert(f.isDefined == false) // 1 != 2 mod 0 - p() = Return(2 *:: p1) - assert(f.isDefined == true) - val s1 = await(f) - assert(s1 match { - case x *:: rest if x == 4 && !rest.isDefined => true - case _ => false - }) - p1() = Return(3 *:: p2) - assert(s1 match { - case x *:: rest if x == 4 && !rest.isDefined => true - case _ => false - }) - p2() = Return(4 *:: Future.value(Spool.empty[Int])) - val s1s = s1.toSeq - assert(s1s.isDefined == true) - assert(await(s1s) == Seq(4, 8)) - } + test("Simple delayed Spool should collect") { + val h = new SimpleDelayedSpoolHelper + import h._ + + val f = s collect { + case x if x % 2 == 0 => x * 2 + } + + assert(f.isDefined == false) // 1 != 2 mod 0 + p() = Return(2 *:: p1) + assert(f.isDefined == true) + val s1 = await(f) + assert(s1 match { + case x *:: rest if x == 4 && !rest.isDefined => true + case _ => false + }) + p1() = Return(3 *:: p2) + assert(s1 match { + case x *:: rest if x == 4 && !rest.isDefined => true + case _ => false + }) + p2() = Return(4 *:: Future.value(Spool.empty[Int])) + val s1s = s1.toSeq + assert(s1s.isDefined == true) + assert(await(s1s) == Seq(4, 8)) + } - "fold left" in { - val h = new SimpleDelayedSpoolHelper - import h._ + test("Simple delayed Spool should fold left") { + val h = new SimpleDelayedSpoolHelper + import h._ - val f = s.foldLeft(0) { (x, y) => x + y } + val f = s.foldLeft(0) { (x, y) => x + y } - assert(f.isDefined == false) - p() = Return(2 *:: p1) - assert(f.isDefined == false) - p1() = Return(Spool.empty) - assert(f.isDefined == true) - assert(await(f) == 3) - } + assert(f.isDefined == false) + p() = Return(2 *:: p1) + assert(f.isDefined == false) + p1() = Return(Spool.empty) + assert(f.isDefined == true) + assert(await(f) == 3) + } - "take while" in { - val h = new SimpleDelayedSpoolHelper - import h._ - - val taken = s.takeWhile(_ < 3) - assert(taken.isEmpty == false) - val f = taken.toSeq - - assert(f.isDefined == false) - p() = Return(2 *:: p1) - assert(f.isDefined == false) - p1() = Return(3 *:: p2) - // despite the Spool having an unfulfilled tail, the takeWhile is satisfied - assert(f.isDefined == true) - assert(await(f) == Seq(1, 2)) - } + test("Simple delayed Spool should take while") { + val h = new SimpleDelayedSpoolHelper + import h._ + + val taken = s.takeWhile(_ < 3) + assert(taken.isEmpty == false) + val f = taken.toSeq + + assert(f.isDefined == false) + p() = Return(2 *:: p1) + assert(f.isDefined == false) + p1() = Return(3 *:: p2) + // despite the Spool having an unfulfilled tail, the takeWhile is satisfied + assert(f.isDefined == true) + assert(await(f) == Seq(1, 2)) } - // These set of tests assert that any method called on Spool consumes only the + // These next few tests assert that any method called on Spool consumes only the // head, and doesn't force the tail. An exception is collect, which consumes // up to the first defined value. This is because collect takes a // PartialFunction, which if not defined for head, recurses on the tail, // forcing it. - "Lazily evaluated Spool" should { - "be constructed lazily" in { - applyLazily(Future.value _) - } + test("Lazily evaluated Spool should be constructed lazily") { + applyLazily(Future.value _) + } - "collect lazily" in { - applyLazily { spool => - spool.collect { - case x if x % 2 == 0 => x - } + test("Lazily evaluated Spool should collect lazily") { + applyLazily { spool => + spool.collect { + case x if x % 2 == 0 => x } } + } - "map lazily" in { - applyLazily { spool => Future.value(spool.map(_ + 1)) } - } + test("Lazily evaluated Spool should map lazily") { + applyLazily { spool => Future.value(spool.map(_ + 1)) } + } - "mapFuture lazily" in { - applyLazily { spool => spool.mapFuture(Future.value(_)) } - } + test("Lazily evaluated Spool should mapFuture lazily") { + applyLazily { spool => spool.mapFuture(Future.value(_)) } + } - "flatMap lazily" in { - applyLazily { spool => spool.flatMap { item => Future.value((item to (item + 5)).toSpool) } } - } + test("Lazily evaluated Spool should flatMap lazily") { + applyLazily { spool => spool.flatMap { item => Future.value((item to (item + 5)).toSpool) } } + } - "takeWhile lazily" in { - applyLazily { spool => - Future.value { - spool.takeWhile(_ < Int.MaxValue) - } + test("Lazily evaluated Spool should takeWhile lazily") { + applyLazily { spool => + Future.value { + spool.takeWhile(_ < Int.MaxValue) } } + } - "take lazily" in { - applyLazily { spool => - Future.value { - spool.take(2) - } + test("Lazily evaluated Spool should take lazily") { + applyLazily { spool => + Future.value { + spool.take(2) } } + } - "zip lazily" in { - applyLazily { spool => Future.value(spool.zip(spool).map { case (a, b) => a + b }) } - } - - "act eagerly when forced" in { - val (spool, tailReached) = - applyLazily { spool => Future.value(spool.map(_ + 1)) } - Await.ready { spool.map(_.force) } - assert(tailReached.isDefined) - } + test("Lazily evaluated Spool should zip lazily") { + applyLazily { spool => Future.value(spool.zip(spool).map { case (a, b) => a + b }) } + } - /** - * Confirms that the given operation does not consume an entire Spool, and then - * returns the resulting Spool and tail check for further validation. - */ - def applyLazily( - f: Spool[Int] => Future[Spool[Int]] - ): (Future[Spool[Int]], Future[Spool[Int]]) = { - val tail = new Promise[Spool[Int]] - - // A spool where only the head is valid. - def spool: Spool[Int] = 0 *:: { tail.setException(new Exception); tail } - - // create, apply, poll - val s = f(spool) - assert(!tail.isDefined) - (s, tail) - } + test("Lazily evaluated Spool should act eagerly when forced") { + val (spool, tailReached) = + applyLazily { spool => Future.value(spool.map(_ + 1)) } + Await.ready { spool.map(_.force) } + assert(tailReached.isDefined) } // Note: ++ is different from the other methods because it doesn't force its // argument. - "lazy ++" in { + test("lazy ++") { val nil = Future.value(Spool.empty[Unit]) val a = () *:: nil val p = new Promise[Unit] @@ -485,7 +482,7 @@ class SpoolTest extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { assert(p.isDefined) } - "Spool.merge should merge" in { + test("Spool.merge should merge") { forAll(Arbitrary.arbitrary[List[List[Int]]]) { ss => val all = Spool.merge(ss.map(s => Future.value(s.toSpool))) val interleaved = ss.flatMap(_.zipWithIndex).sortBy(_._2).map(_._1) @@ -493,7 +490,7 @@ class SpoolTest extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { } } - "Spool.merge should merge round robin" in { + test("Spool.merge should merge round robin") { val spools: Seq[Future[Spool[String]]] = Seq( "a" *:: Future.value("b" *:: Future.value("c" *:: Future.value(Spool.empty[String]))), "1" *:: Future.value("2" *:: Future.value("3" *:: Future.value(Spool.empty[String]))), @@ -506,14 +503,14 @@ class SpoolTest extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { ) } - "Spool.distinctBy should distinct" in { + test("Spool.distinctBy should distinct") { forAll { (s: List[Char]) => val d = s.toSpool.distinctBy(x => x).toSeq assert(await(d).toSet == s.toSet) } } - "Spool.distinctBy should distinct by in order" in { + test("Spool.distinctBy should distinct by in order") { val spool: Spool[String] = "ac" *:: Future.value("bbe" *:: Future.value("ab" *:: Future.value(Spool.empty[String]))) assert(await(spool.distinctBy(_.length).toSeq) == Seq("ac", "bbe")) diff --git a/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala b/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala index ab75e55db5..9b872a8475 100644 --- a/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala +++ b/util-core/src/test/scala/com/twitter/concurrent/TxTest.scala @@ -1,79 +1,77 @@ package com.twitter.concurrent import com.twitter.util.Return -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class TxTest extends AnyWordSpec { - "Tx.twoParty" should { - "commit when everything goes dandy" in { - val (stx, rtx) = Tx.twoParty(123) - val sf = stx.ack() - assert(sf.poll == None) - val rf = rtx.ack() - assert(sf.poll == Some(Return(Tx.Commit(())))) - assert(rf.poll == Some(Return(Tx.Commit(123)))) - } - - "abort when receiver nacks" in { - val (stx, rtx) = Tx.twoParty(123) - val sf = stx.ack() - assert(sf.poll == None) - rtx.nack() - assert(sf.poll == Some(Return(Tx.Abort))) - } +class TxTest extends AnyFunSuite { + test("Tx.twoParty should commit when everything goes dandy") { + val (stx, rtx) = Tx.twoParty(123) + val sf = stx.ack() + assert(sf.poll == None) + val rf = rtx.ack() + assert(sf.poll == Some(Return(Tx.Commit(())))) + assert(rf.poll == Some(Return(Tx.Commit(123)))) + } - "abort when sender nacks" in { - val (stx, rtx) = Tx.twoParty(123) - val rf = rtx.ack() - assert(rf.poll == None) - stx.nack() - assert(rf.poll == Some(Return(Tx.Abort))) - } + test("Tx.twoParty should abort when receiver nacks") { + val (stx, rtx) = Tx.twoParty(123) + val sf = stx.ack() + assert(sf.poll == None) + rtx.nack() + assert(sf.poll == Some(Return(Tx.Abort))) + } - "complain on ack ack" in { - val (stx, rtx) = Tx.twoParty(123) - rtx.ack() + test("Tx.twoParty should abort when sender nacks") { + val (stx, rtx) = Tx.twoParty(123) + val rf = rtx.ack() + assert(rf.poll == None) + stx.nack() + assert(rf.poll == Some(Return(Tx.Abort))) + } - assert(intercept[Exception] { - rtx.ack() - } == Tx.AlreadyAckd) - } + test("Tx.twoParty should complain on ack ack") { + val (stx, rtx) = Tx.twoParty(123) + rtx.ack() - "complain on ack nack" in { - val (stx, rtx) = Tx.twoParty(123) + assert(intercept[Exception] { rtx.ack() + } == Tx.AlreadyAckd) + } - assert(intercept[Exception] { - rtx.nack() - } == Tx.AlreadyAckd) - } + test("Tx.twoParty should complain on ack nack") { + val (stx, rtx) = Tx.twoParty(123) + rtx.ack() - "complain on nack ack" in { - val (stx, rtx) = Tx.twoParty(123) + assert(intercept[Exception] { rtx.nack() + } == Tx.AlreadyAckd) + } - assert(intercept[Exception] { - rtx.ack() - } == Tx.AlreadyNackd) - } + test("Tx.twoParty should complain on nack ack") { + val (stx, rtx) = Tx.twoParty(123) + rtx.nack() - "complain on nack nack" in { - val (stx, rtx) = Tx.twoParty(123) + assert(intercept[Exception] { + rtx.ack() + } == Tx.AlreadyNackd) + } + + test("Tx.twoParty should complain on nack nack") { + val (stx, rtx) = Tx.twoParty(123) + rtx.nack() + + assert(intercept[Exception] { rtx.nack() + } == Tx.AlreadyNackd) + } - assert(intercept[Exception] { - rtx.nack() - } == Tx.AlreadyNackd) - } + test("Tx.twoParty should complain when already done") { + val (stx, rtx) = Tx.twoParty(123) + stx.ack() + rtx.ack() - "complain when already done" in { - val (stx, rtx) = Tx.twoParty(123) + assert(intercept[Exception] { stx.ack() - rtx.ack() - - assert(intercept[Exception] { - stx.ack() - } == Tx.AlreadyDone) - } + } == Tx.AlreadyDone) } } diff --git a/util-core/src/test/scala/com/twitter/io/FilesTest.scala b/util-core/src/test/scala/com/twitter/io/FilesTest.scala index e2023cba7b..ea36ab6ce5 100644 --- a/util-core/src/test/scala/com/twitter/io/FilesTest.scala +++ b/util-core/src/test/scala/com/twitter/io/FilesTest.scala @@ -1,12 +1,11 @@ package com.twitter.io import java.io.File -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class FilesTest extends AnyWordSpec with TempFolder { - "Files" should { - - "delete" in withTempFolder { +class FilesTest extends AnyFunSuite with TempFolder { + test("Files should delete") { + withTempFolder { val tempFolder = new File(canonicalFolderName) val file = new File(tempFolder, "file") diff --git a/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala b/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala index 87f2822b1e..608c5a13dd 100644 --- a/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala +++ b/util-core/src/test/scala/com/twitter/io/StreamIOTest.scala @@ -1,25 +1,22 @@ package com.twitter.io +import org.scalatest.funsuite.AnyFunSuite import scala.util.Random -import org.scalatest.wordspec.AnyWordSpec - -class StreamIOTest extends AnyWordSpec { - "StreamIO.copy" should { - "copy the entire stream" in { - val buf = new Array[Byte](2048) - (new Random).nextBytes(buf) - val bis = new java.io.ByteArrayInputStream(buf) - val bos = new java.io.ByteArrayOutputStream() - StreamIO.copy(bis, bos) - assert(bos.toByteArray.toSeq == buf.toSeq) - } +class StreamIOTest extends AnyFunSuite { + test("StreamIO.copy should copy the entire stream") { + val buf = new Array[Byte](2048) + (new Random).nextBytes(buf) + val bis = new java.io.ByteArrayInputStream(buf) + val bos = new java.io.ByteArrayOutputStream() + StreamIO.copy(bis, bos) + assert(bos.toByteArray.toSeq == buf.toSeq) + } - "produce empty streams from empty streams" in { - val bis = new java.io.ByteArrayInputStream(new Array[Byte](0)) - val bos = new java.io.ByteArrayOutputStream() - StreamIO.copy(bis, bos) - assert(bos.size == 0) - } + test("StreamIO.copy should produce empty streams from empty streams") { + val bis = new java.io.ByteArrayInputStream(new Array[Byte](0)) + val bos = new java.io.ByteArrayOutputStream() + StreamIO.copy(bis, bos) + assert(bos.size == 0) } } diff --git a/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala b/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala index 0c13d2af34..8087f8624e 100644 --- a/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala +++ b/util-core/src/test/scala/com/twitter/io/TempDirectoryTest.scala @@ -1,21 +1,17 @@ package com.twitter.io -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class TempDirectoryTest extends AnyWordSpec { - - "TempDirectory" should { - - "create a directory when deleteAtExit is true" in { - val dir = TempDirectory.create(true) - assert(dir.exists()) - assert(dir.isDirectory) - } +class TempDirectoryTest extends AnyFunSuite { + test("TempDirectory should create a directory when deleteAtExit is true") { + val dir = TempDirectory.create(true) + assert(dir.exists()) + assert(dir.isDirectory) + } - "create a directory when deleteAtExit is false" in { - val dir = TempDirectory.create(false) - assert(dir.exists()) - assert(dir.isDirectory) - } + test("create a directory when deleteAtExit is false") { + val dir = TempDirectory.create(false) + assert(dir.exists()) + assert(dir.isDirectory) } } diff --git a/util-core/src/test/scala/com/twitter/io/TempFileTest.scala b/util-core/src/test/scala/com/twitter/io/TempFileTest.scala index e4bb3da4d2..0a8d775581 100644 --- a/util-core/src/test/scala/com/twitter/io/TempFileTest.scala +++ b/util-core/src/test/scala/com/twitter/io/TempFileTest.scala @@ -1,32 +1,28 @@ package com.twitter.io -import java.io.{ByteArrayInputStream, DataInputStream} +import java.io.ByteArrayInputStream +import java.io.DataInputStream import java.nio.charset.StandardCharsets import java.util.Arrays -import org.scalatest.wordspec.AnyWordSpec - -class TempFileTest extends AnyWordSpec { - - "TempFile" should { - - "load resources" in { - val f1 = TempFile.fromResourcePath("/java/lang/String.class") - val f2 = TempFile.fromResourcePath(getClass, "/java/lang/String.class") - val f3 = TempFile.fromSystemResourcePath("java/lang/String.class") - // Tests the basename allows basename to be less than 3 characters. - val f4 = TempFile.fromResourcePath("/1.txt") - - val c1 = Files.readBytes(f1) - val c2 = Files.readBytes(f2) - val c3 = Files.readBytes(f3) - val c4 = Files.readBytes(f4) - - assert(Arrays.equals(c1, c2)) - assert(Arrays.equals(c2, c3)) - assert(new DataInputStream(new ByteArrayInputStream(c1)).readInt == 0xcafebabe) - assert(new String(c4, StandardCharsets.UTF_8) == "Lorem ipsum dolor sit amet\n") - } - +import org.scalatest.funsuite.AnyFunSuite + +class TempFileTest extends AnyFunSuite { + + test("TempFile should load resources") { + val f1 = TempFile.fromResourcePath("/java/lang/String.class") + val f2 = TempFile.fromResourcePath(getClass, "/java/lang/String.class") + val f3 = TempFile.fromSystemResourcePath("java/lang/String.class") + // Tests the basename allows basename to be less than 3 characters. + val f4 = TempFile.fromResourcePath("/1.txt") + + val c1 = Files.readBytes(f1) + val c2 = Files.readBytes(f2) + val c3 = Files.readBytes(f3) + val c4 = Files.readBytes(f4) + + assert(Arrays.equals(c1, c2)) + assert(Arrays.equals(c2, c3)) + assert(new DataInputStream(new ByteArrayInputStream(c1)).readInt == 0xcafebabe) + assert(new String(c4, StandardCharsets.UTF_8) == "Lorem ipsum dolor sit amet\n") } - } diff --git a/util-core/src/test/scala/com/twitter/util/DurationTest.scala b/util-core/src/test/scala/com/twitter/util/DurationTest.scala index 964703ebf7..c1ead91247 100644 --- a/util-core/src/test/scala/com/twitter/util/DurationTest.scala +++ b/util-core/src/test/scala/com/twitter/util/DurationTest.scala @@ -18,287 +18,282 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ import java.util.concurrent.TimeUnit -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite trait DurationOps { val ops: Duration.type = Duration } -class DurationTest extends AnyWordSpec with DurationOps with TimeLikeSpec[Duration] { - - "Duration" should { - "*" in { - assert(1.second * 2 == 2.seconds) - assert(1.second * Long.MaxValue == Duration.Top) - assert(1.second * Long.MinValue == Duration.Bottom) - assert(-10.seconds * Long.MinValue == Duration.Top) - assert(-10.seconds * Long.MaxValue == Duration.Bottom) - assert(500.milliseconds * 4 == 2.seconds) - assert(1.nanosecond * Long.MaxValue == Long.MaxValue.nanoseconds) - assert(1.nanosecond * Long.MinValue == Long.MinValue.nanoseconds) - - assert(1.second * 2.0 == 2.seconds) - assert(500.milliseconds * 4.0 == 2.seconds) - assert(1.second * 0.5 == 500.milliseconds) - assert(1.second * Double.PositiveInfinity == Duration.Top) - assert(1.second * Double.NegativeInfinity == Duration.Bottom) - assert(1.second * Double.NaN == Duration.Undefined) - assert(1.nanosecond * (Long.MaxValue.toDouble + 1) == Duration.Top) - assert(1.nanosecond * (Long.MinValue.toDouble - 1) == Duration.Bottom) - - assert(Duration.Undefined * Double.NaN == Duration.Undefined) - assert(Duration.Top * Double.NaN == Duration.Undefined) - assert(Duration.Bottom * Double.NaN == Duration.Undefined) - - forAll { (a: Long, b: Long) => - val c = BigInt(a) * BigInt(b) - val d = a.nanoseconds * b - - assert( - (c >= Long.MaxValue && d == Duration.Top) || - (c <= Long.MinValue && d == Duration.Bottom) || - (d == c.toLong.nanoseconds) - ) - } +class DurationTest extends AnyFunSuite with DurationOps with TimeLikeSpec[Duration] { + test("Duration *") { + assert(1.second * 2 == 2.seconds) + assert(1.second * Long.MaxValue == Duration.Top) + assert(1.second * Long.MinValue == Duration.Bottom) + assert(-10.seconds * Long.MinValue == Duration.Top) + assert(-10.seconds * Long.MaxValue == Duration.Bottom) + assert(500.milliseconds * 4 == 2.seconds) + assert(1.nanosecond * Long.MaxValue == Long.MaxValue.nanoseconds) + assert(1.nanosecond * Long.MinValue == Long.MinValue.nanoseconds) + + assert(1.second * 2.0 == 2.seconds) + assert(500.milliseconds * 4.0 == 2.seconds) + assert(1.second * 0.5 == 500.milliseconds) + assert(1.second * Double.PositiveInfinity == Duration.Top) + assert(1.second * Double.NegativeInfinity == Duration.Bottom) + assert(1.second * Double.NaN == Duration.Undefined) + assert(1.nanosecond * (Long.MaxValue.toDouble + 1) == Duration.Top) + assert(1.nanosecond * (Long.MinValue.toDouble - 1) == Duration.Bottom) + + assert(Duration.Undefined * Double.NaN == Duration.Undefined) + assert(Duration.Top * Double.NaN == Duration.Undefined) + assert(Duration.Bottom * Double.NaN == Duration.Undefined) + + forAll { (a: Long, b: Long) => + val c = BigInt(a) * BigInt(b) + val d = a.nanoseconds * b + + assert( + (c >= Long.MaxValue && d == Duration.Top) || + (c <= Long.MinValue && d == Duration.Bottom) || + (d == c.toLong.nanoseconds) + ) } + } - "/" in { - assert(10.seconds / 2 == 5.seconds) - assert(1.seconds / 0 == Duration.Top) - assert((-1).seconds / 0 == Duration.Bottom) - assert(0.seconds / 0 == Duration.Undefined) - assert(Duration.Top / 0 == Duration.Undefined) - assert(Duration.Top / -1 == Duration.Bottom) - assert(Duration.Top / 1 == Duration.Top) - - assert(10.seconds / 2.0 == 5.seconds) - assert(1.seconds / 0.0 == Duration.Top) - assert((-1).seconds / 0.0 == Duration.Bottom) - assert(0.seconds / 0.0 == Duration.Undefined) - assert(Duration.Top / 0.0 == Duration.Undefined) - assert(Duration.Top / -1.0 == Duration.Bottom) - assert(Duration.Top / 1.0 == Duration.Top) - - assert(Duration.Undefined / Double.NaN == Duration.Undefined) - assert(Duration.Top / Double.NaN == Duration.Undefined) - assert(Duration.Bottom / Double.NaN == Duration.Undefined) - } + test("Duration /") { + assert(10.seconds / 2 == 5.seconds) + assert(1.seconds / 0 == Duration.Top) + assert((-1).seconds / 0 == Duration.Bottom) + assert(0.seconds / 0 == Duration.Undefined) + assert(Duration.Top / 0 == Duration.Undefined) + assert(Duration.Top / -1 == Duration.Bottom) + assert(Duration.Top / 1 == Duration.Top) + + assert(10.seconds / 2.0 == 5.seconds) + assert(1.seconds / 0.0 == Duration.Top) + assert((-1).seconds / 0.0 == Duration.Bottom) + assert(0.seconds / 0.0 == Duration.Undefined) + assert(Duration.Top / 0.0 == Duration.Undefined) + assert(Duration.Top / -1.0 == Duration.Bottom) + assert(Duration.Top / 1.0 == Duration.Top) + + assert(Duration.Undefined / Double.NaN == Duration.Undefined) + assert(Duration.Top / Double.NaN == Duration.Undefined) + assert(Duration.Bottom / Double.NaN == Duration.Undefined) + } - "%" in { - assert(10.seconds % 3.seconds == 1.second) - assert(1.second % 300.millis == 100.millis) - assert(1.second % 0.seconds == Duration.Undefined) - assert((-1).second % 0.seconds == Duration.Undefined) - assert(0.seconds % 0.seconds == Duration.Undefined) - assert(Duration.Top % 123.seconds == Duration.Undefined) - assert(Duration.Bottom % 123.seconds == Duration.Undefined) - } + test("Duration %") { + assert(10.seconds % 3.seconds == 1.second) + assert(1.second % 300.millis == 100.millis) + assert(1.second % 0.seconds == Duration.Undefined) + assert((-1).second % 0.seconds == Duration.Undefined) + assert(0.seconds % 0.seconds == Duration.Undefined) + assert(Duration.Top % 123.seconds == Duration.Undefined) + assert(Duration.Bottom % 123.seconds == Duration.Undefined) + } - "unary_-" in { - assert(-((10.seconds).inSeconds) == -10) - assert(-((Long.MinValue + 1).nanoseconds) == Long.MaxValue.nanoseconds) - assert(-(Long.MinValue.nanoseconds) == Duration.Top) - } + test("Duration unary_-") { + assert(-((10.seconds).inSeconds) == -10) + assert(-((Long.MinValue + 1).nanoseconds) == Long.MaxValue.nanoseconds) + assert(-(Long.MinValue.nanoseconds) == Duration.Top) + } - "abs" in { - assert(10.seconds.abs == 10.seconds) - assert((-10).seconds.abs == 10.seconds) - } + test("Duration abs") { + assert(10.seconds.abs == 10.seconds) + assert((-10).seconds.abs == 10.seconds) + } - "afterEpoch" in { - assert(10.seconds.afterEpoch == Time.fromMilliseconds(10000)) - } + test("Duration afterEpoch") { + assert(10.seconds.afterEpoch == Time.fromMilliseconds(10000)) + } - "fromNow" in { - Time.withCurrentTimeFrozen { _ => assert(10.seconds.fromNow == (Time.now + 10.seconds)) } - } + test("Duration fromNow") { + Time.withCurrentTimeFrozen { _ => assert(10.seconds.fromNow == (Time.now + 10.seconds)) } + } - "ago" in { - Time.withCurrentTimeFrozen { _ => assert(10.seconds.ago == (Time.now - 10.seconds)) } - } + test("Duration ago") { + Time.withCurrentTimeFrozen { _ => assert(10.seconds.ago == (Time.now - 10.seconds)) } + } - "compare" in { - assert(10.seconds < 11.seconds) - assert(10.seconds < 11000.milliseconds) - assert(11.seconds > 10.seconds) - assert(11000.milliseconds > 10.seconds) - assert(10.seconds >= 10.seconds) - assert(10.seconds <= 10.seconds) - assert(new Duration(Long.MaxValue) > 0.seconds) - } + test("Duration compare") { + assert(10.seconds < 11.seconds) + assert(10.seconds < 11000.milliseconds) + assert(11.seconds > 10.seconds) + assert(11000.milliseconds > 10.seconds) + assert(10.seconds >= 10.seconds) + assert(10.seconds <= 10.seconds) + assert(new Duration(Long.MaxValue) > 0.seconds) + } - "equals" in { - assert(Duration.Top == Duration.Top) - assert(Duration.Top != Duration.Bottom) - assert(Duration.Top != Duration.Undefined) + test("Duration equals") { + assert(Duration.Top == Duration.Top) + assert(Duration.Top != Duration.Bottom) + assert(Duration.Top != Duration.Undefined) - assert(Duration.Bottom != Duration.Top) - assert(Duration.Bottom == Duration.Bottom) - assert(Duration.Bottom != Duration.Undefined) + assert(Duration.Bottom != Duration.Top) + assert(Duration.Bottom == Duration.Bottom) + assert(Duration.Bottom != Duration.Undefined) - assert(Duration.Undefined != Duration.Top) - assert(Duration.Undefined != Duration.Bottom) - assert(Duration.Undefined == Duration.Undefined) + assert(Duration.Undefined != Duration.Top) + assert(Duration.Undefined != Duration.Bottom) + assert(Duration.Undefined == Duration.Undefined) - val tenSecs = 10.seconds - assert(tenSecs == tenSecs) - assert(tenSecs == 10.seconds) - assert(tenSecs != 11.seconds) - } + val tenSecs = 10.seconds + assert(tenSecs == tenSecs) + assert(tenSecs == 10.seconds) + assert(tenSecs != 11.seconds) + } - "+ delta" in { - forAll { (a: Long, b: Long) => - val c = BigInt(a) + BigInt(b) - val d = a.nanoseconds + b.nanosecond + test("Duration + delta") { + forAll { (a: Long, b: Long) => + val c = BigInt(a) + BigInt(b) + val d = a.nanoseconds + b.nanosecond - assert( - (c >= Long.MaxValue && d == Duration.Top) || - (c <= Long.MinValue && d == Duration.Bottom) || - (d == c.toLong.nanoseconds) - ) - } + assert( + (c >= Long.MaxValue && d == Duration.Top) || + (c <= Long.MinValue && d == Duration.Bottom) || + (d == c.toLong.nanoseconds) + ) } + } - "- delta" in { - assert(10.seconds - 5.seconds == 5.seconds) - } + test("Duration - delta") { + assert(10.seconds - 5.seconds == 5.seconds) + } - "max" in { - assert((10.seconds max 5.seconds) == 10.seconds) - assert((5.seconds max 10.seconds) == 10.seconds) - } + test("Duration max") { + assert((10.seconds max 5.seconds) == 10.seconds) + assert((5.seconds max 10.seconds) == 10.seconds) + } - "min" in { - assert((10.seconds min 5.seconds) == 5.seconds) - assert((5.seconds min 10.seconds) == 5.seconds) - } + test("Duration min") { + assert((10.seconds min 5.seconds) == 5.seconds) + assert((5.seconds min 10.seconds) == 5.seconds) + } - "moreOrLessEquals" in { - assert(10.seconds.moreOrLessEquals(9.seconds, 1.second) == true) - assert(10.seconds.moreOrLessEquals(11.seconds, 1.second) == true) - assert(10.seconds.moreOrLessEquals(8.seconds, 1.second) == false) - assert(10.seconds.moreOrLessEquals(12.seconds, 1.second) == false) - } + test("Duration moreOrLessEquals") { + assert(10.seconds.moreOrLessEquals(9.seconds, 1.second)) + assert(10.seconds.moreOrLessEquals(11.seconds, 1.second)) + assert(!10.seconds.moreOrLessEquals(8.seconds, 1.second)) + assert(!10.seconds.moreOrLessEquals(12.seconds, 1.second)) + } - "inTimeUnit" in { - assert(23.nanoseconds.inTimeUnit == ((23, TimeUnit.NANOSECONDS))) - assert(23.microseconds.inTimeUnit == ((23000, TimeUnit.NANOSECONDS))) - assert(23.milliseconds.inTimeUnit == ((23, TimeUnit.MILLISECONDS))) - assert(23.seconds.inTimeUnit == ((23, TimeUnit.SECONDS))) - assert(60.seconds.inTimeUnit == ((1, TimeUnit.MINUTES))) - } + test("Duration inTimeUnit") { + assert(23.nanoseconds.inTimeUnit == ((23, TimeUnit.NANOSECONDS))) + assert(23.microseconds.inTimeUnit == ((23000, TimeUnit.NANOSECONDS))) + assert(23.milliseconds.inTimeUnit == ((23, TimeUnit.MILLISECONDS))) + assert(23.seconds.inTimeUnit == ((23, TimeUnit.SECONDS))) + assert(60.seconds.inTimeUnit == ((1, TimeUnit.MINUTES))) + } - "inUnit" in { - assert(23.seconds.inUnit(TimeUnit.SECONDS) == 23L) - assert(23.seconds.inUnit(TimeUnit.MILLISECONDS) == 23000L) - assert(2301.millis.inUnit(TimeUnit.SECONDS) == 2L) - assert(2301.millis.inUnit(TimeUnit.MICROSECONDS) == 2301000L) - assert(4680.nanoseconds.inUnit(TimeUnit.MICROSECONDS) == 4L) - } + test("Duration inUnit") { + assert(23.seconds.inUnit(TimeUnit.SECONDS) == 23L) + assert(23.seconds.inUnit(TimeUnit.MILLISECONDS) == 23000L) + assert(2301.millis.inUnit(TimeUnit.SECONDS) == 2L) + assert(2301.millis.inUnit(TimeUnit.MICROSECONDS) == 2301000L) + assert(4680.nanoseconds.inUnit(TimeUnit.MICROSECONDS) == 4L) + } - "be hashable" in { - val map = new java.util.concurrent.ConcurrentHashMap[Duration, Int] - map.put(23.millis, 23) - assert(map.get(23.millis) == 23) - map.put(44.millis, 44) - assert(map.get(44.millis) == 44) - } + test("Duration be hashable") { + val map = new java.util.concurrent.ConcurrentHashMap[Duration, Int] + map.put(23.millis, 23) + assert(map.get(23.millis) == 23) + map.put(44.millis, 44) + assert(map.get(44.millis) == 44) + } - "toString should display as sums" in { - assert((9999999.seconds).toString == "115.days+17.hours+46.minutes+39.seconds") - } + test("Duration toString should display as sums") { + assert((9999999.seconds).toString == "115.days+17.hours+46.minutes+39.seconds") + } - "toString should handle negative durations" in { - assert((-9999999.seconds).toString == "-115.days-17.hours-46.minutes-39.seconds") - } + test("Duration toString should handle negative durations") { + assert((-9999999.seconds).toString == "-115.days-17.hours-46.minutes-39.seconds") + } - "convert java.time.Duration" in { - val javaD = java.time.Duration.ofDays(365) - val twitterD = Duration.fromJava(javaD) - assert(twitterD.inDays == 365) + test("Duration convert java.time.Duration") { + val javaD = java.time.Duration.ofDays(365) + val twitterD = Duration.fromJava(javaD) + assert(twitterD.inDays == 365) - assert(Duration.fromJava(javaD) == Duration.fromJava(javaD)) - } + assert(Duration.fromJava(javaD) == Duration.fromJava(javaD)) + } - "asJavaDuration" in { - val t0 = Duration.fromDays(10) - val t1 = t0 + 100.nanosecond - assert(t1.asJava.getNano - t0.asJava.getNano == 100) - } + test("Duration asJavaDuration") { + val t0 = Duration.fromDays(10) + val t1 = t0 + 100.nanosecond + assert(t1.asJava.getNano - t0.asJava.getNano == 100) + } - "parse the format from toString" in { - Seq( - -10.minutes, - -9999999.seconds, - 1.day + 3.hours, - 1.day, - 1.nanosecond, - 42.milliseconds, - 9999999.seconds, - Duration.Bottom, - Duration.Top, - Duration.Undefined - ).foreach { d => assert(Duration.parse(d.toString) == d) } - } + test("Duration parse the format from toString") { + Seq( + -10.minutes, + -9999999.seconds, + 1.day + 3.hours, + 1.day, + 1.nanosecond, + 42.milliseconds, + 9999999.seconds, + Duration.Bottom, + Duration.Top, + Duration.Undefined + ).foreach { d => assert(Duration.parse(d.toString) == d) } + } - "parse" in { - Seq( - " 1.second" -> 1.second, - "+1.second" -> 1.second, - "-1.second" -> -1.second, - "1.SECOND" -> 1.second, - "1.day - 1.second" -> (1.day - 1.second), - "1.day" -> 1.day, - "1.microsecond" -> 1.microsecond, - "1.millisecond" -> 1.millisecond, - "1.second" -> 1.second, - "1.second+1.minute + 1.day" -> (1.second + 1.minute + 1.day), - "1.second+1.second" -> 2.seconds, - "2.hours" -> 2.hours, - "3.days" -> 3.days, - "321.nanoseconds" -> 321.nanoseconds, - "65.minutes" -> 65.minutes, - "876.milliseconds" -> 876.milliseconds, - "98.seconds" -> 98.seconds, - "Duration.Bottom" -> Duration.Bottom, - "Duration.Top" -> Duration.Top, - "Duration.Undefined" -> Duration.Undefined, - "duration.TOP" -> Duration.Top - ).foreach { - case (s, d) => - assert(Duration.parse(s) == d) - } + test("Duration parse") { + Seq( + " 1.second" -> 1.second, + "+1.second" -> 1.second, + "-1.second" -> -1.second, + "1.SECOND" -> 1.second, + "1.day - 1.second" -> (1.day - 1.second), + "1.day" -> 1.day, + "1.microsecond" -> 1.microsecond, + "1.millisecond" -> 1.millisecond, + "1.second" -> 1.second, + "1.second+1.minute + 1.day" -> (1.second + 1.minute + 1.day), + "1.second+1.second" -> 2.seconds, + "2.hours" -> 2.hours, + "3.days" -> 3.days, + "321.nanoseconds" -> 321.nanoseconds, + "65.minutes" -> 65.minutes, + "876.milliseconds" -> 876.milliseconds, + "98.seconds" -> 98.seconds, + "Duration.Bottom" -> Duration.Bottom, + "Duration.Top" -> Duration.Top, + "Duration.Undefined" -> Duration.Undefined, + "duration.TOP" -> Duration.Top + ).foreach { + case (s, d) => + assert(Duration.parse(s) == d) } + } - "reject obvious human impostors" in { - Seq( - "", - "++1.second", - "1. second", - "1.milli", - "1.s", - "10.stardates", - "2.minutes 1.second", - "98 milliseconds", - "98 millisecons", - "99.minutes +" - ).foreach { s => - assertThrows[NumberFormatException] { - Duration.parse(s) - } + test("Duration reject obvious human impostors") { + Seq( + "", + "++1.second", + "1. second", + "1.milli", + "1.s", + "10.stardates", + "2.minutes 1.second", + "98 milliseconds", + "98 millisecons", + "99.minutes +" + ).foreach { s => + assertThrows[NumberFormatException] { + Duration.parse(s) } } } - "Top" should { - "Be scaling resistant" in { - assert(Duration.Top / 100 == Duration.Top) - assert(Duration.Top * 100 == Duration.Top) - } + test("Top should be scaling resistant") { + assert(Duration.Top / 100 == Duration.Top) + assert(Duration.Top * 100 == Duration.Top) + } - "-Top == Bottom" in { - assert(-Duration.Top == Duration.Bottom) - } + test("-Top == Bottom") { + assert(-Duration.Top == Duration.Bottom) + } - "--Top == Top" in { - assert(-(-Duration.Top) == Duration.Top) - } + test("--Top == Top") { + assert(-(-Duration.Top) == Duration.Top) } } diff --git a/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala b/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala index 1aeef90299..04efa1ad32 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureOfferTest.scala @@ -1,24 +1,23 @@ package com.twitter.concurrent +import com.twitter.util.Promise +import com.twitter.util.Return +import org.scalatest.funsuite.AnyFunSuite import org.scalatestplus.mockito.MockitoSugar -import com.twitter.util.{Promise, Return} -import org.scalatest.wordspec.AnyWordSpec -class FutureOfferTest extends AnyWordSpec with MockitoSugar { - "Future.toOffer" should { - "activate when future is satisfied (poll)" in { - val p = new Promise[Int] - val o = p.toOffer - assert(o.prepare().poll == None) - p() = Return(123) - assert(o.prepare().poll match { - case Some(Return(tx)) => - tx.ack().poll match { - case Some(Return(Tx.Commit(Return(123)))) => true - case _ => false - } - case _ => false - }) - } +class FutureOfferTest extends AnyFunSuite with MockitoSugar { + test("Future.toOffer should activate when future is satisfied (poll)") { + val p = new Promise[Int] + val o = p.toOffer + assert(o.prepare().poll == None) + p() = Return(123) + assert(o.prepare().poll match { + case Some(Return(tx)) => + tx.ack().poll match { + case Some(Return(Tx.Commit(Return(123)))) => true + case _ => false + } + case _ => false + }) } } diff --git a/util-core/src/test/scala/com/twitter/util/FutureTest.scala b/util-core/src/test/scala/com/twitter/util/FutureTest.scala index 4b8b4837ca..387abb7b1d 100644 --- a/util-core/src/test/scala/com/twitter/util/FutureTest.scala +++ b/util-core/src/test/scala/com/twitter/util/FutureTest.scala @@ -12,8 +12,9 @@ import org.mockito.Mockito.verify import org.mockito.Mockito.when import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer -import org.scalacheck.Arbitrary +import org.scalacheck.Arbitrary.arbitrary import org.scalacheck.Gen +import org.scalatest.funsuite.AnyFunSuite import org.scalatestplus.mockito.MockitoSugar import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks import scala.jdk.CollectionConverters._ @@ -21,7 +22,6 @@ import scala.runtime.NonLocalReturnControl import scala.util.Random import scala.util.control.ControlThrowable import scala.util.control.NonFatal -import org.scalatest.wordspec.AnyWordSpec private object FutureTest { def await[A](f: Future[A], timeout: Duration = 5.seconds): A = @@ -44,7 +44,7 @@ private object FutureTest { } } -class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenPropertyChecks { +class FutureTest extends AnyFunSuite with MockitoSugar with ScalaCheckDrivenPropertyChecks { import FutureTest._ implicit class FutureMatcher[A](future: Future[A]) { @@ -77,6 +77,75 @@ class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenProp } } + trait TimesHelper { + val queue = new ConcurrentLinkedQueue[Promise[Unit]] + var complete = false + var failure = false + var ninterrupt = 0 + val iteration: Future[Unit] = Future.times(3) { + val promise = new Promise[Unit] + promise.setInterruptHandler { case _ => ninterrupt += 1 } + queue add promise + promise + } + iteration + .onSuccess { _ => complete = true } + .onFailure { _ => failure = true } + assert(!complete) + assert(!failure) + } + + trait WhileDoHelper { + var i = 0 + val queue = new ConcurrentLinkedQueue[HandledPromise[Unit]] + var complete = false + var failure = false + val iteration: Future[Unit] = Future.whileDo(i < 3) { + i += 1 + val promise = new HandledPromise[Unit] + queue.add(promise) + promise + } + + iteration + .onSuccess { _ => complete = true } + .onFailure { _ => failure = true } + assert(!complete) + assert(!failure) + } + + class TraverseTestSpy() { + var goWasCalled = false + var promise: Promise[Int] = Promise[Int]() + val go: () => Promise[Int] = () => { + goWasCalled = true + promise + } + } + + trait CollectHelper { + val p0, p1 = new HandledPromise[Int] + val f: Future[Seq[Int]] = Future.collect(Seq(p0, p1)) + assert(!f.isDefined) + } + + trait CollectToTryHelper { + val p0, p1 = new HandledPromise[Int] + val f: Future[Seq[Try[Int]]] = Future.collectToTry(Seq(p0, p1)) + assert(!f.isDefined) + } + + trait SelectHelper { + var nhandled = 0 + val p0, p1 = new HandledPromise[Int] + val f: Future[Int] = p0.select(p1) + assert(!f.isDefined) + } + + trait PollHelper { + val p = new Promise[Int] + } + class FatalException extends ControlThrowable trait MkConst { @@ -85,1666 +154,1559 @@ class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenProp def exception[A](exc: Throwable): Future[A] = this(Throw(exc)) } - def test(name: String, const: MkConst): Unit = { - s"object Future ($name)" when { - "times" should { - trait TimesHelper { - val queue = new ConcurrentLinkedQueue[Promise[Unit]] - var complete = false - var failure = false - var ninterrupt = 0 - val iteration: Future[Unit] = Future.times(3) { - val promise = new Promise[Unit] - promise.setInterruptHandler { case _ => ninterrupt += 1 } - queue add promise - promise - } - iteration - .onSuccess { _ => complete = true } - .onFailure { _ => failure = true } - assert(!complete) - assert(!failure) - } + trait MonitoredHelper { + val inner = new HandledPromise[Int] + val exc = new Exception("some exception") + } - "when everything succeeds" in { - new TimesHelper { - queue.poll().setDone() - assert(!complete) - assert(!failure) + def tests(name: String, const: MkConst): Unit = { + test(s"object Future ($name) times when everything succeeds") { + new TimesHelper { + queue.poll().setDone() + assert(!complete) + assert(!failure) - queue.poll().setDone() - assert(!complete) - assert(!failure) + queue.poll().setDone() + assert(!complete) + assert(!failure) - queue.poll().setDone() - assert(complete) - assert(!failure) - } - } + queue.poll().setDone() + assert(complete) + assert(!failure) + } + } - "when some succeed and some fail" in { - new TimesHelper { - queue.poll().setDone() - assert(!complete) - assert(!failure) + test(s"object Future ($name) times when some succeed and some fail") { + new TimesHelper { + queue.poll().setDone() + assert(!complete) + assert(!failure) - queue.poll().setException(new Exception("")) - assert(!complete) - assert(failure) - } - } + queue.poll().setException(new Exception("")) + assert(!complete) + assert(failure) + } + } - "when interrupted" in { - new TimesHelper { - assert(ninterrupt == 0) - iteration.raise(new Exception) - for (i <- 1 to 3) { - assert(ninterrupt == i) - queue.poll().setDone() - } - } + test(s"object Future ($name) times when interrupted") { + new TimesHelper { + assert(ninterrupt == 0) + iteration.raise(new Exception) + for (i <- 1 to 3) { + assert(ninterrupt == i) + queue.poll().setDone() } } + } - "when" in { - var i = 0 - - await { - Future.when(false) { - Future { i += 1 } - } - } - assert(i == 0) + test(s"object Future ($name) when") { + var i = 0 - await { - Future.when(true) { - Future { i += 1 } - } + await { + Future.when(false) { + Future { i += 1 } } - assert(i == 1) - } - - "whileDo" should { - trait WhileDoHelper { - var i = 0 - val queue = new ConcurrentLinkedQueue[HandledPromise[Unit]] - var complete = false - var failure = false - val iteration: Future[Unit] = Future.whileDo(i < 3) { - i += 1 - val promise = new HandledPromise[Unit] - queue.add(promise) - promise - } + } + assert(i == 0) - iteration - .onSuccess { _ => complete = true } - .onFailure { _ => failure = true } - assert(!complete) - assert(!failure) + await { + Future.when(true) { + Future { i += 1 } } + } + assert(i == 1) + } - "when everything succeeds" in { - new WhileDoHelper { - queue.poll().setDone() - assert(!complete) - assert(!failure) + test(s"object Future ($name) whileDo when everything succeeds") { + new WhileDoHelper { + queue.poll().setDone() + assert(!complete) + assert(!failure) - queue.poll().setDone() - assert(!complete) - assert(!failure) + queue.poll().setDone() + assert(!complete) + assert(!failure) - queue.poll().setDone() + queue.poll().setDone() - assert(complete) - assert(!failure) - } - } + assert(complete) + assert(!failure) + } + } - "when some succeed and some fail" in { - new WhileDoHelper { - queue.poll().setDone() - assert(!complete) - assert(!failure) + test(s"object Future ($name) whileDo when some succeed and some fail") { + new WhileDoHelper { + queue.poll().setDone() + assert(!complete) + assert(!failure) - queue.poll().setException(new Exception("")) - assert(!complete) - assert(failure) - } - } + queue.poll().setException(new Exception("")) + assert(!complete) + assert(failure) + } + } - "when interrupted" in { - new WhileDoHelper { - assert(!queue.asScala.exists(_.handled.isDefined)) - iteration.raise(new Exception) - assert(queue.asScala.forall(_.handled.isDefined)) - } - } + test(s"object Future ($name) whileDo when interrupted") { + new WhileDoHelper { + assert(!queue.asScala.exists(_.handled.isDefined)) + iteration.raise(new Exception) + assert(queue.asScala.forall(_.handled.isDefined)) } + } - "proxyTo" should { - "reject satisfied promises" in { - val str = "um um excuse me um" - val p1 = new Promise[String]() - p1.update(Return(str)) + test(s"object Future ($name) proxyTo should reject satisfied promises") { + val str = "um um excuse me um" + val p1 = new Promise[String]() + p1.update(Return(str)) - val p2 = new Promise[String]() - val ex = intercept[IllegalStateException] { p2.proxyTo(p1) } - assert(ex.getMessage.contains(str)) - } + val p2 = new Promise[String]() + val ex = intercept[IllegalStateException] { p2.proxyTo(p1) } + assert(ex.getMessage.contains(str)) + } - "proxies success" in { - val p1 = new Promise[Int]() - val p2 = new Promise[Int]() - p2.proxyTo(p1) - p2.update(Return(5)) - assert(5 == await(p1)) - assert(5 == await(p2)) - } + test(s"object Future ($name) proxyTo proxies success") { + val p1 = new Promise[Int]() + val p2 = new Promise[Int]() + p2.proxyTo(p1) + p2.update(Return(5)) + assert(5 == await(p1)) + assert(5 == await(p2)) + } - "proxies failure" in { - val p1 = new Promise[Int]() - val p2 = new Promise[Int]() - p2.proxyTo(p1) - - val t = new RuntimeException("wurmp") - p2.update(Throw(t)) - val ex1 = intercept[RuntimeException] { await(p1) } - assert(ex1.getMessage == t.getMessage) - val ex2 = intercept[RuntimeException] { await(p2) } - assert(ex2.getMessage == t.getMessage) - } + test(s"object Future ($name) proxyTo proxies failure") { + val p1 = new Promise[Int]() + val p2 = new Promise[Int]() + p2.proxyTo(p1) + + val t = new RuntimeException("wurmp") + p2.update(Throw(t)) + val ex1 = intercept[RuntimeException] { await(p1) } + assert(ex1.getMessage == t.getMessage) + val ex2 = intercept[RuntimeException] { await(p2) } + assert(ex2.getMessage == t.getMessage) + } + + def batchedTests() = { + implicit val batchedTimer: MockTimer = new MockTimer + val batchedResult = Seq(4, 5, 6) + + test(s"object Future ($name) batched should execute after threshold is reached") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + + val batcher = Future.batched(3)(f) + + when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(batchedResult)) + batcher(1) + verify(m, never()).apply(any[Seq[Int]]) + batcher(2) + verify(m, never()).apply(any[Seq[Int]]) + batcher(3) + verify(m).apply(Iterable(1, 2, 3)) } - "batched" should { - implicit val timer: MockTimer = new MockTimer - val result = Seq(4, 5, 6) + test( + s"object Future ($name) batched should execute after bufSizeFraction threshold is reached") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + + val batcher = Future.batched(3, sizePercentile = 0.67f)(f) + when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(batchedResult)) + batcher(1) + verify(m, never()).apply(any[Seq[Int]]) + batcher(2) + verify(m).apply(Iterable(1, 2)) + } - "execute after threshold is reached" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) + test(s"object Future ($name) batched should treat bufSizeFraction return value < 0.0f as 1") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - val batcher = Future.batched(3)(f) + val batcher = Future.batched(3, sizePercentile = 0.4f)(f) - when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(result)) - batcher(1) - verify(m, never()).apply(any[Seq[Int]]) - batcher(2) - verify(m, never()).apply(any[Seq[Int]]) - batcher(3) - verify(m).apply(Iterable(1, 2, 3)) - } + when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(batchedResult)) + batcher(1) + verify(m).apply(Iterable(1)) + } + + test( + s"object Future ($name) batched should treat bufSizeFraction return value > 1.0f should return maxSizeThreshold") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + + val batcher = Future.batched(3, sizePercentile = 1.3f)(f) + + when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(batchedResult)) + batcher(1) + verify(m, never()).apply(any[Seq[Int]]) + batcher(2) + verify(m, never()).apply(any[Seq[Int]]) + batcher(3) + verify(m).apply(Iterable(1, 2, 3)) + } - "execute after bufSizeFraction threshold is reached" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) + test(s"object Future ($name) batched should execute after time threshold") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - val batcher = Future.batched(3, sizePercentile = 0.67f)(f) - when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(result)) + val batcher = Future.batched(3, 3.seconds)(f) + + Time.withCurrentTimeFrozen { control => + when(m(Iterable(1))).thenReturn(Future.value(Seq(4))) batcher(1) verify(m, never()).apply(any[Seq[Int]]) - batcher(2) - verify(m).apply(Iterable(1, 2)) - } - "treat bufSizeFraction return value < 0.0f as 1" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) + control.advance(1.second) + batchedTimer.tick() + verify(m, never()).apply(any[Seq[Int]]) - val batcher = Future.batched(3, sizePercentile = 0.4f)(f) + control.advance(1.second) + batchedTimer.tick() + verify(m, never()).apply(any[Seq[Int]]) - when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(result)) - batcher(1) + control.advance(1.second) + batchedTimer.tick() verify(m).apply(Iterable(1)) } + } - "treat bufSizeFraction return value > 1.0f should return maxSizeThreshold" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) + test(s"object Future ($name) batched should only execute once if both are reached") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - val batcher = Future.batched(3, sizePercentile = 1.3f)(f) + val batcher = Future.batched(3)(f) - when(m.apply(Iterable(1, 2, 3))).thenReturn(Future.value(result)) + Time.withCurrentTimeFrozen { control => + when(m(Iterable(1, 2, 3))).thenReturn(Future.value(batchedResult)) batcher(1) - verify(m, never()).apply(any[Seq[Int]]) batcher(2) - verify(m, never()).apply(any[Seq[Int]]) batcher(3) + control.advance(10.seconds) + batchedTimer.tick() + verify(m).apply(Iterable(1, 2, 3)) } + } - "execute after time threshold" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) - - val batcher = Future.batched(3, 3.seconds)(f) - - Time.withCurrentTimeFrozen { control => - when(m(Iterable(1))).thenReturn(Future.value(Seq(4))) - batcher(1) - verify(m, never()).apply(any[Seq[Int]]) + test(s"object Future ($name) batched should execute when flushBatch is called") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - control.advance(1.second) - timer.tick() - verify(m, never()).apply(any[Seq[Int]]) + val batcher = Future.batched(4)(f) - control.advance(1.second) - timer.tick() - verify(m, never()).apply(any[Seq[Int]]) + batcher(1) + batcher(2) + batcher(3) + batcher.flushBatch() - control.advance(1.second) - timer.tick() - verify(m).apply(Iterable(1)) - } - } + verify(m).apply(Iterable(1, 2, 3)) + } - "only execute once if both are reached" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) + test( + s"object Future ($name) batched should only execute for remaining items when flushBatch is called after size threshold is reached") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - val batcher = Future.batched(3)(f) + val batcher = Future.batched(4)(f) - Time.withCurrentTimeFrozen { control => - when(m(Iterable(1, 2, 3))).thenReturn(Future.value(result)) - batcher(1) - batcher(2) - batcher(3) - control.advance(10.seconds) - timer.tick() + batcher(1) + batcher(2) + batcher(3) + batcher(4) + batcher(5) + verify(m, times(1)).apply(Iterable(1, 2, 3, 4)) - verify(m).apply(Iterable(1, 2, 3)) - } - } + batcher.flushBatch() + verify(m, times(1)).apply(Iterable(5)) + } - "execute when flushBatch is called" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) + test( + s"object Future ($name) batched should only execute once when time threshold is reached after flushBatch is called") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - val batcher = Future.batched(4)(f) + val batcher = Future.batched(4, 3.seconds)(f) + Time.withCurrentTimeFrozen { control => batcher(1) batcher(2) batcher(3) batcher.flushBatch() + control.advance(10.seconds) + batchedTimer.tick() - verify(m).apply(Iterable(1, 2, 3)) + verify(m, times(1)).apply(Iterable(1, 2, 3)) } + } - "only execute for remaining items when flushBatch is called after size threshold is reached" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) + test( + s"object Future ($name) batched should only execute once when time threshold is reached before flushBatch is called") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - val batcher = Future.batched(4)(f) + val batcher = Future.batched(4, 3.seconds)(f) + Time.withCurrentTimeFrozen { control => batcher(1) batcher(2) batcher(3) - batcher(4) - batcher(5) - verify(m, times(1)).apply(Iterable(1, 2, 3, 4)) - + control.advance(10.seconds) + batchedTimer.tick() batcher.flushBatch() - verify(m, times(1)).apply(Iterable(5)) - } - - "only execute once when time threshold is reached after flushBatch is called" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) - - val batcher = Future.batched(4, 3.seconds)(f) - Time.withCurrentTimeFrozen { control => - batcher(1) - batcher(2) - batcher(3) - batcher.flushBatch() - control.advance(10.seconds) - timer.tick() - - verify(m, times(1)).apply(Iterable(1, 2, 3)) - } + verify(m, times(1)).apply(Iterable(1, 2, 3)) } + } - "only execute once when time threshold is reached before flushBatch is called" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) - - val batcher = Future.batched(4, 3.seconds)(f) - - Time.withCurrentTimeFrozen { control => - batcher(1) - batcher(2) - batcher(3) - control.advance(10.seconds) - timer.tick() - batcher.flushBatch() + test(s"object Future ($name) batched should propagates results") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - verify(m, times(1)).apply(Iterable(1, 2, 3)) - } - } + val batcher = Future.batched(3)(f) - "propagates results" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) + Time.withCurrentTimeFrozen { _ => + when(m(Iterable(1, 2, 3))).thenReturn(Future.value(batchedResult)) + val res1 = batcher(1) + assert(!res1.isDefined) + val res2 = batcher(2) + assert(!res2.isDefined) + val res3 = batcher(3) + assert(res1.isDefined) + assert(res2.isDefined) + assert(res3.isDefined) - val batcher = Future.batched(3)(f) + assert(await(res1) == 4) + assert(await(res2) == 5) + assert(await(res3) == 6) - Time.withCurrentTimeFrozen { _ => - when(m(Iterable(1, 2, 3))).thenReturn(Future.value(result)) - val res1 = batcher(1) - assert(!res1.isDefined) - val res2 = batcher(2) - assert(!res2.isDefined) - val res3 = batcher(3) - assert(res1.isDefined) - assert(res2.isDefined) - assert(res3.isDefined) - - assert(await(res1) == 4) - assert(await(res2) == 5) - assert(await(res3) == 6) - - verify(m).apply(Iterable(1, 2, 3)) - } + verify(m).apply(Iterable(1, 2, 3)) } + } - "not block other batches" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) - - val batcher = Future.batched(3)(f) - - Time.withCurrentTimeFrozen { _ => - val blocker = new Promise[Unit] - val thread = new Thread { - override def run(): Unit = { - when(m(result)).thenReturn(Future.value(Seq(7, 8, 9))) - batcher(4) - batcher(5) - batcher(6) - verify(m).apply(result) - blocker.setValue(()) - } - } - - when(m(Seq(1, 2, 3))).thenAnswer { - new Answer[Future[Seq[Int]]] { - def answer(invocation: InvocationOnMock): Future[Seq[Int]] = { - thread.start() - await(blocker) - Future.value(result) - } - } + test(s"object Future ($name) batched should not block other batches") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) + + val batcher = Future.batched(3)(f) + + Time.withCurrentTimeFrozen { _ => + val blocker = new Promise[Unit] + val thread = new Thread { + override def run(): Unit = { + when(m(batchedResult)).thenReturn(Future.value(Seq(7, 8, 9))) + batcher(4) + batcher(5) + batcher(6) + verify(m).apply(batchedResult) + blocker.setValue(()) } - - batcher(1) - batcher(2) - batcher(3) - verify(m).apply(Iterable(1, 2, 3)) } - } - "swallow exceptions" in { - val m = mock[Any => Future[Seq[Int]]] - val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] - when(f.compose(any())).thenReturn(m) - - val batcher = Future.batched(3)(f) - - when(m(Iterable(1, 2, 3))).thenAnswer { - new Answer[Unit] { - def answer(invocation: InvocationOnMock): Unit = { - throw new Exception + when(m(Seq(1, 2, 3))).thenAnswer { + new Answer[Future[Seq[Int]]] { + def answer(invocation: InvocationOnMock): Future[Seq[Int]] = { + thread.start() + await(blocker) + Future.value(batchedResult) } } } batcher(1) batcher(2) - batcher(3) // Success here implies no exception was thrown. + batcher(3) + verify(m).apply(Iterable(1, 2, 3)) } } - "interruptible" should { - "properly ignore the underlying future on interruption" in { - val p = Promise[Unit]() - val i = p.interruptible() - val e = new Exception - i.raise(e) - p.setDone() - assert(p.poll.contains(Return(()))) - assert(i.poll.contains(Throw(e))) - } - - "respect the underlying future" in { - val p = Promise[Unit]() - val i = p.interruptible() - p.setDone() - assert(p.poll.contains(Return(()))) - assert(i.poll.contains(Return(()))) - } + test(s"object Future ($name) batched should swallow exceptions") { + val m = mock[Any => Future[Seq[Int]]] + val f = mock[scala.collection.Seq[Int] => Future[Seq[Int]]] + when(f.compose(any())).thenReturn(m) - "do nothing for const" in { - val f = const.value(()) - val i = f.interruptible() - i.raise(new Exception()) - assert(f.poll.contains(Return(()))) - assert(i.poll.contains(Return(()))) - } - } + val batcher = Future.batched(3)(f) - "traverseSequentially" should { - class TraverseTestSpy() { - var goWasCalled = false - var promise: Promise[Int] = Promise[Int]() - val go: () => Promise[Int] = () => { - goWasCalled = true - promise + when(m(Iterable(1, 2, 3))).thenAnswer { + new Answer[Unit] { + def answer(invocation: InvocationOnMock): Unit = { + throw new Exception + } } } - "execute futures in order" in { - val first = new TraverseTestSpy() - val second = new TraverseTestSpy() - val events = Seq(first.go, second.go) + batcher(1) + batcher(2) + batcher(3) // Success here implies no exception was thrown. + } + } - val results = Future.traverseSequentially(events)(f => f()) + batchedTests() + + test( + s"object Future ($name) interruptible should properly ignore the underlying future on interruption") { + val p = Promise[Unit]() + val i = p.interruptible() + val e = new Exception + i.raise(e) + p.setDone() + assert(p.poll.contains(Return(()))) + assert(i.poll.contains(Throw(e))) + } - // At this point, none of the promises - // have been fufilled, so only the first function - // should have been called - assert(first.goWasCalled) - assert(!second.goWasCalled) + test(s"object Future ($name) interruptible should respect the underlying future") { + val p = Promise[Unit]() + val i = p.interruptible() + p.setDone() + assert(p.poll.contains(Return(()))) + assert(i.poll.contains(Return(()))) + } - // once the first promise completes, the next - // function in the sequence should be executed - first.promise.setValue(1) - assert(second.goWasCalled) + test(s"object Future ($name) interruptible should do nothing for const") { + val f = const.value(()) + val i = f.interruptible() + i.raise(new Exception()) + assert(f.poll.contains(Return(()))) + assert(i.poll.contains(Return(()))) + } - // finally, the second promise is fufilled so - // we can Await on and check the results - second.promise.setValue(2) - assert(await(results) == Seq(1, 2)) - } + test(s"object Future ($name) transverseSequentially should execute futures in order") { + val first = new TraverseTestSpy() + val second = new TraverseTestSpy() + val events = Seq(first.go, second.go) - "return with exception when the first future throws" in { - val first = new TraverseTestSpy() - val second = new TraverseTestSpy() - val events = Seq(first.go, second.go) - val results = Future.traverseSequentially(events)(f => f()) + val results = Future.traverseSequentially(events)(f => f()) - first.promise.setException(new Exception) + // At this point, none of the promises + // have been fufilled, so only the first function + // should have been called + assert(first.goWasCalled) + assert(!second.goWasCalled) - intercept[Exception] { await(results) } + // once the first promise completes, the next + // function in the sequence should be executed + first.promise.setValue(1) + assert(second.goWasCalled) - // Since first returned an exception, second should - // never have been called - assert(!second.goWasCalled) - } - } + // finally, the second promise is fufilled so + // we can Await on and check the results + second.promise.setValue(2) + assert(await(results) == Seq(1, 2)) + } - "collect" should { - trait CollectHelper { - val p0, p1 = new HandledPromise[Int] - val f: Future[Seq[Int]] = Future.collect(Seq(p0, p1)) - assert(!f.isDefined) - } + test( + s"object Future ($name) transverseSequentially should return with exception when the first future throws") { + val first = new TraverseTestSpy() + val second = new TraverseTestSpy() + val events = Seq(first.go, second.go) + val results = Future.traverseSequentially(events)(f => f()) - "only return when both futures complete" in { - new CollectHelper { - p0() = Return(1) - assert(!f.isDefined) - p1() = Return(2) - assert(f.isDefined) - assert(await(f) == Seq(1, 2)) - } - } + first.promise.setException(new Exception) - "return with exception if the first future throws" in { - new CollectHelper { - p0() = Throw(new Exception) - intercept[Exception] { await(f) } - } - } + intercept[Exception] { await(results) } - "return with exception if the second future throws" in { - new CollectHelper { - p0() = Return(1) - assert(!f.isDefined) - p1() = Throw(new Exception) - intercept[Exception] { await(f) } - } - } + // Since first returned an exception, second should + // never have been called + assert(!second.goWasCalled) + } - "propagate interrupts" in { - new CollectHelper { - val ps = Seq(p0, p1) - assert(ps.count(_.handled.isDefined) == 0) - f.raise(new Exception) - assert(ps.count(_.handled.isDefined) == 2) - } - } + test(s"object Future ($name) collect should only return when both futures complete") { + new CollectHelper { + p0() = Return(1) + assert(!f.isDefined) + p1() = Return(2) + assert(f.isDefined) + assert(await(f) == Seq(1, 2)) + } + } - "accept maps of futures" in { - val map = Map( - "1" -> Future.value("1"), - "2" -> Future.value("2") - ) + test(s"object Future ($name) collect should return with exception if the first future throws") { + new CollectHelper { + p0() = Throw(new Exception) + intercept[Exception] { await(f) } + } + } - assert(await(Future.collect(map)) == Map("1" -> "1", "2" -> "2")) - } + test( + s"object Future ($name) collect should return with exception if the second future throws") { + new CollectHelper { + p0() = Return(1) + assert(!f.isDefined) + p1() = Throw(new Exception) + intercept[Exception] { await(f) } + } + } - "work correctly if the given map is empty" in { - val map = Map.empty[String, Future[String]] - assert(await(Future.collect(map)).isEmpty) - } + test(s"object Future ($name) collect should propagate interrupts") { + new CollectHelper { + val ps = Seq(p0, p1) + assert(ps.count(_.handled.isDefined) == 0) + f.raise(new Exception) + assert(ps.count(_.handled.isDefined) == 2) + } + } - "return future exception if one of the map values is future exception" in { - val map = Map( - "1" -> Future.value("1"), - "2" -> Future.exception(new Exception) - ) + test(s"object Future ($name) collect should accept maps of futures") { + val map = Map( + "1" -> Future.value("1"), + "2" -> Future.value("2") + ) - intercept[Exception] { - await(Future.collect(map)) - } - } - } + assert(await(Future.collect(map)) == Map("1" -> "1", "2" -> "2")) + } - "collectToTry" should { + test(s"object Future ($name) collect should work correctly if the given map is empty") { + val map = Map.empty[String, Future[String]] + assert(await(Future.collect(map)).isEmpty) + } - trait CollectToTryHelper { - val p0, p1 = new HandledPromise[Int] - val f: Future[Seq[Try[Int]]] = Future.collectToTry(Seq(p0, p1)) - assert(!f.isDefined) - } + test( + s"object Future ($name) collect should return future exception if one of the map values is future exception") { + val map = Map( + "1" -> Future.value("1"), + "2" -> Future.exception(new Exception) + ) - "only return when both futures complete" in { - new CollectToTryHelper { - p0() = Return(1) - assert(!f.isDefined) - p1() = Return(2) - assert(f.isDefined) - assert(await(f) == Seq(Return(1), Return(2))) - } - } + intercept[Exception] { + await(Future.collect(map)) + } + } - "be undefined if the first future throws and the second is undefined" in { - new CollectToTryHelper { - p0() = Throw(new Exception) - assert(!f.isDefined) - } - } + test(s"object Future ($name) collectToTry should only return when both futures complete") { + new CollectToTryHelper { + p0() = Return(1) + assert(!f.isDefined) + p1() = Return(2) + assert(f.isDefined) + assert(await(f) == Seq(Return(1), Return(2))) + } + } - "return both results if the first is defined second future throws" in { - new CollectToTryHelper { - val ex = new Exception - p0() = Return(1) - assert(!f.isDefined) - p1() = Throw(ex) - assert(await(f) == Seq(Return(1), Throw(ex))) - } - } + test( + s"object Future ($name) collectToTry should be undefined if the first future throws and the second is undefined") { + new CollectToTryHelper { + p0() = Throw(new Exception) + assert(!f.isDefined) + } + } - "propagate interrupts" in { - new CollectToTryHelper { - val ps = Seq(p0, p1) - assert(ps.count(_.handled.isDefined) == 0) - f.raise(new Exception) - assert(ps.count(_.handled.isDefined) == 2) - } - } + test( + s"object Future ($name) collectToTry should return both results if the first is defined second future throws") { + new CollectToTryHelper { + val ex = new Exception + p0() = Return(1) + assert(!f.isDefined) + p1() = Throw(ex) + assert(await(f) == Seq(Return(1), Throw(ex))) + } + } + + test(s"object Future ($name) collectToTry should propagate interrupts") { + new CollectToTryHelper { + val ps = Seq(p0, p1) + assert(ps.count(_.handled.isDefined) == 0) + f.raise(new Exception) + assert(ps.count(_.handled.isDefined) == 2) } + } - "propagate locals, restoring original context" in { - val local = new Local[Int] - val f = const.value(111) + test(s"object Future ($name) should propagate locals, restoring original context") { + val local = new Local[Int] + val f = const.value(111) - var ran = 0 - local() = 1010 + var ran = 0 + local() = 1010 + f.ensure { + assert(local().contains(1010)) + local() = 1212 f.ensure { - assert(local().contains(1010)) - local() = 1212 - f.ensure { - assert(local().contains(1212)) - local() = 1313 - ran += 1 - } assert(local().contains(1212)) + local() = 1313 ran += 1 } - - assert(local().contains(1010)) - assert(ran == 2) + assert(local().contains(1212)) + ran += 1 } - "delay execution" in { - val f = const.value(111) + assert(local().contains(1010)) + assert(ran == 2) + } - var count = 0 - f.onSuccess { _ => - assert(count == 0) - f.ensure { - assert(count == 1) - count += 1 - } + test(s"object Future ($name) should delay execution") { + val f = const.value(111) - assert(count == 0) + var count = 0 + f.onSuccess { _ => + assert(count == 0) + f.ensure { + assert(count == 1) count += 1 } - assert(count == 2) + assert(count == 0) + count += 1 } - "are monitored" in { - val inner = const.value(123) - val exc = new Exception("a raw exception") - - val f = Future.monitored { - inner.ensure { throw exc } - } - - assert(f.poll.contains(Throw(exc))) - } + assert(count == 2) } - s"Future ($name)" should { - "select" which { - trait SelectHelper { - var nhandled = 0 - val p0, p1 = new HandledPromise[Int] - val f: Future[Int] = p0.select(p1) - assert(!f.isDefined) - } - - "select the first [result] to complete" in { - new SelectHelper { - p0() = Return(1) - p1() = Return(2) - assert(await(f) == 1) - } - } + test(s"object Future ($name) should are monitored") { + val inner = const.value(123) + val exc = new Exception("a raw exception") - "select the first [exception] to complete" in { - new SelectHelper { - p0() = Throw(new Exception) - p1() = Return(2) - intercept[Exception] { await(f) } - } - } - - "propagate interrupts" in { - new SelectHelper { - val ps = Seq(p0, p1) - assert(!ps.exists(_.handled.isDefined)) - f.raise(new Exception) - assert(ps.forall(_.handled.isDefined)) - } - } + val f = Future.monitored { + inner.ensure { throw exc } } - def testJoin( - label: String, - joiner: ((Future[Int], Future[Int]) => Future[(Int, Int)]) - ): Unit = { - s"join($label)" should { - trait JoinHelper { - val p0 = new HandledPromise[Int] - val p1 = new HandledPromise[Int] - val f = joiner(p0, p1) - assert(!f.isDefined) - } - - for { - left <- Seq(Ret(1), Thr(new Exception), Nvr) - right <- Seq(Ret(2), Thr(new Exception), Nvr) - leftFirst <- Seq(true, false) - } { - new JoinHelper { - if (leftFirst) { - satisfy(p0, left) - satisfy(p1, right) - } else { - satisfy(p1, right) - satisfy(p0, left) - } - (left, right) match { - // Two Throws are special because leftFirst determines the - // exception (e0 or e1). Otherwise, every other case with a - // Throw will result in just one exception. - case (Thr(e0), Thr(e1)) => - val actual = intercept[Exception] { await(f) } - assert(actual == (if (leftFirst) e0 else e1), explain(left, right, leftFirst)) - case (_, Thr(exn)) => - val actual = intercept[Exception] { await(f) } - assert(actual == exn, explain(left, right, leftFirst)) - case (Thr(exn), _) => - val actual = intercept[Exception] { await(f) } - assert(actual == exn, explain(left, right, leftFirst)) - case (Nvr, Ret(_)) | (Ret(_), Nvr) | (Nvr, Nvr) => assert(!f.isDefined) - case (Ret(a), Ret(b)) => - val expected: (Int, Int) = (a, b) - val result = await(f) - val isEqual = result == expected - assert(isEqual, explain(left, right, leftFirst)) - } - } - } + assert(f.poll.contains(Throw(exc))) + } - "propagate interrupts" in { - new JoinHelper { - assert(p0.handled.isEmpty) - assert(p1.handled.isEmpty) - val exc = new Exception - f.raise(exc) - assert(p0.handled.contains(exc)) - assert(p1.handled.contains(exc)) - } - } - } + test(s"Future ($name) select should select the first [result] to complete") { + new SelectHelper { + p0() = Return(1) + p1() = Return(2) + assert(await(f) == 1) } + } - testJoin("f join g", _ join _) - testJoin("Future.join(f, g)", Future.join(_, _)) - - def testJavaFuture(methodName: String, fn: Future[Int] => JFuture[_ <: Int]): Unit = { - methodName should { - "return the same thing as our Future when initialized" which { - val f = const.value(1) - val jf = fn(f) - assert(await(f) == jf.get()) - "must both be done" in { - assert(f.isDefined) - assert(jf.isDone) - assert(!jf.isCancelled) - } - } - - "return the same thing as our Future when set later" which { - val f = new Promise[Int] - val jf = fn(f) - f.setValue(1) - assert(await(f) == jf.get()) - "must both be done" in { - assert(f.isDefined) - assert(jf.isDone) - assert(!jf.isCancelled) - } - } - - "java future should throw an exception when failed later" in { - val f = new Promise[Int] - val jf = fn(f) - val e = new RuntimeException() - f.setException(e) - val actual = intercept[ExecutionException] { jf.get() } - val cause = intercept[RuntimeException] { throw actual.getCause } - assert(cause == e) - } - - "java future should throw an exception when failed early" in { - val f = new Promise[Int] - val e = new RuntimeException() - f.setException(e) - val jf = fn(f) - val actual = intercept[ExecutionException] { jf.get() } - val cause = intercept[RuntimeException] { throw actual.getCause } - assert(cause == e) - } - - "interrupt Future when cancelled" in { - val f = new HandledPromise[Int] - val jf = fn(f) - assert(f.handled.isEmpty) - jf.cancel(true) - intercept[java.util.concurrent.CancellationException] { - throw f.handled.get - } - } - } + test(s"Future ($name) select should select the first [exception] to complete") { + new SelectHelper { + p0() = Throw(new Exception) + p1() = Return(2) + intercept[Exception] { await(f) } } + } - testJavaFuture("toJavaFuture", { (f: Future[Int]) => f.toJavaFuture }) - testJavaFuture("toCompletableFuture", { (f: Future[Int]) => f.toCompletableFuture }) + test(s"Future ($name) select should propagate interrupts") { + new SelectHelper { + val ps = Seq(p0, p1) + assert(!ps.exists(_.handled.isDefined)) + f.raise(new Exception) + assert(ps.forall(_.handled.isDefined)) + } + } - "monitored" should { - trait MonitoredHelper { - val inner = new HandledPromise[Int] - val exc = new Exception("some exception") + def testJoin( + label: String, + joiner: ((Future[Int], Future[Int]) => Future[(Int, Int)]) + ): Unit = { + test(s"Future ($name) join($label) should propagate interrupts") { + trait JoinHelper { + val p0 = new HandledPromise[Int] + val p1 = new HandledPromise[Int] + val f = joiner(p0, p1) + assert(!f.isDefined) } - "catch raw exceptions (direct)" in { - new MonitoredHelper { - val f = Future.monitored { - throw exc - inner + for { + left <- Seq(Ret(1), Thr(new Exception), Nvr) + right <- Seq(Ret(2), Thr(new Exception), Nvr) + leftFirst <- Seq(true, false) + } { + new JoinHelper { + if (leftFirst) { + satisfy(p0, left) + satisfy(p1, right) + } else { + satisfy(p1, right) + satisfy(p0, left) } - assert(f.poll.contains(Throw(exc))) - } - } - - "catch raw exceptions (indirect), interrupting computation" in { - new MonitoredHelper { - val inner1 = new Promise[Int] - var ran = false - val f: Future[Int] = Future.monitored { - inner1 - .ensure { - // Note that these are sequenced so that interrupts - // will be delivered before inner's handler is cleared. - ran = true - try { - inner.update(Return(1)) - } catch { - case _: Throwable => fail() - } - } - .ensure { - throw exc - } - inner + (left, right) match { + // Two Throws are special because leftFirst determines the + // exception (e0 or e1). Otherwise, every other case with a + // Throw will result in just one exception. + case (Thr(e0), Thr(e1)) => + val actual = intercept[Exception] { await(f) } + assert(actual == (if (leftFirst) e0 else e1), explain(left, right, leftFirst)) + case (_, Thr(exn)) => + val actual = intercept[Exception] { await(f) } + assert(actual == exn, explain(left, right, leftFirst)) + case (Thr(exn), _) => + val actual = intercept[Exception] { await(f) } + assert(actual == exn, explain(left, right, leftFirst)) + case (Nvr, Ret(_)) | (Ret(_), Nvr) | (Nvr, Nvr) => assert(!f.isDefined) + case (Ret(a), Ret(b)) => + val expected: (Int, Int) = (a, b) + val result = await(f) + val isEqual = result == expected + assert(isEqual, explain(left, right, leftFirst)) } - assert(!ran) - assert(f.poll.isEmpty) - assert(inner.handled.isEmpty) - inner1.update(Return(1)) - assert(ran) - assert(inner.isDefined) - assert(f.poll.contains(Throw(exc))) - - assert(inner.handled.contains(exc)) } } - "link" in { - new MonitoredHelper { - val f: Future[Int] = Future.monitored { inner } - assert(inner.handled.isEmpty) - f.raise(exc) - assert(inner.handled.contains(exc)) - } + new JoinHelper { + assert(p0.handled.isEmpty) + assert(p1.handled.isEmpty) + val exc = new Exception + f.raise(exc) + assert(p0.handled.contains(exc)) + assert(p1.handled.contains(exc)) } - - // we only know this works as expected when running with JDK 8 - if (System.getProperty("java.version").startsWith("1.8")) - "doesn't leak the underlying promise after completion" in { - new MonitoredHelper { - val inner1 = new Promise[String] - val inner2 = new Promise[String] - val f: Future[String] = Future.monitored { inner2.ensure(()); inner1 } - val s: String = "." * 1024 * 10 - val sSize: Long = ObjectSizeCalculator.getObjectSize(s) - inner1.setValue(s) - val inner2Size: Long = ObjectSizeCalculator.getObjectSize(inner2) - assert(inner2Size < sSize) - } - } } } - s"Promise ($name)" should { - "apply" which { - "when we're inside of a respond block (without deadlocking)" in { - val f = Future(1) - var didRun = false - f.foreach { _ => - f mustProduce Return(1) - didRun = true - } - assert(didRun) - } - } - - "map" which { - "when it's all chill" in { - val f = Future(1).map { x => x + 1 } - assert(await(f) == 2) - } + testJoin("f join g", _ join _) + testJoin("Future.join(f, g)", Future.join(_, _)) - "when there's a problem in the passed in function" in { - val e = new Exception - val f = Future(1).map { x => - throw e - x + 1 - } - val actual = intercept[Exception] { - await(f) - } - assert(actual == e) - } + def testJavaFuture(methodName: String, fn: Future[Int] => JFuture[_ <: Int]): Unit = { + test( + s"Future ($name) $methodName should return the same thing as our Future when initialized") { + val f = const.value(1) + val jf = fn(f) + assert(await(f) == jf.get()) + assert(f.isDefined) + assert(jf.isDone) + assert(!jf.isCancelled) } - "transform" should { - val e = new Exception("rdrr") - - "values" in { - const - .value(1) - .transform { - case Return(v) => const.value(v + 1) - case Throw(_) => const.value(0) - } - .mustProduce(Return(2)) - } - - "exceptions" in { - const - .exception(e) - .transform { - case Return(_) => const.value(1) - case Throw(_) => const.value(0) - } - .mustProduce(Return(0)) - } - - "exceptions thrown during transformation" in { - const - .value(1) - .transform { - case Return(_) => const.value(throw e) - case Throw(_) => const.value(0) - } - .mustProduce(Throw(e)) - } - - "non local returns executed during transformation" in { - def ret(): String = { - val f = const.value(1).transform { - case Return(_) => - val fn = { () => return "OK" } - fn() - Future.value(ret()) - case Throw(_) => const.value(0) - } - assert(f.poll.isDefined) - val e = intercept[FutureNonLocalReturnControl] { - f.poll.get.get() - } - - val g = e.getCause match { - case t: NonLocalReturnControl[_] => t.asInstanceOf[NonLocalReturnControl[String]] - case _ => - fail() - } - assert(g.value == "OK") - "bleh" - } - ret() - } - - "fatal exceptions thrown during transformation" in { - val e = new FatalException() - - val actual = intercept[FatalException] { - const.value(1).transform { - case Return(_) => const.value(throw e) - case Throw(_) => const.value(0) - } - } - assert(actual == e) - } - - "monitors fatal exceptions" in { - val m = new HandledMonitor() - val exc = new FatalException() - - assert(m.handled == null) - - val actual = intercept[FatalException] { - Monitor.using(m) { - const.value(1).transform { _ => throw exc } - } - } - - assert(actual == exc) - assert(m.handled == exc) - } + test( + s"Future ($name) $methodName should return the same thing as our Future when set later") { + val f = new Promise[Int] + val jf = fn(f) + f.setValue(1) + assert(await(f) == jf.get()) + assert(f.isDefined) + assert(jf.isDone) + assert(!jf.isCancelled) } - "transformedBy" should { - val e = new Exception("rdrr") - - "flatMap" in { - const - .value(1) - .transformedBy(new FutureTransformer[Int, Int] { - override def flatMap(value: Int): Future[Int] = const.value(value + 1) - override def rescue(t: Throwable): Future[Int] = const.value(0) - }) - .mustProduce(Return(2)) - } + test(s"Future ($name) $methodName java future should throw an exception when failed later") { + val f = new Promise[Int] + val jf = fn(f) + val e = new RuntimeException() + f.setException(e) + val actual = intercept[ExecutionException] { jf.get() } + val cause = intercept[RuntimeException] { throw actual.getCause } + assert(cause == e) + } - "rescue" in { - const - .exception(e) - .transformedBy(new FutureTransformer[Int, Int] { - override def flatMap(value: Int): Future[Int] = const.value(value + 1) - override def rescue(t: Throwable): Future[Int] = const.value(0) - }) - .mustProduce(Return(0)) - } + test(s"Future ($name) $methodName java future should throw an exception when failed early") { + val f = new Promise[Int] + val e = new RuntimeException() + f.setException(e) + val jf = fn(f) + val actual = intercept[ExecutionException] { jf.get() } + val cause = intercept[RuntimeException] { throw actual.getCause } + assert(cause == e) + } - "exceptions thrown during transformation" in { - const - .value(1) - .transformedBy(new FutureTransformer[Int, Int] { - override def flatMap(value: Int): Future[Int] = throw e - override def rescue(t: Throwable): Future[Int] = const.value(0) - }) - .mustProduce(Throw(e)) + test(s"Future ($name) $methodName should interrupt Future when cancelled") { + val f = new HandledPromise[Int] + val jf = fn(f) + assert(f.handled.isEmpty) + jf.cancel(true) + intercept[java.util.concurrent.CancellationException] { + throw f.handled.get } + } + } - "map" in { - const - .value(1) - .transformedBy(new FutureTransformer[Int, Int] { - override def map(value: Int): Int = value + 1 - override def handle(t: Throwable): Int = 0 - }) - .mustProduce(Return(2)) - } + testJavaFuture("toJavaFuture", { (f: Future[Int]) => f.toJavaFuture }) + testJavaFuture("toCompletableFuture", { (f: Future[Int]) => f.toCompletableFuture }) - "handle" in { - const - .exception(e) - .transformedBy(new FutureTransformer[Int, Int] { - override def map(value: Int): Int = value + 1 - override def handle(t: Throwable): Int = 0 - }) - .mustProduce(Return(0)) + test(s"Future ($name) monitored should catch raw exceptions (direct)") { + new MonitoredHelper { + val f = Future.monitored { + throw exc + inner } + assert(f.poll.contains(Throw(exc))) } + } - def testSequence( - which: String, - seqop: (Future[Unit], () => Future[Unit]) => Future[Unit] - ): Unit = { - which when { - "successes" should { - "interruption of the produced future" which { - "before the antecedent Future completes, propagates back to the antecedent" in { - val f1, f2 = new HandledPromise[Unit] - val f3 = seqop(f1, () => f2) - assert(f1.handled.isEmpty) - assert(f2.handled.isEmpty) - f3.raise(new Exception) - assert(f1.handled.isDefined) - assert(f2.handled.isEmpty) - f1() = Return.Unit - assert(f2.handled.isDefined) - } - - "after the antecedent Future completes, does not propagate back to the antecedent" in { - val f1, f2 = new HandledPromise[Unit] - val f3 = seqop(f1, () => f2) - assert(f1.handled.isEmpty) - assert(f2.handled.isEmpty) - f1() = Return.Unit - f3.raise(new Exception) - assert(f1.handled.isEmpty) - assert(f2.handled.isDefined) - } - - "forward through chains" in { - val f1, f2 = new Promise[Unit] - val exc = new Exception - val f3 = new Promise[Unit] - var didInterrupt = false - f3.setInterruptHandler { - case `exc` => didInterrupt = true - } - val f4 = seqop(f1, () => seqop(f2, () => f3)) - f4.raise(exc) - assert(!didInterrupt) - f1.setDone() - assert(!didInterrupt) - f2.setDone() - assert(didInterrupt) + test( + s"Future ($name) monitored should catch raw exceptions (indirect), interrupting computation") { + new MonitoredHelper { + val inner1 = new Promise[Int] + var ran = false + val f: Future[Int] = Future.monitored { + inner1 + .ensure { + // Note that these are sequenced so that interrupts + // will be delivered before inner's handler is cleared. + ran = true + try { + inner.update(Return(1)) + } catch { + case _: Throwable => fail() } } - } - - "failures" should { - val e = new Exception - val g = seqop(Future[Unit](throw e), () => Future.Done) - - "apply" in { - val actual = intercept[Exception] { await(g) } - assert(actual == e) - } - - "respond" in { - g mustProduce Throw(e) + .ensure { + throw exc } + inner + } + assert(!ran) + assert(f.poll.isEmpty) + assert(inner.handled.isEmpty) + inner1.update(Return(1)) + assert(ran) + assert(inner.isDefined) + assert(f.poll.contains(Throw(exc))) - "when there is an exception in the passed in function" in { - val e = new Exception - val f = seqop(Future.Done, () => throw e) - val actual = intercept[Exception] { await(f) } - assert(actual == e) - } - } - } + assert(inner.handled.contains(exc)) } + } - testSequence( - "flatMap", - (a, next) => a.flatMap { _ => next() } - ) - testSequence("before", (a, next) => a.before { next() }) - - "flatMap (values)" should { - val f = Future(1).flatMap { x => Future(x + 1) } - - "apply" in { - assert(await(f) == 2) - } - - "respond" in { - f mustProduce Return(2) - } + test(s"Future ($name) monitored should link") { + new MonitoredHelper { + val f: Future[Int] = Future.monitored { inner } + assert(inner.handled.isEmpty) + f.raise(exc) + assert(inner.handled.contains(exc)) } + } - "flatten" should { - "successes" in { - val f = Future(Future(1)) - f.flatten mustProduce Return(1) + // we only know this works as expected when running with JDK 8 + if (System.getProperty("java.version").startsWith("1.8")) + test(s"Future ($name) monitored should not leak the underlying promise after completion") { + new MonitoredHelper { + val inner1 = new Promise[String] + val inner2 = new Promise[String] + val f: Future[String] = Future.monitored { inner2.ensure(()); inner1 } + val s: String = "." * 1024 * 10 + val sSize: Long = ObjectSizeCalculator.getObjectSize(s) + inner1.setValue(s) + val inner2Size: Long = ObjectSizeCalculator.getObjectSize(inner2) + assert(inner2Size < sSize) } + } - "shallow failures" in { - val e = new Exception - val f: Future[Future[Int]] = const.exception(e) - f.flatten mustProduce Throw(e) - } + test( + s"Promise ($name) should apply when we're inside of a respond block (without deadlocking)") { + val f = Future(1) + var didRun = false + f.foreach { _ => + f mustProduce Return(1) + didRun = true + } + assert(didRun) + } - "deep failures" in { - val e = new Exception - val f: Future[Future[Int]] = const.value(const.exception(e)) - f.flatten mustProduce Throw(e) - } + test(s"Promise ($name) should map when it's all chill") { + val f = Future(1).map { x => x + 1 } + assert(await(f) == 2) + } - "interruption" in { - val f1 = new HandledPromise[Future[Int]] - val f2 = new HandledPromise[Int] - val f = f1.flatten - assert(f1.handled.isEmpty) - assert(f2.handled.isEmpty) - f.raise(new Exception) - f1.handled match { - case Some(_) => - case None => fail() - } - assert(f2.handled.isEmpty) - f1() = Return(f2) - f2.handled match { - case Some(_) => - case None => fail() - } - } + test(s"Promise ($name) should map when there's a problem in the passed in function") { + val e = new Exception + val f = Future(1).map { x => + throw e + x + 1 } + val actual = intercept[Exception] { + await(f) + } + assert(actual == e) + } - "rescue" should { - val e = new Exception - - "successes" which { - val f = Future(1).rescue { case _ => Future(2) } + test(s"Promise ($name) should transform values") { + val e = new Exception("rdrr") + const + .value(1) + .transform { + case Return(v) => const.value(v + 1) + case Throw(_) => const.value(0) + } + .mustProduce(Return(2)) + } - "apply" in { - assert(await(f) == 1) - } + test(s"Promise ($name) should transform exceptions") { + val e = new Exception("rdrr") - "respond" in { - f mustProduce Return(1) - } + const + .exception(e) + .transform { + case Return(_) => const.value(1) + case Throw(_) => const.value(0) } + .mustProduce(Return(0)) + } - "failures" which { - val g = Future[Int](throw e).rescue { case _ => Future(2) } + test(s"Promise ($name) should transform exceptions thrown during transformation") { + val e = new Exception("rdrr") - "apply" in { - assert(await(g) == 2) - } + const + .value(1) + .transform { + case Return(_) => const.value(throw e) + case Throw(_) => const.value(0) + } + .mustProduce(Throw(e)) + } - "respond" in { - g mustProduce Return(2) - } + test(s"Promise ($name) should transform non local returns executed during transformation") { + val e = new Exception("rdrr") - "when the error handler errors" in { - val g = Future[Int](throw e).rescue { case x => throw x; Future(2) } - val actual = intercept[Exception] { await(g) } - assert(actual == e) - } + def ret(): String = { + val f = const.value(1).transform { + case Return(_) => + val fn = { () => return "OK" } + fn() + Future.value(ret()) + case Throw(_) => const.value(0) + } + assert(f.poll.isDefined) + val e = intercept[FutureNonLocalReturnControl] { + f.poll.get.get() } - "interruption of the produced future" which { - "before the antecedent Future completes, propagates back to the antecedent" in { - val f1, f2 = new HandledPromise[Int] - val f = f1.rescue { case _ => f2 } - assert(f1.handled.isEmpty) - assert(f2.handled.isEmpty) - f.raise(new Exception) - f1.handled match { - case Some(_) => - case None => fail() - } - assert(f2.handled.isEmpty) - f1() = Throw(new Exception) - f2.handled match { - case Some(_) => - case None => fail() - } - } - - "after the antecedent Future completes, does not propagate back to the antecedent" in { - val f1, f2 = new HandledPromise[Int] - val f = f1.rescue { case _ => f2 } - assert(f1.handled.isEmpty) - assert(f2.handled.isEmpty) - f1() = Throw(new Exception) - f.raise(new Exception) - assert(f1.handled.isEmpty) - f2.handled match { - case Some(_) => - case None => fail() - } - } + val g = e.getCause match { + case t: NonLocalReturnControl[_] => t.asInstanceOf[NonLocalReturnControl[String]] + case _ => + fail() } + assert(g.value == "OK") + "bleh" } + ret() + } - "foreach" in { - var wasCalledWith: Option[Int] = None - val f = Future(1) - f.foreach { i => wasCalledWith = Some(i) } - assert(wasCalledWith.contains(1)) - } + test(s"Promise ($name) should transform fatal exceptions thrown during transformation") { + val e = new FatalException() - "respond" should { - "when the result has arrived" in { - var wasCalledWith: Option[Int] = None - val f = Future(1) - f.respond { - case Return(i) => wasCalledWith = Some(i) - case Throw(e) => fail(e.toString) - } - assert(wasCalledWith.contains(1)) + val actual = intercept[FatalException] { + const.value(1).transform { + case Return(_) => const.value(throw e) + case Throw(_) => const.value(0) } + } + assert(actual == e) + } - "when the result has not yet arrived it buffers computations" in { - var wasCalledWith: Option[Int] = None - val f = new Promise[Int] - f.foreach { i => wasCalledWith = Some(i) } - assert(wasCalledWith.isEmpty) - f() = Return(1) - assert(wasCalledWith.contains(1)) + test(s"Promise ($name) transform monitors fatal exceptions") { + val e = new Exception("rdrr") + + val m = new HandledMonitor() + val exc = new FatalException() + + assert(m.handled == null) + + val actual = intercept[FatalException] { + Monitor.using(m) { + const.value(1).transform { _ => throw exc } } + } - "runs callbacks just once and in order (lifo)" in { - var i, j, k, h = 0 - val p = new Promise[Int] + assert(actual == exc) + assert(m.handled == exc) + } - p.ensure { - i = i + j + k + h + 1 - } - .ensure { - j = i + j + k + h + 1 - } - .ensure { - k = i + j + k + h + 1 - } - .ensure { - h = i + j + k + h + 1 - } + test(s"Promise ($name) transformedBy flatMap") { + const + .value(1) + .transformedBy(new FutureTransformer[Int, Int] { + override def flatMap(value: Int): Future[Int] = const.value(value + 1) + override def rescue(t: Throwable): Future[Int] = const.value(0) + }) + .mustProduce(Return(2)) + } - assert(i == 0) - assert(j == 0) - assert(k == 0) - assert(h == 0) + test(s"Promise ($name) transformedBy rescue") { + val e = new Exception("rdrr") + const + .exception(e) + .transformedBy(new FutureTransformer[Int, Int] { + override def flatMap(value: Int): Future[Int] = const.value(value + 1) + override def rescue(t: Throwable): Future[Int] = const.value(0) + }) + .mustProduce(Return(0)) + } - p.setValue(1) - assert(i == 8) - assert(j == 4) - assert(k == 2) - assert(h == 1) - } + test(s"Promise ($name) transformedBy exceptions thrown during transformation") { + val e = new Exception("rdrr") + const + .value(1) + .transformedBy(new FutureTransformer[Int, Int] { + override def flatMap(value: Int): Future[Int] = throw e + override def rescue(t: Throwable): Future[Int] = const.value(0) + }) + .mustProduce(Throw(e)) + } - "monitor exceptions" in { - val m = new HandledMonitor() - val exc = new Exception + test(s"Promise ($name) transformedBy map") { + const + .value(1) + .transformedBy(new FutureTransformer[Int, Int] { + override def map(value: Int): Int = value + 1 + override def handle(t: Throwable): Int = 0 + }) + .mustProduce(Return(2)) + } - assert(m.handled == null) + test(s"Promise ($name) transformedBy handle") { + val e = new Exception("rdrr") + const + .exception(e) + .transformedBy(new FutureTransformer[Int, Int] { + override def map(value: Int): Int = value + 1 + override def handle(t: Throwable): Int = 0 + }) + .mustProduce(Return(0)) + } - Monitor.using(m) { - const.value(1).ensure { throw exc } - } + def testSequence( + which: String, + seqop: (Future[Unit], () => Future[Unit]) => Future[Unit] + ): Unit = { + test( + s"Promise ($name) $which successes: interruption of the produced future, before the antecedent Future completes, propagates back to the antecedent") { + val f1, f2 = new HandledPromise[Unit] + val f3 = seqop(f1, () => f2) + assert(f1.handled.isEmpty) + assert(f2.handled.isEmpty) + f3.raise(new Exception) + assert(f1.handled.isDefined) + assert(f2.handled.isEmpty) + f1() = Return.Unit + assert(f2.handled.isDefined) + } - assert(m.handled == exc) - } + test( + s"Promise ($name) $which successes: interruption of the produced future, after the antecedent Future completes, does not propagate back to the antecedent") { + val f1, f2 = new HandledPromise[Unit] + val f3 = seqop(f1, () => f2) + assert(f1.handled.isEmpty) + assert(f2.handled.isEmpty) + f1() = Return.Unit + f3.raise(new Exception) + assert(f1.handled.isEmpty) + assert(f2.handled.isDefined) + } + + test( + s"Promise ($name) $which successes: interruption of the produced future, forward through chains") { + val f1, f2 = new Promise[Unit] + val exc = new Exception + val f3 = new Promise[Unit] + var didInterrupt = false + f3.setInterruptHandler { + case `exc` => didInterrupt = true + } + val f4 = seqop(f1, () => seqop(f2, () => f3)) + f4.raise(exc) + assert(!didInterrupt) + f1.setDone() + assert(!didInterrupt) + f2.setDone() + assert(didInterrupt) + } + + test(s"Promise ($name) $which failures shouldapply") { + val e = new Exception + val g = seqop(Future[Unit](throw e), () => Future.Done) + val actual = intercept[Exception] { await(g) } + assert(actual == e) } - "willEqual" in { - assert(await(const.value(1).willEqual(const.value(1)), 1.second)) + test(s"Promise ($name) $which failures shouldrespond") { + val e = new Exception + val g = seqop(Future[Unit](throw e), () => Future.Done) + g mustProduce Throw(e) } - "Future() handles exceptions" in { + test( + s"Promise ($name) $which failures should when there is an exception in the passed in function") { val e = new Exception - val f = Future[Int] { throw e } + val f = seqop(Future.Done, () => throw e) val actual = intercept[Exception] { await(f) } assert(actual == e) } + } - "propagate locals" in { - val local = new Local[Int] - val promise0 = new Promise[Unit] - val promise1 = new Promise[Unit] + testSequence( + "flatMap", + (a, next) => a.flatMap { _ => next() } + ) + testSequence("before", (a, next) => a.before { next() }) - local() = 1010 + test(s"Promise ($name) flatMap (values) apply") { + val f = Future(1).flatMap { x => Future(x + 1) } + assert(await(f) == 2) + } - val both = promise0.flatMap { _ => - val local0 = local() - promise1.map { _ => - val local1 = local() - (local0, local1) - } - } + test(s"Promise ($name) flatMap (values) respond") { + val f = Future(1).flatMap { x => Future(x + 1) } + f mustProduce Return(2) + } - local() = 123 - promise0() = Return.Unit - local() = 321 - promise1() = Return.Unit + test(s"Promise ($name) flatten successes") { + val f = Future(Future(1)) + f.flatten mustProduce Return(1) + } + + test(s"Promise ($name) flatten shallow failures") { + val e = new Exception + val f: Future[Future[Int]] = const.exception(e) + f.flatten mustProduce Throw(e) + } + + test(s"Promise ($name) flatten deep failures") { + val e = new Exception + val f: Future[Future[Int]] = const.value(const.exception(e)) + f.flatten mustProduce Throw(e) + } - assert(both.isDefined) - assert(await(both) == ((Some(1010), Some(1010)))) + test(s"Promise ($name) flatten interruption") { + val f1 = new HandledPromise[Future[Int]] + val f2 = new HandledPromise[Int] + val f = f1.flatten + assert(f1.handled.isEmpty) + assert(f2.handled.isEmpty) + f.raise(new Exception) + f1.handled match { + case Some(_) => + case None => fail() + } + assert(f2.handled.isEmpty) + f1() = Return(f2) + f2.handled match { + case Some(_) => + case None => fail() } + } + + test(s"Promise ($name) rescue successses apply") { + val f = Future(1).rescue { case _ => Future(2) } + assert(await(f) == 1) + } - "propagate locals across threads" in { - val local = new Local[Int] - val promise = new Promise[Option[Int]] + test(s"Promise ($name) rescue successes: respond") { + val f = Future(1).rescue { case _ => Future(2) } + f mustProduce Return(1) + } - local() = 123 - val done = promise.map { otherValue => (otherValue, local()) } + test(s"Promise ($name) rescue failures: apply") { + val e = new Exception + val g = Future[Int](throw e).rescue { case _ => Future(2) } + assert(await(g) == 2) + } - val t = new Thread { - override def run(): Unit = { - local() = 1010 - promise() = Return(local()) - } - } + test(s"Promise ($name) rescue failures: respond") { + val e = new Exception + val g = Future[Int](throw e).rescue { case _ => Future(2) } + g mustProduce Return(2) + } + + test(s"Promise ($name) rescue failures: when the error handler errors") { + val e = new Exception + val g = Future[Int](throw e).rescue { case x => throw x; Future(2) } + val actual = intercept[Exception] { await(g) } + assert(actual == e) + } - t.run() - t.join() + test( + s"Promise ($name) rescue interruption of the produced future before the antecedent Future completes, propagates back to the antecedent") { + val f1, f2 = new HandledPromise[Int] + val f = f1.rescue { case _ => f2 } + assert(f1.handled.isEmpty) + assert(f2.handled.isEmpty) + f.raise(new Exception) + f1.handled match { + case Some(_) => + case None => fail() + } + assert(f2.handled.isEmpty) + f1() = Throw(new Exception) + f2.handled match { + case Some(_) => + case None => fail() + } + } - assert(done.isDefined) - assert(await(done) == ((Some(1010), Some(123)))) + test( + s"Promise ($name) rescue interruption of the produced future after the antecedent Future completes, does not propagate back to the antecedent") { + val f1, f2 = new HandledPromise[Int] + val f = f1.rescue { case _ => f2 } + assert(f1.handled.isEmpty) + assert(f2.handled.isEmpty) + f1() = Throw(new Exception) + f.raise(new Exception) + assert(f1.handled.isEmpty) + f2.handled match { + case Some(_) => + case None => fail() } + } - "poll" should { - trait PollHelper { - val p = new Promise[Int] + test(s"Promise ($name) foreach") { + var wasCalledWith: Option[Int] = None + val f = Future(1) + f.foreach { i => wasCalledWith = Some(i) } + assert(wasCalledWith.contains(1)) + } + + test(s"Promise ($name) respond when the result has arrived") { + var wasCalledWith: Option[Int] = None + val f = Future(1) + f.respond { + case Return(i) => wasCalledWith = Some(i) + case Throw(e) => fail(e.toString) + } + assert(wasCalledWith.contains(1)) + } + + test(s"Promise ($name) respond when the result has not yet arrived it buffers computations") { + var wasCalledWith: Option[Int] = None + val f = new Promise[Int] + f.foreach { i => wasCalledWith = Some(i) } + assert(wasCalledWith.isEmpty) + f() = Return(1) + assert(wasCalledWith.contains(1)) + } + + test(s"Promise ($name) respond runs callbacks just once and in order (lifo)") { + var i, j, k, h = 0 + val p = new Promise[Int] + + p.ensure { + i = i + j + k + h + 1 } - "when waiting" in { - new PollHelper { - assert(p.poll.isEmpty) - } + .ensure { + j = i + j + k + h + 1 } - - "when succeeding" in { - new PollHelper { - p.setValue(1) - assert(p.poll.contains(Return(1))) - } + .ensure { + k = i + j + k + h + 1 } - - "when failing" in { - new PollHelper { - val e = new Exception - p.setException(e) - assert(p.poll.contains(Throw(e))) - } + .ensure { + h = i + j + k + h + 1 } + + assert(i == 0) + assert(j == 0) + assert(k == 0) + assert(h == 0) + + p.setValue(1) + assert(i == 8) + assert(j == 4) + assert(k == 2) + assert(h == 1) + } + + test(s"Promise ($name) respond monitor exceptions") { + val m = new HandledMonitor() + val exc = new Exception + + assert(m.handled == null) + + Monitor.using(m) { + const.value(1).ensure { throw exc } } - val uw = (p: Promise[Int], d: Duration, t: Timer) => { - p.within(d)(t) + assert(m.handled == exc) + } + + test(s"Promise ($name) willEqual") { + assert(await(const.value(1).willEqual(const.value(1)), 1.second)) + } + + test(s"Promise ($name) Future() handles exceptions") { + val e = new Exception + val f = Future[Int] { throw e } + val actual = intercept[Exception] { await(f) } + assert(actual == e) + } + + test(s"Promise ($name) propagate locals") { + val local = new Local[Int] + val promise0 = new Promise[Unit] + val promise1 = new Promise[Unit] + + local() = 1010 + + val both = promise0.flatMap { _ => + val local0 = local() + promise1.map { _ => + val local1 = local() + (local0, local1) + } } - val ub = (p: Promise[Int], d: Duration, t: Timer) => { - p.by(d.fromNow)(t) + local() = 123 + promise0() = Return.Unit + local() = 321 + promise1() = Return.Unit + + assert(both.isDefined) + assert(await(both) == ((Some(1010), Some(1010)))) + } + + test(s"Promise ($name) propagate locals across threads") { + val local = new Local[Int] + val promise = new Promise[Option[Int]] + + local() = 123 + val done = promise.map { otherValue => (otherValue, local()) } + + val t = new Thread { + override def run(): Unit = { + local() = 1010 + promise() = Return(local()) + } } - Seq(("within", uw), ("by", ub)).foreach { - case (label, use) => - label should { - "when we run out of time" in { - implicit val timer: Timer = new JavaTimer - val p = new HandledPromise[Int] - intercept[TimeoutException] { await(use(p, 50.milliseconds, timer)) } - timer.stop() - assert(p.handled.isEmpty) - } + t.run() + t.join() - "when everything is chill" in { - implicit val timer: Timer = new JavaTimer - val p = new Promise[Int] - p.setValue(1) - assert(await(use(p, 50.milliseconds, timer)) == 1) - timer.stop() - } + assert(done.isDefined) + assert(await(done) == ((Some(1010), Some(123)))) + } - "when timeout is forever" in { - // We manage to throw an exception inside - // the scala compiler if we use MockTimer - // here. Sigh. - implicit val timer: Timer = FailingTimer - val p = new Promise[Int] - assert(use(p, Duration.Top, timer) == p) - } + test(s"Promise ($name) poll when waiting") { + new PollHelper { + assert(p.poll.isEmpty) + } + } - "when future already satisfied" in { - implicit val timer: Timer = new NullTimer - val p = new Promise[Int] - p.setValue(3) - assert(use(p, 1.minute, timer) == p) - } + test(s"Promise ($name) poll when succeeding") { + new PollHelper { + p.setValue(1) + assert(p.poll.contains(Return(1))) + } + } - "interruption" in Time.withCurrentTimeFrozen { _ => - implicit val timer: Timer = new MockTimer - val p = new HandledPromise[Int] - val f = use(p, 50.milliseconds, timer) - assert(p.handled.isEmpty) - f.raise(new Exception) - p.handled match { - case Some(_) => - case None => fail() - } - } - } + test(s"Promise ($name) poll when failing") { + new PollHelper { + val e = new Exception + p.setException(e) + assert(p.poll.contains(Throw(e))) } + } - "raiseWithin" should { - "when we run out of time" in { - implicit val timer: Timer = new JavaTimer - val p = new HandledPromise[Int] - intercept[TimeoutException] { - await(p.raiseWithin(50.milliseconds)) - } - timer.stop() - p.handled match { - case Some(_) => - case None => fail() - } - } + val uw = (p: Promise[Int], d: Duration, t: Timer) => { + p.within(d)(t) + } - "when we run out of time, throw our stuff" in { - implicit val timer: Timer = new JavaTimer - class SkyFallException extends Exception("let the skyfall") - val skyFall = new SkyFallException - val p = new HandledPromise[Int] - intercept[SkyFallException] { - await(p.raiseWithin(50.milliseconds, skyFall)) - } - timer.stop() - p.handled match { - case Some(_) => - case None => fail() - } - assert(p.handled.contains(skyFall)) - } + val ub = (p: Promise[Int], d: Duration, t: Timer) => { + p.by(d.fromNow)(t) + } - "when we are within timeout, but inner throws TimeoutException, we don't raise" in { + Seq(("within", uw), ("by", ub)).foreach { + case (label, use) => + test(s"Promise ($name) $label when we run out of time") { implicit val timer: Timer = new JavaTimer - class SkyFallException extends Exception("let the skyfall") - val skyFall = new SkyFallException val p = new HandledPromise[Int] - intercept[TimeoutException] { - await( - p.within(20.milliseconds).raiseWithin(50.milliseconds, skyFall) - ) - } + intercept[TimeoutException] { await(use(p, 50.milliseconds, timer)) } timer.stop() assert(p.handled.isEmpty) } - "when everything is chill" in { + test(s"Promise ($name) $label when everything is chill") { implicit val timer: Timer = new JavaTimer val p = new Promise[Int] p.setValue(1) - assert(await(p.raiseWithin(50.milliseconds)) == 1) + assert(await(use(p, 50.milliseconds, timer)) == 1) timer.stop() } - "when timeout is forever" in { + test(s"Promise ($name) $label when timeout is forever") { // We manage to throw an exception inside // the scala compiler if we use MockTimer // here. Sigh. implicit val timer: Timer = FailingTimer val p = new Promise[Int] - assert(p.raiseWithin(Duration.Top) == p) + assert(use(p, Duration.Top, timer) == p) } - "when future already satisfied" in { + test(s"Promise ($name) $label when future already satisfied") { implicit val timer: Timer = new NullTimer val p = new Promise[Int] p.setValue(3) - assert(p.raiseWithin(1.minute) == p) + assert(use(p, 1.minute, timer) == p) } - "interruption" in Time.withCurrentTimeFrozen { _ => - implicit val timer: Timer = new MockTimer - val p = new HandledPromise[Int] - val f = p.raiseWithin(50.milliseconds) - assert(p.handled.isEmpty) - f.raise(new Exception) - p.handled match { - case Some(_) => - case None => fail() + test(s"Promise ($name) $label interruption") { + Time.withCurrentTimeFrozen { _ => + implicit val timer: Timer = new MockTimer + val p = new HandledPromise[Int] + val f = use(p, 50.milliseconds, timer) + assert(p.handled.isEmpty) + f.raise(new Exception) + p.handled match { + case Some(_) => + case None => fail() + } } } + } + + test(s"Promise ($name) raiseWithin when we run out of time") { + implicit val timer: Timer = new JavaTimer + val p = new HandledPromise[Int] + intercept[TimeoutException] { + await(p.raiseWithin(50.milliseconds)) } + timer.stop() + p.handled match { + case Some(_) => + case None => fail() + } + } - "masked" should { - "do unconditional interruption" in { - val p = new HandledPromise[Unit] - val f = p.masked - f.raise(new Exception()) - assert(p.handled.isEmpty) - } + test(s"Promise ($name) raiseWithin when we run out of time, throw our stuff") { + implicit val timer: Timer = new JavaTimer + class SkyFallException extends Exception("let the skyfall") + val skyFall = new SkyFallException + val p = new HandledPromise[Int] + intercept[SkyFallException] { + await(p.raiseWithin(50.milliseconds, skyFall)) + } + timer.stop() + p.handled match { + case Some(_) => + case None => fail() + } + assert(p.handled.contains(skyFall)) + } - "do conditional interruption" in { - val p = new HandledPromise[Unit] - val f1 = p.mask { - case _: TimeoutException => true - } - val f2 = p.mask { - case _: TimeoutException => true - } - f1.raise(new TimeoutException("bang!")) - assert(p.handled.isEmpty) - f2.raise(new Exception()) - assert(p.handled.isDefined) - } + test( + s"Promise ($name) raiseWithin when we are within timeout, but inner throws TimeoutException, we don't raise") { + implicit val timer: Timer = new JavaTimer + class SkyFallException extends Exception("let the skyfall") + val skyFall = new SkyFallException + val p = new HandledPromise[Int] + intercept[TimeoutException] { + await( + p.within(20.milliseconds).raiseWithin(50.milliseconds, skyFall) + ) } + timer.stop() + assert(p.handled.isEmpty) + } - "liftToTry" should { - "success" in { - val p = const(Return(3)) - assert(await(p.liftToTry) == Return(3)) - } + test(s"Promise ($name) raiseWithin when everything is chill") { + implicit val timer: Timer = new JavaTimer + val p = new Promise[Int] + p.setValue(1) + assert(await(p.raiseWithin(50.milliseconds)) == 1) + timer.stop() + } - "failure" in { - val ex = new Exception() - val p = const(Throw(ex)) - assert(await(p.liftToTry) == Throw(ex)) - } + test(s"Promise ($name) raiseWithin when timeout is forever") { + // We manage to throw an exception inside + // the scala compiler if we use MockTimer + // here. Sigh. + implicit val timer: Timer = FailingTimer + val p = new Promise[Int] + assert(p.raiseWithin(Duration.Top) == p) + } - "propagates interrupt" in { - val p = new HandledPromise[Unit] - p.liftToTry.raise(new Exception()) - assert(p.handled.isDefined) - } - } + test(s"Promise ($name) raiseWithin when future already satisfied") { + implicit val timer: Timer = new NullTimer + val p = new Promise[Int] + p.setValue(3) + assert(p.raiseWithin(1.minute) == p) + } - "lowerFromTry" should { - "success" in { - val f = const(Return(Return(3))) - assert(await(f.lowerFromTry) == 3) + test(s"Promise ($name) raiseWithin interruption") { + Time.withCurrentTimeFrozen { _ => + implicit val timer: Timer = new MockTimer + val p = new HandledPromise[Int] + val f = p.raiseWithin(50.milliseconds) + assert(p.handled.isEmpty) + f.raise(new Exception) + p.handled match { + case Some(_) => + case None => fail() } + } + } - "failure" in { - val ex = new Exception() - val p = const(Return(Throw(ex))) - val ex1 = intercept[Exception] { await(p.lowerFromTry) } - assert(ex == ex1) - } + test(s"Promise ($name) masked should do unconditional interruption") { + val p = new HandledPromise[Unit] + val f = p.masked + f.raise(new Exception()) + assert(p.handled.isEmpty) + } - "propagates interrupt" in { - val p = new HandledPromise[Try[Unit]] - p.lowerFromTry.raise(new Exception()) - assert(p.handled.isDefined) - } + test(s"Promise ($name) masked should do conditional interruption") { + val p = new HandledPromise[Unit] + val f1 = p.mask { + case _: TimeoutException => true } + val f2 = p.mask { + case _: TimeoutException => true + } + f1.raise(new TimeoutException("bang!")) + assert(p.handled.isEmpty) + f2.raise(new Exception()) + assert(p.handled.isDefined) } - s"FutureTask ($name)" should { - "return result" in { - val task = new FutureTask("hello") - task.run() - assert(await(task) == "hello") - } + test(s"Promise ($name) liftToTry success") { + val p = const(Return(3)) + assert(await(p.liftToTry) == Return(3)) + } - "throw result" in { - val task = new FutureTask[String](throw new IllegalStateException) - task.run() - intercept[IllegalStateException] { - await(task) - } + test(s"Promise ($name) liftToTry failure") { + val ex = new Exception() + val p = const(Throw(ex)) + assert(await(p.liftToTry) == Throw(ex)) + } + + test(s"Promise ($name) liftToTry propagates interrupt") { + val p = new HandledPromise[Unit] + p.liftToTry.raise(new Exception()) + assert(p.handled.isDefined) + } + + test(s"Promise ($name) lowerFromTry success") { + val f = const(Return(Return(3))) + assert(await(f.lowerFromTry) == 3) + } + + test(s"Promise ($name) lowerFromTry failure") { + val ex = new Exception() + val p = const(Return(Throw(ex))) + val ex1 = intercept[Exception] { await(p.lowerFromTry) } + assert(ex == ex1) + } + + test(s"Promise ($name) lowerFromTry propagates interrupt") { + val p = new HandledPromise[Try[Unit]] + p.lowerFromTry.raise(new Exception()) + assert(p.handled.isDefined) + } + + test(s"FutureTask ($name) should return result") { + val task = new FutureTask("hello") + task.run() + assert(await(task) == "hello") + } + + test(s"FutureTask ($name) should throw result") { + val task = new FutureTask[String](throw new IllegalStateException) + task.run() + intercept[IllegalStateException] { + await(task) } } } - test( + tests( "ConstFuture", new MkConst { def apply[A](r: Try[A]): Future[A] = Future.const(r) }) - test( + tests( "Promise", new MkConst { def apply[A](r: Try[A]): Future[A] = new Promise(r) }) - test( + tests( "Future.fromCompletableFuture", new MkConst { def apply[A](r: Try[A]): Future[A] = @@ -1752,92 +1714,81 @@ class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenProp } ) - "Future.apply" should { - "fail on NLRC" in { - def ok(): String = { - val f = Future(return "OK") - val t = intercept[FutureNonLocalReturnControl] { - f.poll.get.get() - } - val nlrc = intercept[NonLocalReturnControl[String]] { - throw t.getCause - } - assert(nlrc.value == "OK") - "NOK" + test("Future.apply should fail on NLRC") { + def ok(): String = { + val f = Future(return "OK") + val t = intercept[FutureNonLocalReturnControl] { + f.poll.get.get() + } + val nlrc = intercept[NonLocalReturnControl[String]] { + throw t.getCause } - assert(ok() == "NOK") + assert(nlrc.value == "OK") + "NOK" } + assert(ok() == "NOK") } - "Future.None" should { - "always be defined" in { - assert(Future.None.isDefined) - } - "but still None" in { - assert(await(Future.None).isEmpty) - } + test("Future.None should always be defined") { + assert(Future.None.isDefined) + } + test("Future.None should always be definedbut still None") { + assert(await(Future.None).isEmpty) } - "Future.True" should { - "always be defined" in { - assert(Future.True.isDefined) - } - "but still True" in { - assert(await(Future.True)) - } + test("Future.True should always be defined") { + assert(Future.True.isDefined) + } + test("Future.True should always be definedbut still True") { + assert(await(Future.True)) } - "Future.False" should { - "always be defined" in { - assert(Future.False.isDefined) - } - "but still False" in { - assert(!await(Future.False)) - } + test("Future.False should always be defined") { + assert(Future.False.isDefined) + } + test("Future.False should always be defined but still False") { + assert(!await(Future.False)) } - "Future.never" should { - "must be undefined" in { - assert(!Future.never.isDefined) - assert(Future.never.poll.isEmpty) - } + test("Future.never must be undefined") { + assert(!Future.never.isDefined) + assert(Future.never.poll.isEmpty) + } - "always time out" in { - intercept[TimeoutException] { Await.ready(Future.never, 0.milliseconds) } - } + test("Future.never should always time out") { + intercept[TimeoutException] { Await.ready(Future.never, 0.milliseconds) } } - "Future.onFailure" should { + test("Future.onFailure with Function1") { val nonfatal = Future.exception(new RuntimeException()) val fatal = Future.exception(new FatalException()) + val counter = new AtomicInteger() + val f: Throwable => Unit = _ => counter.incrementAndGet() + nonfatal.onFailure(f) + assert(counter.get() == 1) + fatal.onFailure(f) + assert(counter.get() == 2) + } - "with Function1" in { + test("Future.onFailure with PartialFunction") { + val nonfatal = Future.exception(new RuntimeException()) + val fatal = Future.exception(new FatalException()) + val monitor = new HandledMonitor() + Monitor.using(monitor) { val counter = new AtomicInteger() - val f: Throwable => Unit = _ => counter.incrementAndGet() - nonfatal.onFailure(f) + nonfatal.onFailure { case NonFatal(_) => counter.incrementAndGet() } assert(counter.get() == 1) - fatal.onFailure(f) - assert(counter.get() == 2) - } + assert(monitor.handled == null) - "with PartialFunction" in { - val monitor = new HandledMonitor() - Monitor.using(monitor) { - val counter = new AtomicInteger() - nonfatal.onFailure { case NonFatal(_) => counter.incrementAndGet() } - assert(counter.get() == 1) - assert(monitor.handled == null) - - // this will throw a MatchError and propagated to the monitor - fatal.onFailure { case NonFatal(_) => counter.incrementAndGet() } - assert(counter.get() == 1) - assert(monitor.handled.getClass == classOf[MatchError]) - } + // this will throw a MatchError and propagated to the monitor + fatal.onFailure { case NonFatal(_) => counter.incrementAndGet() } + assert(counter.get() == 1) + assert(monitor.handled.getClass == classOf[MatchError]) } } - "Future.sleep" should { - "Satisfy after the given amount of time" in Time.withCurrentTimeFrozen { tc => + test("Future.sleep should Satisfy after the given amount of time") { + Time.withCurrentTimeFrozen { tc => implicit val timer: MockTimer = new MockTimer val f = Future.sleep(10.seconds) @@ -1850,211 +1801,204 @@ class FutureTest extends AnyWordSpec with MockitoSugar with ScalaCheckDrivenProp assert(f.isDefined) await(f) } + } - "Be interruptible" in { - implicit val timer: MockTimer = new MockTimer + test("Future.sleep should Be interruptible") { + implicit val timer: MockTimer = new MockTimer - // sleep and grab the task that's created - val f = Future.sleep(1.second)(timer) - val task = timer.tasks(0) - // then raise a known exception - val e = new Exception("expected") - f.raise(e) + // sleep and grab the task that's created + val f = Future.sleep(1.second)(timer) + val task = timer.tasks(0) + // then raise a known exception + val e = new Exception("expected") + f.raise(e) - // we were immediately satisfied with the exception and the task was canceled - f mustProduce Throw(e) - assert(task.isCancelled) - } + // we were immediately satisfied with the exception and the task was canceled + f mustProduce Throw(e) + assert(task.isCancelled) + } - "Return Future.Done for durations <= 0" in { - implicit val timer: MockTimer = new MockTimer - assert(Future.sleep(Duration.Zero) eq Future.Done) - assert(Future.sleep((-10).seconds) eq Future.Done) - assert(timer.tasks.isEmpty) - } + test("Future.sleep should Return Future.Done for durations <= 0") { + implicit val timer: MockTimer = new MockTimer + assert(Future.sleep(Duration.Zero) eq Future.Done) + assert(Future.sleep((-10).seconds) eq Future.Done) + assert(timer.tasks.isEmpty) + } - "Return Future.never for Duration.Top" in { - implicit val timer: MockTimer = new MockTimer - assert(Future.sleep(Duration.Top) eq Future.never) - assert(timer.tasks.isEmpty) - } + test("Future.sleep should Return Future.never for Duration.Top") { + implicit val timer: MockTimer = new MockTimer + assert(Future.sleep(Duration.Top) eq Future.never) + assert(timer.tasks.isEmpty) } - "Future.select" should { - import Arbitrary.arbitrary + test("Future.select should return the first result") { val genLen = Gen.choose(1, 10) - - "return the first result" in { - forAll(genLen, arbitrary[Boolean]) { (n, fail) => - val ps = List.fill(n)(new Promise[Int]()) - assert(ps.map(_.waitqLength).sum == 0) - val f = Future.select(ps) - val i = Random.nextInt(ps.length) - val e = new Exception("sad panda") - val t = if (fail) Throw(e) else Return(i) - ps(i).update(t) - assert(f.isDefined) - val (ft, fps) = await(f) - assert(ft == t) - assert(fps.toSet == (ps.toSet - ps(i))) - } + forAll(genLen, arbitrary[Boolean]) { (n, fail) => + val ps = List.fill(n)(new Promise[Int]()) + assert(ps.map(_.waitqLength).sum == 0) + val f = Future.select(ps) + val i = Random.nextInt(ps.length) + val e = new Exception("sad panda") + val t = if (fail) Throw(e) else Return(i) + ps(i).update(t) + assert(f.isDefined) + val (ft, fps) = await(f) + assert(ft == t) + assert(fps.toSet == (ps.toSet - ps(i))) } + } - "not accumulate listeners when losing or" in { - val p = new Promise[Unit] - val q = new Promise[Unit] - p.or(q) - assert(p.waitqLength == 1) - q.setDone() - assert(p.waitqLength == 0) - } + test("Future.select should not accumulate listeners when losing or") { + val p = new Promise[Unit] + val q = new Promise[Unit] + p.or(q) + assert(p.waitqLength == 1) + q.setDone() + assert(p.waitqLength == 0) + } - "not accumulate listeners when losing select" in { - val p = new Promise[Unit] - val q = new Promise[Unit] - Future.select(Seq(p, q)) - assert(p.waitqLength == 1) - q.setDone() - assert(p.waitqLength == 0) - } + test("Future.select should not accumulate listeners when losing select") { + val p = new Promise[Unit] + val q = new Promise[Unit] + Future.select(Seq(p, q)) + assert(p.waitqLength == 1) + q.setDone() + assert(p.waitqLength == 0) + } - "not accumulate listeners if not selected" in { - forAll(genLen, arbitrary[Boolean]) { (n, fail) => - val ps = List.fill(n)(new Promise[Int]()) - assert(ps.map(_.waitqLength).sum == 0) - val f = Future.select(ps) - assert(ps.map(_.waitqLength).sum == n) - val i = Random.nextInt(ps.length) - val e = new Exception("sad panda") - val t = if (fail) Throw(e) else Return(i) - f.respond { _ => () } - assert(ps.map(_.waitqLength).sum == n) - ps(i).update(t) - assert(ps.map(_.waitqLength).sum == 0) - } + test("Future.select should not accumulate listeners if not selected") { + val genLen = Gen.choose(1, 10) + forAll(genLen, arbitrary[Boolean]) { (n, fail) => + val ps = List.fill(n)(new Promise[Int]()) + assert(ps.map(_.waitqLength).sum == 0) + val f = Future.select(ps) + assert(ps.map(_.waitqLength).sum == n) + val i = Random.nextInt(ps.length) + val e = new Exception("sad panda") + val t = if (fail) Throw(e) else Return(i) + f.respond { _ => () } + assert(ps.map(_.waitqLength).sum == n) + ps(i).update(t) + assert(ps.map(_.waitqLength).sum == 0) } + } - "fail if we attempt to select an empty future sequence" in { - val f = Future.select(Nil) - assert(f.isDefined) - val e = new IllegalArgumentException("empty future list") - val actual = intercept[IllegalArgumentException] { await(f) } - assert(actual.getMessage == e.getMessage) - } + test("Future.select should fail if we attempt to select an empty future sequence") { + val f = Future.select(Nil) + assert(f.isDefined) + val e = new IllegalArgumentException("empty future list") + val actual = intercept[IllegalArgumentException] { await(f) } + assert(actual.getMessage == e.getMessage) + } - "propagate interrupts" in { - val fs = (0 until 10).map(_ => new HandledPromise[Int]) - Future.select(fs).raise(new Exception) - assert(fs.forall(_.handled.isDefined)) - } + test("Future.select should propagate interrupts") { + val fs = (0 until 10).map(_ => new HandledPromise[Int]) + Future.select(fs).raise(new Exception) + assert(fs.forall(_.handled.isDefined)) } - // These tests are almost a carbon copy of the "Future.select" tests, they + // These "Future.selectIndex" tests are almost a carbon copy of the "Future.select" tests, they // should evolve in-sync. - "Future.selectIndex" should { - import Arbitrary.arbitrary + test("Future.selectIndex should return the first result") { val genLen = Gen.choose(1, 10) - "return the first result" in { - forAll(genLen, arbitrary[Boolean]) { (n, fail) => - val ps = IndexedSeq.fill(n)(new Promise[Int]()) - assert(ps.map(_.waitqLength).sum == 0) - val f = Future.selectIndex(ps) - val i = Random.nextInt(ps.length) - val e = new Exception("sad panda") - val t = if (fail) Throw(e) else Return(i) - ps(i).update(t) - assert(f.isDefined) - assert(await(f) == i) - } - } - - "not accumulate listeners when losing select" in { - val p = new Promise[Unit] - val q = new Promise[Unit] - Future.selectIndex(IndexedSeq(p, q)) - assert(p.waitqLength == 1) - q.setDone() - assert(p.waitqLength == 0) + forAll(genLen, arbitrary[Boolean]) { (n, fail) => + val ps = IndexedSeq.fill(n)(new Promise[Int]()) + assert(ps.map(_.waitqLength).sum == 0) + val f = Future.selectIndex(ps) + val i = Random.nextInt(ps.length) + val e = new Exception("sad panda") + val t = if (fail) Throw(e) else Return(i) + ps(i).update(t) + assert(f.isDefined) + assert(await(f) == i) } + } - "not accumulate listeners if not selected" in { - forAll(genLen, arbitrary[Boolean]) { (n, fail) => - val ps = IndexedSeq.fill(n)(new Promise[Int]()) - assert(ps.map(_.waitqLength).sum == 0) - val f = Future.selectIndex(ps) - assert(ps.map(_.waitqLength).sum == n) - val i = Random.nextInt(ps.length) - val e = new Exception("sad panda") - val t = if (fail) Throw(e) else Return(i) - f.respond { _ => () } - assert(ps.map(_.waitqLength).sum == n) - ps(i).update(t) - assert(ps.map(_.waitqLength).sum == 0) - } - } + test("Future.selectIndex should not accumulate listeners when losing select") { + val p = new Promise[Unit] + val q = new Promise[Unit] + Future.selectIndex(IndexedSeq(p, q)) + assert(p.waitqLength == 1) + q.setDone() + assert(p.waitqLength == 0) + } - "fail if we attempt to select an empty future sequence" in { - val f = Future.selectIndex(IndexedSeq.empty) - assert(f.isDefined) - val e = new IllegalArgumentException("empty future list") - val actual = intercept[IllegalArgumentException] { await(f) } - assert(actual.getMessage == e.getMessage) + test("Future.selectIndex should not accumulate listeners if not selected") { + val genLen = Gen.choose(1, 10) + forAll(genLen, arbitrary[Boolean]) { (n, fail) => + val ps = IndexedSeq.fill(n)(new Promise[Int]()) + assert(ps.map(_.waitqLength).sum == 0) + val f = Future.selectIndex(ps) + assert(ps.map(_.waitqLength).sum == n) + val i = Random.nextInt(ps.length) + val e = new Exception("sad panda") + val t = if (fail) Throw(e) else Return(i) + f.respond { _ => () } + assert(ps.map(_.waitqLength).sum == n) + ps(i).update(t) + assert(ps.map(_.waitqLength).sum == 0) } + } - "propagate interrupts" in { - val fs = IndexedSeq.fill(10)(new HandledPromise[Int]()) - Future.selectIndex(fs).raise(new Exception) - assert(fs.forall(_.handled.isDefined)) - } + test("Future.selectIndex should fail if we attempt to select an empty future sequence") { + val f = Future.selectIndex(IndexedSeq.empty) + assert(f.isDefined) + val e = new IllegalArgumentException("empty future list") + val actual = intercept[IllegalArgumentException] { await(f) } + assert(actual.getMessage == e.getMessage) } - "Future.each" should { - "iterate until an exception is thrown" in { - val exc = new Exception("done") - var next: Future[Int] = Future.value(10) - val done = Future.each(next) { - case 0 => next = Future.exception(exc) - case n => next = Future.value(n - 1) - } + test("Future.selectIndex should propagate interrupts") { + val fs = IndexedSeq.fill(10)(new HandledPromise[Int]()) + Future.selectIndex(fs).raise(new Exception) + assert(fs.forall(_.handled.isDefined)) + } - assert(done.poll.contains(Throw(exc))) + test("Future.each should iterate until an exception is thrown") { + val exc = new Exception("done") + var next: Future[Int] = Future.value(10) + val done = Future.each(next) { + case 0 => next = Future.exception(exc) + case n => next = Future.value(n - 1) } - "evaluate next one time per iteration" in { - var i, j = 0 - def next(): Future[Int] = - if (i == 10) Future.exception(new Exception) - else { - i += 1 - Future.value(i) - } + assert(done.poll.contains(Throw(exc))) + } - Future.each(next()) { i => - j += 1 - assert(i == j) + test("Future.each should evaluate next one time per iteration") { + var i, j = 0 + def next(): Future[Int] = + if (i == 10) Future.exception(new Exception) + else { + i += 1 + Future.value(i) } - } - "terminate if the body throws an exception" in { - val exc = new Exception("body exception") - var i = 0 - def next(): Future[Int] = Future.value({ i += 1; i }) - val done = Future.each(next()) { - case 10 => throw exc - case _ => - } + Future.each(next()) { i => + j += 1 + assert(i == j) + } + } - assert(done.poll.contains(Throw(exc))) - assert(i == 10) + test("Future.each should terminate if the body throws an exception") { + val exc = new Exception("body exception") + var i = 0 + def next(): Future[Int] = Future.value({ i += 1; i }) + val done = Future.each(next()) { + case 10 => throw exc + case _ => } - "terminate when 'next' throws" in { - val exc = new Exception - def next(): Future[Int] = throw exc - val done = Future.each(next()) { _ => throw exc } + assert(done.poll.contains(Throw(exc))) + assert(i == 10) + } - assert(done.poll.contains(Throw(Future.NextThrewException(exc)))) - } + test("Future.each should terminate when 'next' throws") { + val exc = new Exception + def next(): Future[Int] = throw exc + val done = Future.each(next()) { _ => throw exc } + + assert(done.poll.contains(Throw(Future.NextThrewException(exc)))) } } diff --git a/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala b/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala index 2d61bbe02d..da989d1fc9 100644 --- a/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala +++ b/util-core/src/test/scala/com/twitter/util/LastWriteWinsQueueTest.scala @@ -1,26 +1,24 @@ package com.twitter.util -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class LastWriteWinsQueueTest extends AnyWordSpec { - "LastWriteWinsQueue" should { - val queue = new LastWriteWinsQueue[String] +class LastWriteWinsQueueTest extends AnyFunSuite { + val queue = new LastWriteWinsQueue[String] - "add & remove items" in { - assert(queue.size == 0) - queue.add("1") - assert(queue.size == 1) - assert(queue.remove == "1") - assert(queue.size == 0) - } + test("LastWriteWinsQueue should add & remove items") { + assert(queue.size == 0) + queue.add("1") + assert(queue.size == 1) + assert(queue.remove == "1") + assert(queue.size == 0) + } - "last write wins" in { - queue.add("1") - queue.add("2") - assert(queue.size == 1) - assert(queue.poll() == "2") - assert(queue.size == 0) - assert(queue.poll() == null) - } + test("LastWriteWinsQueue should last write wins") { + queue.add("1") + queue.add("2") + assert(queue.size == 1) + assert(queue.poll() == "2") + assert(queue.size == 0) + assert(queue.poll() == null) } } diff --git a/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala b/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala index ccabff1ef4..2ea31df956 100644 --- a/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala +++ b/util-core/src/test/scala/com/twitter/util/NetUtilTest.scala @@ -1,147 +1,144 @@ package com.twitter.util import java.net.InetAddress - -import org.scalatest.wordspec.AnyWordSpec - -class NetUtilTest extends AnyWordSpec { - "NetUtil" should { - "isIpv4Address" in { - for (i <- 0.to(255)) { - assert(NetUtil.isIpv4Address("%d.0.0.0".format(i)) == true) - assert(NetUtil.isIpv4Address("0.%d.0.0".format(i)) == true) - assert(NetUtil.isIpv4Address("0.0.%d.0".format(i)) == true) - assert(NetUtil.isIpv4Address("0.0.0.%d".format(i)) == true) - assert(NetUtil.isIpv4Address("%d.%d.%d.%d".format(i, i, i, i)) == true) - } - - assert(NetUtil.isIpv4Address("") == false) - assert(NetUtil.isIpv4Address("no") == false) - assert(NetUtil.isIpv4Address("::127.0.0.1") == false) - assert(NetUtil.isIpv4Address("-1.0.0.0") == false) - assert(NetUtil.isIpv4Address("256.0.0.0") == false) - assert(NetUtil.isIpv4Address("0.256.0.0") == false) - assert(NetUtil.isIpv4Address("0.0.256.0") == false) - assert(NetUtil.isIpv4Address("0.0.0.256") == false) - assert(NetUtil.isIpv4Address("x1.2.3.4") == false) - assert(NetUtil.isIpv4Address("1.x2.3.4") == false) - assert(NetUtil.isIpv4Address("1.2.x3.4") == false) - assert(NetUtil.isIpv4Address("1.2.3.x4") == false) - assert(NetUtil.isIpv4Address("1.2.3.4x") == false) - assert(NetUtil.isIpv4Address(" 1.2.3.4") == false) - assert(NetUtil.isIpv4Address("1.2.3.4 ") == false) - assert(NetUtil.isIpv4Address(".") == false) - assert(NetUtil.isIpv4Address("....") == false) - assert(NetUtil.isIpv4Address("1....") == false) - assert(NetUtil.isIpv4Address("1.2...") == false) - assert(NetUtil.isIpv4Address("1.2.3.") == false) - assert(NetUtil.isIpv4Address(".2.3.4") == false) +import org.scalatest.funsuite.AnyFunSuite + +class NetUtilTest extends AnyFunSuite { + test("NetUtil isIpv4Address") { + for (i <- 0.to(255)) { + assert(NetUtil.isIpv4Address("%d.0.0.0".format(i)) == true) + assert(NetUtil.isIpv4Address("0.%d.0.0".format(i)) == true) + assert(NetUtil.isIpv4Address("0.0.%d.0".format(i)) == true) + assert(NetUtil.isIpv4Address("0.0.0.%d".format(i)) == true) + assert(NetUtil.isIpv4Address("%d.%d.%d.%d".format(i, i, i, i)) == true) } - "isPrivate" in { - assert(NetUtil.isPrivateAddress(InetAddress.getByName("0.0.0.0")) == false) - assert(NetUtil.isPrivateAddress(InetAddress.getByName("199.59.148.13")) == false) - assert(NetUtil.isPrivateAddress(InetAddress.getByName("10.0.0.0")) == true) - assert(NetUtil.isPrivateAddress(InetAddress.getByName("10.255.255.255")) == true) - assert(NetUtil.isPrivateAddress(InetAddress.getByName("172.16.0.0")) == true) - assert(NetUtil.isPrivateAddress(InetAddress.getByName("172.31.255.255")) == true) - assert(NetUtil.isPrivateAddress(InetAddress.getByName("192.168.0.0")) == true) - assert(NetUtil.isPrivateAddress(InetAddress.getByName("192.168.255.255")) == true) - } - - "ipToInt" in { - assert(NetUtil.ipToInt("0.0.0.0") == 0) - assert(NetUtil.ipToInt("255.255.255.255") == 0xffffffff) - assert(NetUtil.ipToInt("255.255.255.0") == 0xffffff00) - assert(NetUtil.ipToInt("255.0.255.0") == 0xff00ff00) - assert(NetUtil.ipToInt("61.197.253.56") == 0x3dc5fd38) - intercept[IllegalArgumentException] { - NetUtil.ipToInt("256.0.255.0") - } - } + assert(NetUtil.isIpv4Address("") == false) + assert(NetUtil.isIpv4Address("no") == false) + assert(NetUtil.isIpv4Address("::127.0.0.1") == false) + assert(NetUtil.isIpv4Address("-1.0.0.0") == false) + assert(NetUtil.isIpv4Address("256.0.0.0") == false) + assert(NetUtil.isIpv4Address("0.256.0.0") == false) + assert(NetUtil.isIpv4Address("0.0.256.0") == false) + assert(NetUtil.isIpv4Address("0.0.0.256") == false) + assert(NetUtil.isIpv4Address("x1.2.3.4") == false) + assert(NetUtil.isIpv4Address("1.x2.3.4") == false) + assert(NetUtil.isIpv4Address("1.2.x3.4") == false) + assert(NetUtil.isIpv4Address("1.2.3.x4") == false) + assert(NetUtil.isIpv4Address("1.2.3.4x") == false) + assert(NetUtil.isIpv4Address(" 1.2.3.4") == false) + assert(NetUtil.isIpv4Address("1.2.3.4 ") == false) + assert(NetUtil.isIpv4Address(".") == false) + assert(NetUtil.isIpv4Address("....") == false) + assert(NetUtil.isIpv4Address("1....") == false) + assert(NetUtil.isIpv4Address("1.2...") == false) + assert(NetUtil.isIpv4Address("1.2.3.") == false) + assert(NetUtil.isIpv4Address(".2.3.4") == false) + } - "inetAddressToInt" in { - assert(NetUtil.inetAddressToInt(InetAddress.getByName("0.0.0.0")) == 0) - assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.255.255.255")) == 0xffffffff) - assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.255.255.0")) == 0xffffff00) - assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.0.255.0")) == 0xff00ff00) - assert(NetUtil.inetAddressToInt(InetAddress.getByName("61.197.253.56")) == 0x3dc5fd38) - intercept[IllegalArgumentException] { - NetUtil.inetAddressToInt(InetAddress.getByName("::1")) - } - } + test("NetUtil isPrivate") { + assert(NetUtil.isPrivateAddress(InetAddress.getByName("0.0.0.0")) == false) + assert(NetUtil.isPrivateAddress(InetAddress.getByName("199.59.148.13")) == false) + assert(NetUtil.isPrivateAddress(InetAddress.getByName("10.0.0.0")) == true) + assert(NetUtil.isPrivateAddress(InetAddress.getByName("10.255.255.255")) == true) + assert(NetUtil.isPrivateAddress(InetAddress.getByName("172.16.0.0")) == true) + assert(NetUtil.isPrivateAddress(InetAddress.getByName("172.31.255.255")) == true) + assert(NetUtil.isPrivateAddress(InetAddress.getByName("192.168.0.0")) == true) + assert(NetUtil.isPrivateAddress(InetAddress.getByName("192.168.255.255")) == true) + } - "cidrToIpBlock" in { - assert(NetUtil.cidrToIpBlock("127") == ((0x7f000000, 0xff000000))) - assert(NetUtil.cidrToIpBlock("127.0.0") == ((0x7f000000, 0xffffff00))) - assert(NetUtil.cidrToIpBlock("127.0.0.1") == ((0x7f000001, 0xffffffff))) - assert(NetUtil.cidrToIpBlock("127.0.0.1/1") == ((0x7f000001, 0x80000000))) - assert(NetUtil.cidrToIpBlock("127.0.0.1/4") == ((0x7f000001, 0xf0000000))) - assert(NetUtil.cidrToIpBlock("127.0.0.1/32") == ((0x7f000001, 0xffffffff))) - assert(NetUtil.cidrToIpBlock("127/24") == ((0x7f000000, 0xffffff00))) + test("NetUtil ipToInt") { + assert(NetUtil.ipToInt("0.0.0.0") == 0) + assert(NetUtil.ipToInt("255.255.255.255") == 0xffffffff) + assert(NetUtil.ipToInt("255.255.255.0") == 0xffffff00) + assert(NetUtil.ipToInt("255.0.255.0") == 0xff00ff00) + assert(NetUtil.ipToInt("61.197.253.56") == 0x3dc5fd38) + intercept[IllegalArgumentException] { + NetUtil.ipToInt("256.0.255.0") } + } - "isInetAddressInBlock" in { - val block = NetUtil.cidrToIpBlock("192.168.0.0/16") - - assert(NetUtil.isInetAddressInBlock(InetAddress.getByName("192.168.0.1"), block) == true) - assert(NetUtil.isInetAddressInBlock(InetAddress.getByName("192.168.255.254"), block) == true) - assert(NetUtil.isInetAddressInBlock(InetAddress.getByName("192.169.0.1"), block) == false) + test("NetUtil inetAddressToInt") { + assert(NetUtil.inetAddressToInt(InetAddress.getByName("0.0.0.0")) == 0) + assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.255.255.255")) == 0xffffffff) + assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.255.255.0")) == 0xffffff00) + assert(NetUtil.inetAddressToInt(InetAddress.getByName("255.0.255.0")) == 0xff00ff00) + assert(NetUtil.inetAddressToInt(InetAddress.getByName("61.197.253.56")) == 0x3dc5fd38) + intercept[IllegalArgumentException] { + NetUtil.inetAddressToInt(InetAddress.getByName("::1")) } + } - "isIpInBlocks" in { - val blocks = Seq( - NetUtil.cidrToIpBlock("127"), - NetUtil.cidrToIpBlock("10.1.1.0/24"), - NetUtil.cidrToIpBlock("192.168.0.0/16"), - NetUtil.cidrToIpBlock("200.1.1.1"), - NetUtil.cidrToIpBlock("200.1.1.2/32") - ) - - assert(NetUtil.isIpInBlocks("127.0.0.1", blocks) == true) - assert(NetUtil.isIpInBlocks("128.0.0.1", blocks) == false) - assert(NetUtil.isIpInBlocks("127.255.255.255", blocks) == true) - - assert(NetUtil.isIpInBlocks("10.1.1.1", blocks) == true) - assert(NetUtil.isIpInBlocks("10.1.1.255", blocks) == true) - assert(NetUtil.isIpInBlocks("10.1.0.255", blocks) == false) - assert(NetUtil.isIpInBlocks("10.1.2.0", blocks) == false) + test("NetUtil cidrToIpBlock") { + assert(NetUtil.cidrToIpBlock("127") == ((0x7f000000, 0xff000000))) + assert(NetUtil.cidrToIpBlock("127.0.0") == ((0x7f000000, 0xffffff00))) + assert(NetUtil.cidrToIpBlock("127.0.0.1") == ((0x7f000001, 0xffffffff))) + assert(NetUtil.cidrToIpBlock("127.0.0.1/1") == ((0x7f000001, 0x80000000))) + assert(NetUtil.cidrToIpBlock("127.0.0.1/4") == ((0x7f000001, 0xf0000000))) + assert(NetUtil.cidrToIpBlock("127.0.0.1/32") == ((0x7f000001, 0xffffffff))) + assert(NetUtil.cidrToIpBlock("127/24") == ((0x7f000000, 0xffffff00))) + } - assert(NetUtil.isIpInBlocks("192.168.0.1", blocks) == true) - assert(NetUtil.isIpInBlocks("192.168.255.255", blocks) == true) - assert(NetUtil.isIpInBlocks("192.167.255.255", blocks) == false) - assert(NetUtil.isIpInBlocks("192.169.0.0", blocks) == false) - assert(NetUtil.isIpInBlocks("200.168.0.0", blocks) == false) + test("NetUtil isInetAddressInBlock") { + val block = NetUtil.cidrToIpBlock("192.168.0.0/16") - assert(NetUtil.isIpInBlocks("200.1.1.1", blocks) == true) - assert(NetUtil.isIpInBlocks("200.1.3.1", blocks) == false) - assert(NetUtil.isIpInBlocks("200.1.1.2", blocks) == true) - assert(NetUtil.isIpInBlocks("200.1.3.2", blocks) == false) + assert(NetUtil.isInetAddressInBlock(InetAddress.getByName("192.168.0.1"), block) == true) + assert(NetUtil.isInetAddressInBlock(InetAddress.getByName("192.168.255.254"), block) == true) + assert(NetUtil.isInetAddressInBlock(InetAddress.getByName("192.169.0.1"), block) == false) + } - intercept[IllegalArgumentException] { - NetUtil.isIpInBlocks("", blocks) - } - intercept[IllegalArgumentException] { - NetUtil.isIpInBlocks("no", blocks) - } - intercept[IllegalArgumentException] { - NetUtil.isIpInBlocks("::127.0.0.1", blocks) - } - intercept[IllegalArgumentException] { - NetUtil.isIpInBlocks("-1.0.0.0", blocks) - } - intercept[IllegalArgumentException] { - NetUtil.isIpInBlocks("256.0.0.0", blocks) - } - intercept[IllegalArgumentException] { - NetUtil.isIpInBlocks("0.256.0.0", blocks) - } - intercept[IllegalArgumentException] { - NetUtil.isIpInBlocks("0.0.256.0", blocks) - } - intercept[IllegalArgumentException] { - NetUtil.isIpInBlocks("0.0.0.256", blocks) - } + test("NetUtil isIpInBlocks") { + val blocks = Seq( + NetUtil.cidrToIpBlock("127"), + NetUtil.cidrToIpBlock("10.1.1.0/24"), + NetUtil.cidrToIpBlock("192.168.0.0/16"), + NetUtil.cidrToIpBlock("200.1.1.1"), + NetUtil.cidrToIpBlock("200.1.1.2/32") + ) + + assert(NetUtil.isIpInBlocks("127.0.0.1", blocks) == true) + assert(NetUtil.isIpInBlocks("128.0.0.1", blocks) == false) + assert(NetUtil.isIpInBlocks("127.255.255.255", blocks) == true) + + assert(NetUtil.isIpInBlocks("10.1.1.1", blocks) == true) + assert(NetUtil.isIpInBlocks("10.1.1.255", blocks) == true) + assert(NetUtil.isIpInBlocks("10.1.0.255", blocks) == false) + assert(NetUtil.isIpInBlocks("10.1.2.0", blocks) == false) + + assert(NetUtil.isIpInBlocks("192.168.0.1", blocks) == true) + assert(NetUtil.isIpInBlocks("192.168.255.255", blocks) == true) + assert(NetUtil.isIpInBlocks("192.167.255.255", blocks) == false) + assert(NetUtil.isIpInBlocks("192.169.0.0", blocks) == false) + assert(NetUtil.isIpInBlocks("200.168.0.0", blocks) == false) + + assert(NetUtil.isIpInBlocks("200.1.1.1", blocks) == true) + assert(NetUtil.isIpInBlocks("200.1.3.1", blocks) == false) + assert(NetUtil.isIpInBlocks("200.1.1.2", blocks) == true) + assert(NetUtil.isIpInBlocks("200.1.3.2", blocks) == false) + + intercept[IllegalArgumentException] { + NetUtil.isIpInBlocks("", blocks) + } + intercept[IllegalArgumentException] { + NetUtil.isIpInBlocks("no", blocks) + } + intercept[IllegalArgumentException] { + NetUtil.isIpInBlocks("::127.0.0.1", blocks) + } + intercept[IllegalArgumentException] { + NetUtil.isIpInBlocks("-1.0.0.0", blocks) + } + intercept[IllegalArgumentException] { + NetUtil.isIpInBlocks("256.0.0.0", blocks) + } + intercept[IllegalArgumentException] { + NetUtil.isIpInBlocks("0.256.0.0", blocks) + } + intercept[IllegalArgumentException] { + NetUtil.isIpInBlocks("0.0.256.0", blocks) + } + intercept[IllegalArgumentException] { + NetUtil.isIpInBlocks("0.0.0.256", blocks) } } } diff --git a/util-core/src/test/scala/com/twitter/util/PoolTest.scala b/util-core/src/test/scala/com/twitter/util/PoolTest.scala index 11d15401f6..71cb107617 100644 --- a/util-core/src/test/scala/com/twitter/util/PoolTest.scala +++ b/util-core/src/test/scala/com/twitter/util/PoolTest.scala @@ -1,74 +1,68 @@ package com.twitter.util import com.twitter.conversions.DurationOps._ +import org.scalatest.funsuite.AnyFunSuite import scala.collection.mutable -import org.scalatest.wordspec.AnyWordSpec -class PoolTest extends AnyWordSpec { +class PoolTest extends AnyFunSuite { private[this] def await[T](f: Awaitable[T]): T = Await.result(f, 2.seconds) - "SimplePool" should { - "with a simple queue of items" should { - "it reseves items in FIFO order" in { - val queue = new mutable.Queue[Int] ++ List(1, 2, 3) - val pool = new SimplePool(queue) - assert(await(pool.reserve()) == 1) - assert(await(pool.reserve()) == 2) - pool.release(2) - assert(await(pool.reserve()) == 3) - assert(await(pool.reserve()) == 2) - pool.release(1) - pool.release(2) - pool.release(3) - } + class PoolSpecHelper { + var count = 0 + val pool = new FactoryPool[Int](4) { + def makeItem() = { count += 1; Future(count) } + def isHealthy(i: Int) = i % 2 == 0 } + } - "with an object factory and a health check" should { - class PoolSpecHelper { - var count = 0 - val pool = new FactoryPool[Int](4) { - def makeItem() = { count += 1; Future(count) } - def isHealthy(i: Int) = i % 2 == 0 - } - } + test("SimplePool with a simple queue of items should reserve items in FIFO order") { + val queue = new mutable.Queue[Int] ++ List(1, 2, 3) + val pool = new SimplePool(queue) + assert(await(pool.reserve()) == 1) + assert(await(pool.reserve()) == 2) + pool.release(2) + assert(await(pool.reserve()) == 3) + assert(await(pool.reserve()) == 2) + pool.release(1) + pool.release(2) + pool.release(3) + } - "reserve & release" in { - val h = new PoolSpecHelper - import h._ + test("SimplePool with an object factory and a health check should reserve & release") { + val h = new PoolSpecHelper + import h._ - assert(await(pool.reserve()) == 2) - assert(await(pool.reserve()) == 4) - assert(await(pool.reserve()) == 6) - assert(await(pool.reserve()) == 8) - val promise = pool.reserve() - intercept[TimeoutException] { - Await.result(promise, 1.millisecond) - } - pool.release(8) - pool.release(6) - assert(await(promise) == 8) - assert(await(pool.reserve()) == 6) - intercept[TimeoutException] { - Await.result(pool.reserve(), 1.millisecond) - } - } + assert(await(pool.reserve()) == 2) + assert(await(pool.reserve()) == 4) + assert(await(pool.reserve()) == 6) + assert(await(pool.reserve()) == 8) + val promise = pool.reserve() + intercept[TimeoutException] { + Await.result(promise, 1.millisecond) + } + pool.release(8) + pool.release(6) + assert(await(promise) == 8) + assert(await(pool.reserve()) == 6) + intercept[TimeoutException] { + Await.result(pool.reserve(), 1.millisecond) + } + } - "reserve & dispose" in { - val h = new PoolSpecHelper - import h._ + test("SimplePool with an object factory and a health check should reserve & dispose") { + val h = new PoolSpecHelper + import h._ - assert(await(pool.reserve()) == 2) - assert(await(pool.reserve()) == 4) - assert(await(pool.reserve()) == 6) - assert(await(pool.reserve()) == 8) - intercept[TimeoutException] { - Await.result(pool.reserve(), 1.millisecond) - } - pool.dispose(2) - assert(Await.result(pool.reserve(), 1.millisecond) == 10) - } + assert(await(pool.reserve()) == 2) + assert(await(pool.reserve()) == 4) + assert(await(pool.reserve()) == 6) + assert(await(pool.reserve()) == 8) + intercept[TimeoutException] { + Await.result(pool.reserve(), 1.millisecond) } + pool.dispose(2) + assert(Await.result(pool.reserve(), 1.millisecond) == 10) } } diff --git a/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala b/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala index dcb037c786..cbb541d68e 100644 --- a/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala +++ b/util-core/src/test/scala/com/twitter/util/StateMachineTest.scala @@ -1,40 +1,38 @@ package com.twitter.util -import org.scalatest.wordspec.AnyWordSpec - -class StateMachineTest extends AnyWordSpec { - "StateMachine" should { - class StateMachineHelper { - class Machine extends StateMachine { - case class State1() extends State - case class State2() extends State - state = State1() - - def command1(): Unit = { - transition("command1") { - case State1() => state = State2() - } +import org.scalatest.funsuite.AnyFunSuite + +class StateMachineTest extends AnyFunSuite { + class StateMachineHelper { + class Machine extends StateMachine { + case class State1() extends State + case class State2() extends State + state = State1() + + def command1(): Unit = { + transition("command1") { + case State1() => state = State2() } } - - val stateMachine = new Machine() } - "allows transitions that are permitted" in { - val h = new StateMachineHelper - import h._ + val stateMachine = new Machine() + } - stateMachine.command1() - } + test("StateMachine allows transitions that are permitted") { + val h = new StateMachineHelper + import h._ - "throws exceptions when a transition is not permitted" in { - val h = new StateMachineHelper - import h._ + stateMachine.command1() + } + + test("StateMachine throws exceptions when a transition is not permitted") { + val h = new StateMachineHelper + import h._ + stateMachine.command1() + intercept[StateMachine.InvalidStateTransition] { stateMachine.command1() - intercept[StateMachine.InvalidStateTransition] { - stateMachine.command1() - } } } } diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index 6fb25bf350..bc2e02b993 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -1,754 +1,730 @@ package com.twitter.util +import com.twitter.conversions.DurationOps._ import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.ObjectInputStream import java.io.ObjectOutputStream +import java.time.Instant +import java.time.OffsetDateTime +import java.time.ZoneOffset +import java.time.ZonedDateTime import java.util.Locale import java.util.TimeZone import java.util.concurrent.TimeUnit import org.scalatest.concurrent.Eventually import org.scalatest.concurrent.IntegrationPatience +import org.scalatest.funsuite.AnyFunSuite import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -import com.twitter.conversions.DurationOps._ -import java.time.Instant -import java.time.OffsetDateTime -import java.time.ZoneOffset -import java.time.ZonedDateTime -import org.scalatest.wordspec.AnyWordSpec -trait TimeLikeSpec[T <: TimeLike[T]] extends AnyWordSpec with ScalaCheckDrivenPropertyChecks { +trait TimeLikeSpec[T <: TimeLike[T]] extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { val ops: TimeLikeOps[T] import ops._ - "Top, Bottom, Undefined, Nanoseconds(_), Finite(_)" should { - val easyVs = Seq(Zero, Top, Bottom, Undefined, fromNanoseconds(1), fromNanoseconds(-1)) - val vs = easyVs ++ Seq(fromNanoseconds(Long.MaxValue - 1), fromNanoseconds(Long.MinValue + 1)) - - "behave like boxed doubles" in { - assert((Top compare Undefined) < 0) - assert((Bottom compare Top) < 0) - assert((Undefined compare Undefined) == 0) - assert((Top compare Top) == 0) - assert((Bottom compare Bottom) == 0) - - assert(Top + Duration.Top == Top) - assert(Bottom - Duration.Bottom == Undefined) - assert(Top - Duration.Top == Undefined) - assert(Bottom + Duration.Bottom == Bottom) - } + val easyVs = Seq(Zero, Top, Bottom, Undefined, fromNanoseconds(1), fromNanoseconds(-1)) + val vs = easyVs ++ Seq(fromNanoseconds(Long.MaxValue - 1), fromNanoseconds(Long.MinValue + 1)) - "complementary diff" in { - // Note that this doesn't always hold because of two's - // complement arithmetic. - for (a <- easyVs; b <- easyVs) - assert((a diff b) == -(b diff a)) + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_) should behave like boxed doubles") { + assert((Top compare Undefined) < 0) + assert((Bottom compare Top) < 0) + assert((Undefined compare Undefined) == 0) + assert((Top compare Top) == 0) + assert((Bottom compare Bottom) == 0) - } + assert(Top + Duration.Top == Top) + assert(Bottom - Duration.Bottom == Undefined) + assert(Top - Duration.Top == Undefined) + assert(Bottom + Duration.Bottom == Bottom) + } - "complementary compare" in { - for (a <- vs; b <- vs) { - val x = a compare b - val y = b compare a - assert(((x == 0 && y == 0) || (x < 0 != y < 0)) == true) - } - } + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_) should complementary diff") { + // Note that this doesn't always hold because of two's + // complement arithmetic. + for (a <- easyVs; b <- easyVs) + assert((a diff b) == -(b diff a)) - "commutative max" in { - for (a <- vs; b <- vs) - assert((a max b) == (b max a)) - } + } - "commutative min" in { - for (a <- vs; b <- vs) - assert((a min b) == (b min a)) + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_) should complementary compare") { + for (a <- vs; b <- vs) { + val x = a compare b + val y = b compare a + assert(((x == 0 && y == 0) || (x < 0 != y < 0)) == true) } + } - "handle underflows" in { - assert(fromNanoseconds(Long.MinValue) - 1.nanosecond == Bottom) - assert(fromMicroseconds(Long.MinValue) - 1.nanosecond == Bottom) - } + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_) should commutative max") { + for (a <- vs; b <- vs) + assert((a max b) == (b max a)) + } - "handle overflows" in { - assert(fromNanoseconds(Long.MaxValue) + 1.nanosecond == Top) - assert(fromMicroseconds(Long.MaxValue) + 1.nanosecond == Top) - } + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_) should commutative min") { + for (a <- vs; b <- vs) + assert((a min b) == (b min a)) + } - "Nanoseconds(_) extracts only finite values, in nanoseconds" in { - for (t <- Seq(Top, Bottom, Undefined)) - assert(t match { - case Nanoseconds(_) => false - case _ => true - }) - - for (ns <- Seq(Long.MinValue, -1, 0, 1, Long.MaxValue); t = fromNanoseconds(ns)) - assert(t match { - case Nanoseconds(`ns`) => true - case _ => false - }) - } + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_) should handle underflows") { + assert(fromNanoseconds(Long.MinValue) - 1.nanosecond == Bottom) + assert(fromMicroseconds(Long.MinValue) - 1.nanosecond == Bottom) + } - "Finite(_) extracts only finite values" in { - for (t <- Seq(Top, Bottom, Undefined)) - assert(t match { - case Finite(_) => false - case _ => true - }) - - for (ns <- Seq(Long.MinValue, -1, 0, 1, Long.MaxValue); t = fromNanoseconds(ns)) - assert(t match { - case Finite(`t`) => true - case _ => false - }) - } + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_) should handle overflows") { + assert(fromNanoseconds(Long.MaxValue) + 1.nanosecond == Top) + assert(fromMicroseconds(Long.MaxValue) + 1.nanosecond == Top) + } - "roundtrip through serialization" in { - for (v <- vs) { - val bytes = new ByteArrayOutputStream - val out = new ObjectOutputStream(bytes) - out.writeObject(v) - val in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray)) - assert(in.readObject() == v) - } - } + test( + "Top, Bottom, Undefined, Nanoseconds(_), Finite(_): Nanoseconds(_) extracts only finite values, in nanoseconds") { + for (t <- Seq(Top, Bottom, Undefined)) + assert(t match { + case Nanoseconds(_) => false + case _ => true + }) + + for (ns <- Seq(Long.MinValue, -1, 0, 1, Long.MaxValue); t = fromNanoseconds(ns)) + assert(t match { + case Nanoseconds(`ns`) => true + case _ => false + }) + } - "has correct isZero behaviour" in { - for (t <- Seq(Top, Bottom, Undefined, fromNanoseconds(1L))) { - assert(t.isZero == false) - } + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_): Finite(_) extracts only finite values") { + for (t <- Seq(Top, Bottom, Undefined)) + assert(t match { + case Finite(_) => false + case _ => true + }) + + for (ns <- Seq(Long.MinValue, -1, 0, 1, Long.MaxValue); t = fromNanoseconds(ns)) + assert(t match { + case Finite(`t`) => true + case _ => false + }) + } - for (z <- Seq( - Zero, - fromNanoseconds(0), - fromMicroseconds(0), - fromFractionalSeconds(0), - fromMilliseconds(0), - fromSeconds(0), - fromMinutes(0) - )) { - assert(z.isZero == true) - } + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_) should roundtrip through serialization") { + for (v <- vs) { + val bytes = new ByteArrayOutputStream + val out = new ObjectOutputStream(bytes) + out.writeObject(v) + val in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray)) + assert(in.readObject() == v) } } - "Top" should { - "be impermeable to finite arithmetic" in { - assert(Top - 0.seconds == Top) - assert(Top - 100.seconds == Top) - assert(Top - Duration.fromNanoseconds(Long.MaxValue) == Top) + test("Top, Bottom, Undefined, Nanoseconds(_), Finite(_) has correct isZero behaviour") { + for (t <- Seq(Top, Bottom, Undefined, fromNanoseconds(1L))) { + assert(t.isZero == false) } - "become undefined when subtracted from itself, or added to bottom" in { - assert(Top - Duration.Top == Undefined) - assert(Top + Duration.Bottom == Undefined) + for (z <- Seq( + Zero, + fromNanoseconds(0), + fromMicroseconds(0), + fromFractionalSeconds(0), + fromMilliseconds(0), + fromSeconds(0), + fromMinutes(0) + )) { + assert(z.isZero == true) } + } - "not be equal to the maximum value" in { - assert(fromNanoseconds(Long.MaxValue) != Top) - } + test("Top should be impermeable to finite arithmetic") { + assert(Top - 0.seconds == Top) + assert(Top - 100.seconds == Top) + assert(Top - Duration.fromNanoseconds(Long.MaxValue) == Top) + } - "always be max" in { - assert((Top max fromSeconds(1)) == Top) - assert((Top max fromFractionalSeconds(1.0)) == Top) - assert((Top max fromNanoseconds(Long.MaxValue)) == Top) - assert((Top max Bottom) == Top) - } + test("Top should become undefined when subtracted from itself, or added to bottom") { + assert(Top - Duration.Top == Undefined) + assert(Top + Duration.Bottom == Undefined) + } - "greater than everything else" in { - assert(fromSeconds(0) < Top) - assert(fromFractionalSeconds(Double.MaxValue) < Top) - assert(fromNanoseconds(Long.MaxValue) < Top) - } + test("Top should not be equal to the maximum value") { + assert(fromNanoseconds(Long.MaxValue) != Top) + } - "equal to itself" in { - assert(Top == Top) - } + test("Top should always be max") { + assert((Top max fromSeconds(1)) == Top) + assert((Top max fromFractionalSeconds(1.0)) == Top) + assert((Top max fromNanoseconds(Long.MaxValue)) == Top) + assert((Top max Bottom) == Top) + } - "more or less equals only to itself" in { - assert(Top.moreOrLessEquals(Top, Duration.Top) == true) - assert(Top.moreOrLessEquals(Top, Duration.Zero) == true) - assert(Top.moreOrLessEquals(Bottom, Duration.Top) == true) - assert(Top.moreOrLessEquals(Bottom, Duration.Zero) == false) - assert(Top.moreOrLessEquals(fromSeconds(0), Duration.Top) == true) - assert(Top.moreOrLessEquals(fromSeconds(0), Duration.Bottom) == false) - } + test("Top should greater than everything else") { + assert(fromSeconds(0) < Top) + assert(fromFractionalSeconds(Double.MaxValue) < Top) + assert(fromNanoseconds(Long.MaxValue) < Top) + } - "Undefined diff to Top" in { - assert((Top diff Top) == Duration.Undefined) - } + test("Top should equal to itself") { + assert(Top == Top) } - "Bottom" should { - "be impermeable to finite arithmetic" in { - assert(Bottom + 0.seconds == Bottom) - assert(Bottom + 100.seconds == Bottom) - assert(Bottom + Duration.fromNanoseconds(Long.MaxValue) == Bottom) - } + test("Top should more or less equals only to itself") { + assert(Top.moreOrLessEquals(Top, Duration.Top) == true) + assert(Top.moreOrLessEquals(Top, Duration.Zero) == true) + assert(Top.moreOrLessEquals(Bottom, Duration.Top) == true) + assert(Top.moreOrLessEquals(Bottom, Duration.Zero) == false) + assert(Top.moreOrLessEquals(fromSeconds(0), Duration.Top) == true) + assert(Top.moreOrLessEquals(fromSeconds(0), Duration.Bottom) == false) + } - "become undefined when added with Top or subtracted by bottom" in { - assert(Bottom + Duration.Top == Undefined) - assert(Bottom - Duration.Bottom == Undefined) - } + test("Top should Undefined diff to Top") { + assert((Top diff Top) == Duration.Undefined) + } - "always be min" in { - assert((Bottom min Top) == Bottom) - assert((Bottom min fromNanoseconds(0)) == Bottom) - } + test("Bottom should be impermeable to finite arithmetic") { + assert(Bottom + 0.seconds == Bottom) + assert(Bottom + 100.seconds == Bottom) + assert(Bottom + Duration.fromNanoseconds(Long.MaxValue) == Bottom) + } - "less than everything else" in { - assert(Bottom < fromSeconds(0)) - assert(Bottom < fromNanoseconds(Long.MaxValue)) - assert(Bottom < fromNanoseconds(Long.MinValue)) - } + test("Bottom should become undefined when added with Top or subtracted by bottom") { + assert(Bottom + Duration.Top == Undefined) + assert(Bottom - Duration.Bottom == Undefined) + } - "less than Top" in { - assert(Bottom < Top) - } + test("Bottom should always be min") { + assert((Bottom min Top) == Bottom) + assert((Bottom min fromNanoseconds(0)) == Bottom) + } - "equal to itself" in { - assert(Bottom == Bottom) - } + test("Bottom should less than everything else") { + assert(Bottom < fromSeconds(0)) + assert(Bottom < fromNanoseconds(Long.MaxValue)) + assert(Bottom < fromNanoseconds(Long.MinValue)) + } - "more or less equals only to itself" in { - assert(Bottom.moreOrLessEquals(Bottom, Duration.Top) == true) - assert(Bottom.moreOrLessEquals(Bottom, Duration.Zero) == true) - assert(Bottom.moreOrLessEquals(Top, Duration.Bottom) == false) - assert(Bottom.moreOrLessEquals(Top, Duration.Zero) == false) - assert(Bottom.moreOrLessEquals(fromSeconds(0), Duration.Top) == true) - assert(Bottom.moreOrLessEquals(fromSeconds(0), Duration.Bottom) == false) - } + test("Bottom should less than Top") { + assert(Bottom < Top) + } - "Undefined diff to Bottom" in { - assert((Bottom diff Bottom) == Duration.Undefined) - } + test("Bottom should equal to itself") { + assert(Bottom == Bottom) } - "Undefined" should { - "be impermeable to any arithmetic" in { - assert(Undefined + 0.seconds == Undefined) - assert(Undefined + 100.seconds == Undefined) - assert(Undefined + Duration.fromNanoseconds(Long.MaxValue) == Undefined) - } + test("Bottom should more or less equals only to itself") { + assert(Bottom.moreOrLessEquals(Bottom, Duration.Top) == true) + assert(Bottom.moreOrLessEquals(Bottom, Duration.Zero) == true) + assert(Bottom.moreOrLessEquals(Top, Duration.Bottom) == false) + assert(Bottom.moreOrLessEquals(Top, Duration.Zero) == false) + assert(Bottom.moreOrLessEquals(fromSeconds(0), Duration.Top) == true) + assert(Bottom.moreOrLessEquals(fromSeconds(0), Duration.Bottom) == false) + } - "become undefined when added with Top or subtracted by bottom" in { - assert(Undefined + Duration.Top == Undefined) - assert(Undefined - Duration.Undefined == Undefined) - } + test("Bottom should Undefined diff to Bottom") { + assert((Bottom diff Bottom) == Duration.Undefined) + } - "always be max" in { - assert((Undefined max Top) == Undefined) - assert((Undefined max fromNanoseconds(0)) == Undefined) - } + test("Undefined should be impermeable to any arithmetic") { + assert(Undefined + 0.seconds == Undefined) + assert(Undefined + 100.seconds == Undefined) + assert(Undefined + Duration.fromNanoseconds(Long.MaxValue) == Undefined) + } - "greater than everything else" in { - assert(fromSeconds(0) < Undefined) - assert(Top < Undefined) - assert(fromNanoseconds(Long.MaxValue) < Undefined) - } + test("Undefined should become undefined when added with Top or subtracted by bottom") { + assert(Undefined + Duration.Top == Undefined) + assert(Undefined - Duration.Undefined == Undefined) + } - "equal to itself" in { - assert(Undefined == Undefined) - } + test("Undefined should always be max") { + assert((Undefined max Top) == Undefined) + assert((Undefined max fromNanoseconds(0)) == Undefined) + } - "not more or less equal to anything" in { - assert(Undefined.moreOrLessEquals(Undefined, Duration.Top) == false) - assert(Undefined.moreOrLessEquals(Undefined, Duration.Zero) == false) - assert(Undefined.moreOrLessEquals(Top, Duration.Undefined) == true) - assert(Undefined.moreOrLessEquals(Top, Duration.Zero) == false) - assert(Undefined.moreOrLessEquals(fromSeconds(0), Duration.Top) == false) - assert(Undefined.moreOrLessEquals(fromSeconds(0), Duration.Undefined) == true) - } + test("Undefined should greater than everything else") { + assert(fromSeconds(0) < Undefined) + assert(Top < Undefined) + assert(fromNanoseconds(Long.MaxValue) < Undefined) + } - "Undefined on diff" in { - assert((Undefined diff Top) == Duration.Undefined) - assert((Undefined diff Bottom) == Duration.Undefined) - assert((Undefined diff fromNanoseconds(123)) == Duration.Undefined) - } + test("Undefined should equal to itself") { + assert(Undefined == Undefined) } - "values" should { - "reflect their underlying value" in { - val nss = Seq( - 2592000000000000000L, // 30000.days - 1040403005001003L, // 12.days+1.hour+3.seconds+5.milliseconds+1.microsecond+3.nanoseconds - 123000000000L, // 123.seconds - 1L - ) + test("Undefined should not more or less equal to anything") { + assert(Undefined.moreOrLessEquals(Undefined, Duration.Top) == false) + assert(Undefined.moreOrLessEquals(Undefined, Duration.Zero) == false) + assert(Undefined.moreOrLessEquals(Top, Duration.Undefined) == true) + assert(Undefined.moreOrLessEquals(Top, Duration.Zero) == false) + assert(Undefined.moreOrLessEquals(fromSeconds(0), Duration.Top) == false) + assert(Undefined.moreOrLessEquals(fromSeconds(0), Duration.Undefined) == true) + } - for (ns <- nss) { - val t = fromNanoseconds(ns) - assert(t.inNanoseconds == ns) - assert(t.inMicroseconds == ns / 1000L) - assert(t.inMilliseconds == ns / 1000000L) - assert(t.inLongSeconds == ns / 1000000000L) - assert(t.inMinutes == ns / 60000000000L) - assert(t.inHours == ns / 3600000000000L) - assert(t.inDays == ns / 86400000000000L) - } - } + test("Undefined should Undefined on diff") { + assert((Undefined diff Top) == Duration.Undefined) + assert((Undefined diff Bottom) == Duration.Undefined) + assert((Undefined diff fromNanoseconds(123)) == Duration.Undefined) } - "inSeconds" should { - "equal inLongSeconds when in 32-bit range" in { - val nss = Seq( - 315370851000000000L, // 3650.days+3.hours+51.seconds - 1040403005001003L, // 12.days+1.hour+3.seconds+5.milliseconds+1.microsecond+3.nanoseconds - 1L - ) - for (ns <- nss) { - val t = fromNanoseconds(ns) - assert(t.inLongSeconds == t.inSeconds) - } - } - "clamp value to Int.MinValue or MaxValue when out of range" in { - val longNs = 2160000000000000000L // 25000.days - assert(fromNanoseconds(longNs).inSeconds == Int.MaxValue) - assert(fromNanoseconds(-longNs).inSeconds == Int.MinValue) + test("values should reflect their underlying value") { + val nss = Seq( + 2592000000000000000L, // 30000.days + 1040403005001003L, // 12.days+1.hour+3.seconds+5.milliseconds+1.microsecond+3.nanoseconds + 123000000000L, // 123.seconds + 1L + ) + + for (ns <- nss) { + val t = fromNanoseconds(ns) + assert(t.inNanoseconds == ns) + assert(t.inMicroseconds == ns / 1000L) + assert(t.inMilliseconds == ns / 1000000L) + assert(t.inLongSeconds == ns / 1000000000L) + assert(t.inMinutes == ns / 60000000000L) + assert(t.inHours == ns / 3600000000000L) + assert(t.inDays == ns / 86400000000000L) } } - "rounding" should { - - "maintain top and bottom" in { - assert(Top.floor(1.hour) == Top) - assert(Bottom.floor(1.hour) == Bottom) + test("inSeconds should equal inLongSeconds when in 32-bit range") { + val nss = Seq( + 315370851000000000L, // 3650.days+3.hours+51.seconds + 1040403005001003L, // 12.days+1.hour+3.seconds+5.milliseconds+1.microsecond+3.nanoseconds + 1L + ) + for (ns <- nss) { + val t = fromNanoseconds(ns) + assert(t.inLongSeconds == t.inSeconds) } + } + test("inSeconds should clamp value to Int.MinValue or MaxValue when out of range") { + val longNs = 2160000000000000000L // 25000.days + assert(fromNanoseconds(longNs).inSeconds == Int.MaxValue) + assert(fromNanoseconds(-longNs).inSeconds == Int.MinValue) + } - "divide by zero" in { - assert(Zero.floor(Duration.Zero) == Undefined) - assert(fromSeconds(1).floor(Duration.Zero) == Top) - assert(fromSeconds(-1).floor(Duration.Zero) == Bottom) - } + test("rounding should maintain top and bottom") { + assert(Top.floor(1.hour) == Top) + assert(Bottom.floor(1.hour) == Bottom) + } - "deal with undefineds" in { - assert(Undefined.floor(0.seconds) == Undefined) - assert(Undefined.floor(Duration.Top) == Undefined) - assert(Undefined.floor(Duration.Bottom) == Undefined) - assert(Undefined.floor(Duration.Undefined) == Undefined) - } + test("rounding should divide by zero") { + assert(Zero.floor(Duration.Zero) == Undefined) + assert(fromSeconds(1).floor(Duration.Zero) == Top) + assert(fromSeconds(-1).floor(Duration.Zero) == Bottom) + } - "round to itself" in { - for (s <- Seq(Long.MinValue, -1, 1, Long.MaxValue); t = s.nanoseconds) - assert(t.floor(t.inNanoseconds.nanoseconds) == t) - } + test("rounding should deal with undefineds") { + assert(Undefined.floor(0.seconds) == Undefined) + assert(Undefined.floor(Duration.Top) == Undefined) + assert(Undefined.floor(Duration.Bottom) == Undefined) + assert(Undefined.floor(Duration.Undefined) == Undefined) } - "floor" should { - "round down" in { - assert(60.seconds.floor(1.minute) == 60.seconds) - assert(100.seconds.floor(1.minute) == 60.seconds) - assert(119.seconds.floor(1.minute) == 60.seconds) - assert(120.seconds.floor(1.minute) == 120.seconds) - } + test("rounding should round to itself") { + for (s <- Seq(Long.MinValue, -1, 1, Long.MaxValue); t = s.nanoseconds) + assert(t.floor(t.inNanoseconds.nanoseconds) == t) } - "ceiling" should { - "round up" in { - assert(60.seconds.ceil(1.minute) == 60.seconds) - assert(100.seconds.ceil(1.minute) == 120.seconds) - assert(119.seconds.ceil(1.minute) == 120.seconds) - assert(120.seconds.ceil(1.minute) == 120.seconds) - } + test("floor should round down") { + assert(60.seconds.floor(1.minute) == 60.seconds) + assert(100.seconds.floor(1.minute) == 60.seconds) + assert(119.seconds.floor(1.minute) == 60.seconds) + assert(120.seconds.floor(1.minute) == 120.seconds) } - "from*" should { - "never over/under flow nanos" in { - for (v <- Seq(Long.MinValue, Long.MaxValue)) { - fromNanoseconds(v) match { - case Nanoseconds(ns) => assert(ns == v) - } + test("ceiling should round up") { + assert(60.seconds.ceil(1.minute) == 60.seconds) + assert(100.seconds.ceil(1.minute) == 120.seconds) + assert(119.seconds.ceil(1.minute) == 120.seconds) + assert(120.seconds.ceil(1.minute) == 120.seconds) + } + + test("from* should never over/under flow nanos") { + for (v <- Seq(Long.MinValue, Long.MaxValue)) { + fromNanoseconds(v) match { + case Nanoseconds(ns) => assert(ns == v) } } + } - "overflow millis" in { - val millis = TimeUnit.NANOSECONDS.toMillis(Long.MaxValue) - fromMilliseconds(millis) match { - case Nanoseconds(ns) => assert(ns == millis * 1e6) - } - assert(fromMilliseconds(millis + 1) == Top) + test("from* should overflow millis") { + val millis = TimeUnit.NANOSECONDS.toMillis(Long.MaxValue) + fromMilliseconds(millis) match { + case Nanoseconds(ns) => assert(ns == millis * 1e6) } + assert(fromMilliseconds(millis + 1) == Top) + } - "underflow millis" in { - val millis = TimeUnit.NANOSECONDS.toMillis(Long.MinValue) - fromMilliseconds(millis) match { - case Nanoseconds(ns) => assert(ns == millis * 1e6) - } - assert(fromMilliseconds(millis - 1) == Bottom) + test("from* should underflow millis") { + val millis = TimeUnit.NANOSECONDS.toMillis(Long.MinValue) + fromMilliseconds(millis) match { + case Nanoseconds(ns) => assert(ns == millis * 1e6) } + assert(fromMilliseconds(millis - 1) == Bottom) } } -class TimeFormatTest extends AnyWordSpec { - "TimeFormat" should { - "format correctly with non US locale" in { - val locale = Locale.GERMAN - val format = "EEEE" - val timeFormat = new TimeFormat(format, Some(locale)) - val day = "Donnerstag" - assert(timeFormat.parse(day).format(format, locale) == day) - } +class TimeFormatTest extends AnyFunSuite { + test("TimeFormat should format correctly with non US locale") { + val locale = Locale.GERMAN + val format = "EEEE" + val timeFormat = new TimeFormat(format, Some(locale)) + val day = "Donnerstag" + assert(timeFormat.parse(day).format(format, locale) == day) + } - "set UTC timezone as default" in { - val format = "HH:mm" - val timeFormat = new TimeFormat(format) - assert(timeFormat.parse("16:04").format(format) == "16:04") - } + test("TimeFormat should set UTC timezone as default") { + val format = "HH:mm" + val timeFormat = new TimeFormat(format) + assert(timeFormat.parse("16:04").format(format) == "16:04") + } - "set non-UTC timezone correctly" in { - val format = "HH:mm" - val timeFormat = new TimeFormat(format, TimeZone.getTimeZone("EST")) - assert(timeFormat.parse("16:04").format(format) == "21:04") - } + test("TimeFormat should set non-UTC timezone correctly") { + val format = "HH:mm" + val timeFormat = new TimeFormat(format, TimeZone.getTimeZone("EST")) + assert(timeFormat.parse("16:04").format(format) == "21:04") } } trait TimeOps { val ops: Time.type = Time } class TimeTest - extends AnyWordSpec + extends AnyFunSuite with TimeOps with TimeLikeSpec[Time] with Eventually with IntegrationPatience { - "Time" should { - "work in collections" in { - val t0 = Time.fromSeconds(100) - val t1 = Time.fromSeconds(100) - assert(t0 == t1) - assert(t0.hashCode == t1.hashCode) - val pairs = List((t0, "foo"), (t1, "bar")) - assert(pairs.groupBy { case (time: Time, value: String) => time } == Map(t0 -> pairs)) - } + test("Time should work in collections") { + val t0 = Time.fromSeconds(100) + val t1 = Time.fromSeconds(100) + assert(t0 == t1) + assert(t0.hashCode == t1.hashCode) + val pairs = List((t0, "foo"), (t1, "bar")) + assert(pairs.groupBy { case (time: Time, value: String) => time } == Map(t0 -> pairs)) + } - "now should be now" in { - assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) - } + test("Time now should be now") { + assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + } - "withTimeAt" in { - val t0 = new Time(123456789L) - Time.withTimeAt(t0) { _ => - assert(Time.now == t0) - Thread.sleep(50) - assert(Time.now == t0) - } - assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + test("Time withTimeAt") { + val t0 = new Time(123456789L) + Time.withTimeAt(t0) { _ => + assert(Time.now == t0) + Thread.sleep(50) + assert(Time.now == t0) } + assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + } - "withTimeAt nested" in { - val t0 = new Time(123456789L) - val t1 = t0 + 10.minutes - Time.withTimeAt(t0) { _ => - assert(Time.now == t0) - Time.withTimeAt(t1) { _ => assert(Time.now == t1) } - assert(Time.now == t0) - } - assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + test("Time withTimeAt nested") { + val t0 = new Time(123456789L) + val t1 = t0 + 10.minutes + Time.withTimeAt(t0) { _ => + assert(Time.now == t0) + Time.withTimeAt(t1) { _ => assert(Time.now == t1) } + assert(Time.now == t0) } + assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + } - "withTimeAt threaded" in { - val t0 = new Time(314159L) - val t1 = new Time(314160L) - Time.withTimeAt(t0) { tc => - assert(Time.now == t0) - Thread.sleep(50) - assert(Time.now == t0) - tc.advance(Duration.fromNanoseconds(1)) - assert(Time.now == t1) - tc.set(t0) - assert(Time.now == t0) - @volatile var threadTime: Option[Time] = None - val thread = new Thread { - override def run(): Unit = { - threadTime = Some(Time.now) - } + test("Time withTimeAt threaded") { + val t0 = new Time(314159L) + val t1 = new Time(314160L) + Time.withTimeAt(t0) { tc => + assert(Time.now == t0) + Thread.sleep(50) + assert(Time.now == t0) + tc.advance(Duration.fromNanoseconds(1)) + assert(Time.now == t1) + tc.set(t0) + assert(Time.now == t0) + @volatile var threadTime: Option[Time] = None + val thread = new Thread { + override def run(): Unit = { + threadTime = Some(Time.now) } - thread.start() - thread.join() - assert(threadTime.get != t0) } - assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + thread.start() + thread.join() + assert(threadTime.get != t0) } + assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + } - "withTimeFunction" in { - val t0 = Time.now - var t = t0 - Time.withTimeFunction(t) { _ => - assert(Time.now == t0) - Thread.sleep(50) - assert(Time.now == t0) - val delta = 100.milliseconds - t += delta - assert(Time.now == t0 + delta) - } + test("Time withTimeFunction") { + val t0 = Time.now + var t = t0 + Time.withTimeFunction(t) { _ => + assert(Time.now == t0) + Thread.sleep(50) + assert(Time.now == t0) + val delta = 100.milliseconds + t += delta + assert(Time.now == t0 + delta) } + } - "withCurrentTimeFrozen" in { - val t0 = new Time(123456789L) - Time.withCurrentTimeFrozen { _ => - val t0 = Time.now - Thread.sleep(50) - assert(Time.now == t0) - } - assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + test("Time withCurrentTimeFrozen") { + val t0 = new Time(123456789L) + Time.withCurrentTimeFrozen { _ => + val t0 = Time.now + Thread.sleep(50) + assert(Time.now == t0) } + assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + } - "advance" in { - val t0 = new Time(123456789L) - val delta = 5.seconds - Time.withTimeAt(t0) { tc => - assert(Time.now == t0) - tc.advance(delta) - assert(Time.now == (t0 + delta)) - } - assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + test("Time advance") { + val t0 = new Time(123456789L) + val delta = 5.seconds + Time.withTimeAt(t0) { tc => + assert(Time.now == t0) + tc.advance(delta) + assert(Time.now == (t0 + delta)) } + assert((Time.now.inMillis - System.currentTimeMillis).abs < 20L) + } - "sleep" in { - Time.withCurrentTimeFrozen { ctl => - val ctx = Local.save() - val r = new Runnable { - def run(): Unit = { - Local.restore(ctx) - Time.sleep(5.seconds) - } + test("Time sleep") { + Time.withCurrentTimeFrozen { ctl => + val ctx = Local.save() + val r = new Runnable { + def run(): Unit = { + Local.restore(ctx) + Time.sleep(5.seconds) } - @volatile var x = 0 - val t = new Thread(r) - t.start() - assert(t.isAlive == true) - eventually { - assert(t.getState == Thread.State.TIMED_WAITING) - } - ctl.advance(5.seconds) - t.join() - assert(t.isAlive == false) } + @volatile var x = 0 + val t = new Thread(r) + t.start() + assert(t.isAlive == true) + eventually { + assert(t.getState == Thread.State.TIMED_WAITING) + } + ctl.advance(5.seconds) + t.join() + assert(t.isAlive == false) } + } - "compare" in { - assert(10.seconds.afterEpoch < 11.seconds.afterEpoch) - assert(10.seconds.afterEpoch == 10.seconds.afterEpoch) - assert(11.seconds.afterEpoch > 10.seconds.afterEpoch) - assert(Time.fromMilliseconds(Long.MaxValue) > Time.now) - } + test("Time compare") { + assert(10.seconds.afterEpoch < 11.seconds.afterEpoch) + assert(10.seconds.afterEpoch == 10.seconds.afterEpoch) + assert(11.seconds.afterEpoch > 10.seconds.afterEpoch) + assert(Time.fromMilliseconds(Long.MaxValue) > Time.now) + } - "equals" in { - assert(Time.Top == Time.Top) - assert(Time.Top != Time.Bottom) - assert(Time.Top != Time.Undefined) + test("Time equals") { + assert(Time.Top == Time.Top) + assert(Time.Top != Time.Bottom) + assert(Time.Top != Time.Undefined) - assert(Time.Bottom != Time.Top) - assert(Time.Bottom == Time.Bottom) - assert(Time.Bottom != Time.Undefined) + assert(Time.Bottom != Time.Top) + assert(Time.Bottom == Time.Bottom) + assert(Time.Bottom != Time.Undefined) - assert(Time.Undefined != Time.Top) - assert(Time.Undefined != Time.Bottom) - assert(Time.Undefined == Time.Undefined) + assert(Time.Undefined != Time.Top) + assert(Time.Undefined != Time.Bottom) + assert(Time.Undefined == Time.Undefined) - val now = Time.now - assert(now == now) - assert(now == Time.fromNanoseconds(now.inNanoseconds)) - assert(now != now + 1.nanosecond) - } + val now = Time.now + assert(now == now) + assert(now == Time.fromNanoseconds(now.inNanoseconds)) + assert(now != now + 1.nanosecond) + } - "+ delta" in { - assert(10.seconds.afterEpoch + 5.seconds == 15.seconds.afterEpoch) - } + test("Time + delta") { + assert(10.seconds.afterEpoch + 5.seconds == 15.seconds.afterEpoch) + } - "- delta" in { - assert(10.seconds.afterEpoch - 5.seconds == 5.seconds.afterEpoch) - } + test("Time - delta") { + assert(10.seconds.afterEpoch - 5.seconds == 5.seconds.afterEpoch) + } - "- time" in { - assert(10.seconds.afterEpoch - 5.seconds.afterEpoch == 5.seconds) - } + test("Time - time") { + assert(10.seconds.afterEpoch - 5.seconds.afterEpoch == 5.seconds) + } - "max" in { - assert((10.seconds.afterEpoch max 5.seconds.afterEpoch) == 10.seconds.afterEpoch) - assert((5.seconds.afterEpoch max 10.seconds.afterEpoch) == 10.seconds.afterEpoch) - } + test("Time max") { + assert((10.seconds.afterEpoch max 5.seconds.afterEpoch) == 10.seconds.afterEpoch) + assert((5.seconds.afterEpoch max 10.seconds.afterEpoch) == 10.seconds.afterEpoch) + } - "min" in { - assert((10.seconds.afterEpoch min 5.seconds.afterEpoch) == 5.seconds.afterEpoch) - assert((5.seconds.afterEpoch min 10.seconds.afterEpoch) == 5.seconds.afterEpoch) - } + test("Time min") { + assert((10.seconds.afterEpoch min 5.seconds.afterEpoch) == 5.seconds.afterEpoch) + assert((5.seconds.afterEpoch min 10.seconds.afterEpoch) == 5.seconds.afterEpoch) + } - "moreOrLessEquals" in { - val now = Time.now - assert(now.moreOrLessEquals(now + 1.second, 1.second) == true) - assert(now.moreOrLessEquals(now - 1.seconds, 1.second) == true) - assert(now.moreOrLessEquals(now + 2.seconds, 1.second) == false) - assert(now.moreOrLessEquals(now - 2.seconds, 1.second) == false) - } + test("Time moreOrLessEquals") { + val now = Time.now + assert(now.moreOrLessEquals(now + 1.second, 1.second) == true) + assert(now.moreOrLessEquals(now - 1.seconds, 1.second) == true) + assert(now.moreOrLessEquals(now + 2.seconds, 1.second) == false) + assert(now.moreOrLessEquals(now - 2.seconds, 1.second) == false) + } - "floor" in { - val format = new TimeFormat("yyyy-MM-dd HH:mm:ss.SSS") - val t0 = format.parse("2010-12-24 11:04:07.567") - assert(t0.floor(1.millisecond) == t0) - assert(t0.floor(10.milliseconds) == format.parse("2010-12-24 11:04:07.560")) - assert(t0.floor(1.second) == format.parse("2010-12-24 11:04:07.000")) - assert(t0.floor(5.second) == format.parse("2010-12-24 11:04:05.000")) - assert(t0.floor(1.minute) == format.parse("2010-12-24 11:04:00.000")) - assert(t0.floor(1.hour) == format.parse("2010-12-24 11:00:00.000")) - } + test("Time floor") { + val format = new TimeFormat("yyyy-MM-dd HH:mm:ss.SSS") + val t0 = format.parse("2010-12-24 11:04:07.567") + assert(t0.floor(1.millisecond) == t0) + assert(t0.floor(10.milliseconds) == format.parse("2010-12-24 11:04:07.560")) + assert(t0.floor(1.second) == format.parse("2010-12-24 11:04:07.000")) + assert(t0.floor(5.second) == format.parse("2010-12-24 11:04:05.000")) + assert(t0.floor(1.minute) == format.parse("2010-12-24 11:04:00.000")) + assert(t0.floor(1.hour) == format.parse("2010-12-24 11:00:00.000")) + } - "since" in { - val t0 = Time.now - val t1 = t0 + 10.seconds - assert(t1.since(t0) == 10.seconds) - assert(t0.since(t1) == (-10).seconds) - } + test("Time since") { + val t0 = Time.now + val t1 = t0 + 10.seconds + assert(t1.since(t0) == 10.seconds) + assert(t0.since(t1) == (-10).seconds) + } - "sinceEpoch" in { - val t0 = Time.epoch + 100.hours - assert(t0.sinceEpoch == 100.hours) - } + test("Time sinceEpoch") { + val t0 = Time.epoch + 100.hours + assert(t0.sinceEpoch == 100.hours) + } - "sinceNow" in { - Time.withCurrentTimeFrozen { _ => - val t0 = Time.now + 100.hours - assert(t0.sinceNow == 100.hours) - } + test("Time sinceNow") { + Time.withCurrentTimeFrozen { _ => + val t0 = Time.now + 100.hours + assert(t0.sinceNow == 100.hours) } + } - "fromFractionalSeconds" in { - val tolerance = 2.microseconds // we permit 1us slop - - forAll { (i: Int) => - assert( - Time.fromSeconds(i).moreOrLessEquals(Time.fromFractionalSeconds(i.toDouble), tolerance) - ) - } + test("Time fromFractionalSeconds") { + val tolerance = 2.microseconds // we permit 1us slop - forAll { (d: Double) => - val magic = 9223372036854775L // cribbed from Time.fromMicroseconds - val microseconds = d * 1.second.inMicroseconds - whenever(microseconds > -magic && microseconds < magic) { - assert( - Time - .fromMicroseconds(microseconds.toLong) - .moreOrLessEquals(Time.fromFractionalSeconds(d), tolerance) - ) - } - } + forAll { (i: Int) => + assert( + Time.fromSeconds(i).moreOrLessEquals(Time.fromFractionalSeconds(i.toDouble), tolerance) + ) + } - forAll { (l: Long) => - val seconds: Double = l.toDouble / 1.second.inNanoseconds + forAll { (d: Double) => + val magic = 9223372036854775L // cribbed from Time.fromMicroseconds + val microseconds = d * 1.second.inMicroseconds + whenever(microseconds > -magic && microseconds < magic) { assert( - Time.fromFractionalSeconds(seconds).moreOrLessEquals(Time.fromNanoseconds(l), tolerance) + Time + .fromMicroseconds(microseconds.toLong) + .moreOrLessEquals(Time.fromFractionalSeconds(d), tolerance) ) } } - "fromMicroseconds" in { - assert(Time.fromMicroseconds(0).inNanoseconds == 0L) - assert(Time.fromMicroseconds(-1).inNanoseconds == -1L * 1000L) - - assert(Time.fromMicroseconds(Long.MaxValue).inNanoseconds == Long.MaxValue) - assert(Time.fromMicroseconds(Long.MaxValue - 1) == Time.Top) - - assert(Time.fromMicroseconds(Long.MinValue) == Time.Bottom) - assert(Time.fromMicroseconds(Long.MinValue + 1) == Time.Bottom) - - val currentTimeMicros = System.currentTimeMillis() * 1000 + forAll { (l: Long) => + val seconds: Double = l.toDouble / 1.second.inNanoseconds assert( - Time - .fromMicroseconds(currentTimeMicros) - .inNanoseconds == currentTimeMicros.microseconds.inNanoseconds + Time.fromFractionalSeconds(seconds).moreOrLessEquals(Time.fromNanoseconds(l), tolerance) ) } + } - "fromMillis" in { - assert(Time.fromMilliseconds(0).inNanoseconds == 0L) - assert(Time.fromMilliseconds(-1).inNanoseconds == -1L * 1000000L) + test("Time fromMicroseconds") { + assert(Time.fromMicroseconds(0).inNanoseconds == 0L) + assert(Time.fromMicroseconds(-1).inNanoseconds == -1L * 1000L) - assert(Time.fromMilliseconds(Long.MaxValue).inNanoseconds == Long.MaxValue) - assert(Time.fromMilliseconds(Long.MaxValue - 1) == Time.Top) + assert(Time.fromMicroseconds(Long.MaxValue).inNanoseconds == Long.MaxValue) + assert(Time.fromMicroseconds(Long.MaxValue - 1) == Time.Top) - assert(Time.fromMilliseconds(Long.MinValue) == Time.Bottom) - assert(Time.fromMilliseconds(Long.MinValue + 1) == Time.Bottom) + assert(Time.fromMicroseconds(Long.MinValue) == Time.Bottom) + assert(Time.fromMicroseconds(Long.MinValue + 1) == Time.Bottom) - val currentTimeMs = System.currentTimeMillis - assert(Time.fromMilliseconds(currentTimeMs).inNanoseconds == currentTimeMs * 1000000L) - } + val currentTimeMicros = System.currentTimeMillis() * 1000 + assert( + Time + .fromMicroseconds(currentTimeMicros) + .inNanoseconds == currentTimeMicros.microseconds.inNanoseconds + ) + } - "fromMinutes" in { - assert(Time.fromMinutes(0).inNanoseconds == 0L) - assert(Time.fromMinutes(-1).inNanoseconds == -60L * 1000000000L) + test("Time fromMillis") { + assert(Time.fromMilliseconds(0).inNanoseconds == 0L) + assert(Time.fromMilliseconds(-1).inNanoseconds == -1L * 1000000L) - assert(Time.fromMinutes(Int.MaxValue).inNanoseconds == Long.MaxValue) - assert(Time.fromMinutes(Int.MaxValue) == Time.Top) - assert(Time.fromMinutes(Int.MinValue) == Time.Bottom) - } + assert(Time.fromMilliseconds(Long.MaxValue).inNanoseconds == Long.MaxValue) + assert(Time.fromMilliseconds(Long.MaxValue - 1) == Time.Top) - "fromHours" in { - assert(Time.fromHours(1).inNanoseconds == Time.fromMinutes(60).inNanoseconds) + assert(Time.fromMilliseconds(Long.MinValue) == Time.Bottom) + assert(Time.fromMilliseconds(Long.MinValue + 1) == Time.Bottom) - assert(Time.fromHours(0).inNanoseconds == 0L) - assert(Time.fromHours(-1).inNanoseconds == -3600L * 1000000000L) + val currentTimeMs = System.currentTimeMillis + assert(Time.fromMilliseconds(currentTimeMs).inNanoseconds == currentTimeMs * 1000000L) + } - assert(Time.fromHours(Int.MaxValue).inNanoseconds == Long.MaxValue) - assert(Time.fromHours(Int.MaxValue) == Time.Top) - assert(Time.fromHours(Int.MinValue) == Time.Bottom) - } + test("Time fromMinutes") { + assert(Time.fromMinutes(0).inNanoseconds == 0L) + assert(Time.fromMinutes(-1).inNanoseconds == -60L * 1000000000L) - "fromDays" in { - assert(Time.fromDays(1).inNanoseconds == Time.fromHours(24).inNanoseconds) + assert(Time.fromMinutes(Int.MaxValue).inNanoseconds == Long.MaxValue) + assert(Time.fromMinutes(Int.MaxValue) == Time.Top) + assert(Time.fromMinutes(Int.MinValue) == Time.Bottom) + } - assert(Time.fromDays(0).inNanoseconds == 0L) - assert(Time.fromDays(-1).inNanoseconds == -3600L * 24L * 1000000000L) + test("Time fromHours") { + assert(Time.fromHours(1).inNanoseconds == Time.fromMinutes(60).inNanoseconds) - assert(Time.fromDays(Int.MaxValue).inNanoseconds == Long.MaxValue) - assert(Time.fromDays(Int.MaxValue) == Time.Top) - assert(Time.fromDays(Int.MinValue) == Time.Bottom) - } + assert(Time.fromHours(0).inNanoseconds == 0L) + assert(Time.fromHours(-1).inNanoseconds == -3600L * 1000000000L) - "until" in { - val t0 = Time.now - val t1 = t0 + 10.seconds - assert(t0.until(t1) == 10.seconds) - assert(t1.until(t0) == (-10).seconds) - } + assert(Time.fromHours(Int.MaxValue).inNanoseconds == Long.MaxValue) + assert(Time.fromHours(Int.MaxValue) == Time.Top) + assert(Time.fromHours(Int.MinValue) == Time.Bottom) + } - "untilEpoch" in { - val t0 = Time.epoch - 100.hours - assert(t0.untilEpoch == 100.hours) - } + test("Time fromDays") { + assert(Time.fromDays(1).inNanoseconds == Time.fromHours(24).inNanoseconds) - "untilNow" in { - Time.withCurrentTimeFrozen { _ => - val t0 = Time.now - 100.hours - assert(t0.untilNow == 100.hours) - } + assert(Time.fromDays(0).inNanoseconds == 0L) + assert(Time.fromDays(-1).inNanoseconds == -3600L * 24L * 1000000000L) + + assert(Time.fromDays(Int.MaxValue).inNanoseconds == Long.MaxValue) + assert(Time.fromDays(Int.MaxValue) == Time.Top) + assert(Time.fromDays(Int.MinValue) == Time.Bottom) + } + + test("Time until") { + val t0 = Time.now + val t1 = t0 + 10.seconds + assert(t0.until(t1) == 10.seconds) + assert(t1.until(t0) == (-10).seconds) + } + + test("Time untilEpoch") { + val t0 = Time.epoch - 100.hours + assert(t0.untilEpoch == 100.hours) + } + + test("Time untilNow") { + Time.withCurrentTimeFrozen { _ => + val t0 = Time.now - 100.hours + assert(t0.untilNow == 100.hours) } + } - "toInstant" in { - Time.withCurrentTimeFrozen { _ => - val instant = Instant.ofEpochMilli(Time.now.inMilliseconds) - assert(instant.toEpochMilli == Time.now.inMillis) - // java.time.Instant:getNano returns the nanoseconds of the second - assert(instant.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) - } + test("Time toInstant") { + Time.withCurrentTimeFrozen { _ => + val instant = Instant.ofEpochMilli(Time.now.inMilliseconds) + assert(instant.toEpochMilli == Time.now.inMillis) + // java.time.Instant:getNano returns the nanoseconds of the second + assert(instant.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) } + } - "toZonedDateTime" in { - Time.withCurrentTimeFrozen { _ => - val zonedDateTime = - ZonedDateTime.ofInstant(Instant.ofEpochMilli(Time.now.inMilliseconds), ZoneOffset.UTC) - assert(Time.now.toZonedDateTime == zonedDateTime) - // java.time.Instant:getNano returns the nanoseconds of the second - assert(zonedDateTime.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) - } + test("Time toZonedDateTime") { + Time.withCurrentTimeFrozen { _ => + val zonedDateTime = + ZonedDateTime.ofInstant(Instant.ofEpochMilli(Time.now.inMilliseconds), ZoneOffset.UTC) + assert(Time.now.toZonedDateTime == zonedDateTime) + // java.time.Instant:getNano returns the nanoseconds of the second + assert(zonedDateTime.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) } + } - "toOffsetDateTime" in { - Time.withCurrentTimeFrozen { _ => - val offsetDateTime = - OffsetDateTime.ofInstant(Instant.ofEpochMilli(Time.now.inMilliseconds), ZoneOffset.UTC) - assert(Time.now.toOffsetDateTime == offsetDateTime) - // java.time.Instant:getNano returns the nanoseconds of the second - assert(offsetDateTime.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) - } + test("Time toOffsetDateTime") { + Time.withCurrentTimeFrozen { _ => + val offsetDateTime = + OffsetDateTime.ofInstant(Instant.ofEpochMilli(Time.now.inMilliseconds), ZoneOffset.UTC) + assert(Time.now.toOffsetDateTime == offsetDateTime) + // java.time.Instant:getNano returns the nanoseconds of the second + assert(offsetDateTime.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) } } } diff --git a/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala b/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala index 9e17fef3ff..84bbb5c19f 100644 --- a/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala @@ -1,44 +1,41 @@ package com.twitter.util import java.util.Locale +import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.wordspec.AnyWordSpec - -class TwitterDateFormatTest extends AnyWordSpec { - "TwitterDateFormat" should { - "disallow Y without w" in { - intercept[IllegalArgumentException] { - TwitterDateFormat("YYYYMMDD") - } - intercept[IllegalArgumentException] { - TwitterDateFormat("YMD", Locale.GERMAN) - } +class TwitterDateFormatTest extends AnyFunSuite { + test("TwitterDateFormat should disallow Y without w") { + intercept[IllegalArgumentException] { + TwitterDateFormat("YYYYMMDD") } - - "allow Y with w" in { - TwitterDateFormat("YYYYww") + intercept[IllegalArgumentException] { + TwitterDateFormat("YMD", Locale.GERMAN) } + } - "allow Y when quoted" in { - TwitterDateFormat("yyyy 'Year'") - } + test("TwitterDateFormat should allow Y with w") { + TwitterDateFormat("YYYYww") + } + + test("TwitterDateFormat should allow Y when quoted") { + TwitterDateFormat("yyyy 'Year'") + } - "stripSingleQuoted" in { - import TwitterDateFormat._ - assert(stripSingleQuoted("") == "") - assert(stripSingleQuoted("YYYY") == "YYYY") - assert(stripSingleQuoted("''") == "") - assert(stripSingleQuoted("'abc'") == "") - assert(stripSingleQuoted("x'abc'") == "x") - assert(stripSingleQuoted("'abc'x") == "x") - assert(stripSingleQuoted("'abc'def'ghi'") == "def") + test("TwitterDateFormat should stripSingleQuoted") { + import TwitterDateFormat._ + assert(stripSingleQuoted("") == "") + assert(stripSingleQuoted("YYYY") == "YYYY") + assert(stripSingleQuoted("''") == "") + assert(stripSingleQuoted("'abc'") == "") + assert(stripSingleQuoted("x'abc'") == "x") + assert(stripSingleQuoted("'abc'x") == "x") + assert(stripSingleQuoted("'abc'def'ghi'") == "def") - intercept[IllegalArgumentException] { - stripSingleQuoted("'abc") - } - intercept[IllegalArgumentException] { - stripSingleQuoted("'abc'def'ghi") - } + intercept[IllegalArgumentException] { + stripSingleQuoted("'abc") + } + intercept[IllegalArgumentException] { + stripSingleQuoted("'abc'def'ghi") } } } From 5a780df5c366192e32e8247a31935d4294783af0 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 7 Oct 2022 14:21:32 +0000 Subject: [PATCH 732/796] util-logging: migrate tests off of WordSpec Problem: Our recommendation for testing is to use AnyFunSuite, but our own testing uses AnyWordSpec Solution: Migrate JIRA Issues: CSL-9463 Differential Revision: https://phabricator.twitter.biz/D990058 --- .../com/twitter/logging/FileHandlerTest.scala | 483 +++++++------- .../com/twitter/logging/FormatterTest.scala | 307 +++++---- .../com/twitter/logging/LoggerTest.scala | 619 +++++++++--------- .../twitter/logging/QueueingHandlerTest.scala | 247 ++++--- .../twitter/logging/SyslogHandlerTest.scala | 152 +++-- .../logging/ThrottledHandlerTest.scala | 110 ++-- 6 files changed, 929 insertions(+), 989 deletions(-) diff --git a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala index dac13d0161..f8a7f587b4 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FileHandlerTest.scala @@ -30,9 +30,9 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.util.function.BiPredicate -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class FileHandlerTest extends AnyWordSpec with TempFolder { +class FileHandlerTest extends AnyFunSuite with TempFolder { def reader(filename: String): BufferedReader = { new BufferedReader(new InputStreamReader(new FileInputStream(new File(folderName, filename)))) } @@ -60,309 +60,268 @@ class FileHandlerTest extends AnyWordSpec with TempFolder { matcher(name) ).toArray.map(_.toString) - "FileHandler" should { - val record1 = new javalog.LogRecord(Level.INFO, "first post!") - val record2 = new javalog.LogRecord(Level.INFO, "second post") + val record1 = new javalog.LogRecord(Level.INFO, "first post!") + val record2 = new javalog.LogRecord(Level.INFO, "second post") - "honor append setting on logfiles" in { - withTempFolder { - val f = writer("test.log") - f.write("hello!\n") - f.close() + test("FileHandler should honor append setting on logfiles") { + withTempFolder { + val f = writer("test.log") + f.write("hello!\n") + f.close() - val handler = FileHandler( - filename = folderName + "/test.log", - rollPolicy = Policy.Hourly, - append = true, - formatter = BareFormatter - ).apply() + val handler = FileHandler( + filename = folderName + "/test.log", + rollPolicy = Policy.Hourly, + append = true, + formatter = BareFormatter + ).apply() - handler.publish(record1) + handler.publish(record1) - val f2 = reader("test.log") - assert(f2.readLine == "hello!") - } + val f2 = reader("test.log") + assert(f2.readLine == "hello!") + } - withTempFolder { - val f = writer("test.log") - f.write("hello!\n") - f.close() + withTempFolder { + val f = writer("test.log") + f.write("hello!\n") + f.close() - val handler = FileHandler( - filename = folderName + "/test.log", - rollPolicy = Policy.Hourly, - append = false, - formatter = BareFormatter - ).apply() + val handler = FileHandler( + filename = folderName + "/test.log", + rollPolicy = Policy.Hourly, + append = false, + formatter = BareFormatter + ).apply() - handler.publish(record1) + handler.publish(record1) - val f2 = reader("test.log") - assert(f2.readLine == "first post!") - } + val f2 = reader("test.log") + assert(f2.readLine == "first post!") } + } - // /* Test is commented out according to REPLA-618 */ - // - // "respond to a sighup to reopen a logfile with sun.misc" in { - // try { - // val signalClass = Class.forName("sun.misc.Signal") - // val sighup = signalClass.getConstructor(classOf[String]).newInstance("HUP").asInstanceOf[Object] - // val raiseMethod = signalClass.getMethod("raise", signalClass) - - // withTempFolder { - // val handler = FileHandler( - // filename = folderName + "/new.log", - // rollPolicy = Policy.SigHup, - // append = true, - // formatter = BareFormatter - // ).apply() - - // val logFile = new File(folderName, "new.log") - // logFile.renameTo(new File(folderName, "old.log")) - // handler.publish(record1) - - // raiseMethod.invoke(null, sighup) - - // val newLogFile = new File(folderName, "new.log") - // newLogFile.exists() should eventually(be_==(true)) - - // handler.publish(record2) - - // val oldReader = reader("old.log") - // assert(oldReader.readLine == "first post!") - // val newReader = reader("new.log") - // assert(newReader.readLine == "second post") - // } - // } catch { - // case ex: ClassNotFoundException => - // } - // } - - "roll logs on time" should { - "hourly" in { - withTempFolder { - val handler = FileHandler( - filename = folderName + "/test.log", - rollPolicy = Policy.Hourly, - append = true, - formatter = BareFormatter - ).apply() - assert(handler.computeNextRollTime(1206769996722L).contains(1206770400000L)) - assert(handler.computeNextRollTime(1206770400000L).contains(1206774000000L)) - assert(handler.computeNextRollTime(1206774000001L).contains(1206777600000L)) - } - } + test("FileHandler should roll hourly logs on time") { + withTempFolder { + val handler = FileHandler( + filename = folderName + "/test.log", + rollPolicy = Policy.Hourly, + append = true, + formatter = BareFormatter + ).apply() + assert(handler.computeNextRollTime(1206769996722L).contains(1206770400000L)) + assert(handler.computeNextRollTime(1206770400000L).contains(1206774000000L)) + assert(handler.computeNextRollTime(1206774000001L).contains(1206777600000L)) + } + } - "weekly" in { - withTempFolder { - val handler = FileHandler( - filename = folderName + "/test.log", - rollPolicy = Policy.Weekly(Calendar.SUNDAY), - append = true, - formatter = new Formatter(timezone = Some("GMT-7:00")) - ).apply() - assert(handler.computeNextRollTime(1250354734000L).contains(1250406000000L)) - assert(handler.computeNextRollTime(1250404734000L).contains(1250406000000L)) - assert(handler.computeNextRollTime(1250406001000L).contains(1251010800000L)) - assert(handler.computeNextRollTime(1250486000000L).contains(1251010800000L)) - assert(handler.computeNextRollTime(1250496000000L).contains(1251010800000L)) - } - } + test("FileHandler should roll weekly logs on time") { + withTempFolder { + val handler = FileHandler( + filename = folderName + "/test.log", + rollPolicy = Policy.Weekly(Calendar.SUNDAY), + append = true, + formatter = new Formatter(timezone = Some("GMT-7:00")) + ).apply() + assert(handler.computeNextRollTime(1250354734000L).contains(1250406000000L)) + assert(handler.computeNextRollTime(1250404734000L).contains(1250406000000L)) + assert(handler.computeNextRollTime(1250406001000L).contains(1251010800000L)) + assert(handler.computeNextRollTime(1250486000000L).contains(1251010800000L)) + assert(handler.computeNextRollTime(1250496000000L).contains(1251010800000L)) } + } - // verify that at the proper time, the log file rolls and resets. - "roll logs into new files" in { - withTempFolder { - val handler = - new FileHandler(folderName + "/test.log", Policy.Hourly, true, -1, BareFormatter, None) - Time.withCurrentTimeFrozen { time => - handler.publish(record1) - val date = new Date(Time.now.inMilliseconds) + // verify that at the proper time, the log file rolls and resets. + test("FileHandler should roll logs into new files") { + withTempFolder { + val handler = + new FileHandler(folderName + "/test.log", Policy.Hourly, true, -1, BareFormatter, None) + Time.withCurrentTimeFrozen { time => + handler.publish(record1) + val date = new Date(Time.now.inMilliseconds) - time.advance(1.hour) + time.advance(1.hour) - handler.publish(record2) - handler.close() + handler.publish(record2) + handler.close() - assert(reader("test-" + handler.timeSuffix(date) + ".log").readLine == "first post!") - assert(reader("test.log").readLine == "second post") - } + assert(reader("test-" + handler.timeSuffix(date) + ".log").readLine == "first post!") + assert(reader("test.log").readLine == "second post") } } + } - "keep no more than N log files around" in { - withTempFolder { - assert(new File(folderName).list().length == 0) - - val handler = FileHandler( - filename = folderName + "/test.log", - rollPolicy = Policy.Hourly, - append = true, - rotateCount = 2, - formatter = BareFormatter - ).apply() - - handler.publish(record1) - assert(new File(folderName).list().length == 1) - handler.roll() + test("FileHandler should keep no more than N log files around") { + withTempFolder { + assert(new File(folderName).list().length == 0) + + val handler = FileHandler( + filename = folderName + "/test.log", + rollPolicy = Policy.Hourly, + append = true, + rotateCount = 2, + formatter = BareFormatter + ).apply() + + handler.publish(record1) + assert(new File(folderName).list().length == 1) + handler.roll() + + handler.publish(record1) + assert(new File(folderName).list().length == 2) + handler.roll() + + handler.publish(record1) + assert(new File(folderName).list().length == 2) + handler.close() + } + } + test("FileHandler ignores the target filename despite shorter filenames") { + // even if the sort order puts the target filename before `rotateCount` other + // files, it should not be removed + withTempFolder { + assert(new File(folderName).list().length == 0) + val namePrefix = "test" + val name = namePrefix + ".log" + + val handler = FileHandler( + filename = folderName + "/" + name, + rollPolicy = Policy.Hourly, + append = true, + rotateCount = 1, + formatter = BareFormatter + ).apply() + + // create a file without the '.log' suffix, which will sort before the target + new File(folderName, namePrefix).createNewFile() + + def flush() = { handler.publish(record1) - assert(new File(folderName).list().length == 2) handler.roll() - - handler.publish(record1) - assert(new File(folderName).list().length == 2) - handler.close() + assert(new File(folderName).list().length == 3) } + + // the target, 1 rotated file, and the short file should all remain + (1 to 5).foreach { _ => flush() } + val fileSet = new File(folderName).list().toSet + assert(fileSet.contains(name)) + assert(fileSet.contains(namePrefix)) } + } - "ignores the target filename despite shorter filenames" in { - // even if the sort order puts the target filename before `rotateCount` other - // files, it should not be removed - withTempFolder { - assert(new File(folderName).list().length == 0) - val namePrefix = "test" - val name = namePrefix + ".log" - - val handler = FileHandler( - filename = folderName + "/" + name, - rollPolicy = Policy.Hourly, - append = true, - rotateCount = 1, - formatter = BareFormatter - ).apply() - - // create a file without the '.log' suffix, which will sort before the target - new File(folderName, namePrefix).createNewFile() - - def flush() = { - handler.publish(record1) - handler.roll() - assert(new File(folderName).list().length == 3) - } + test("FileHandler correctly handles relative paths") { - // the target, 1 rotated file, and the short file should all remain - (1 to 5).foreach { _ => flush() } - val fileSet = new File(folderName).list().toSet - assert(fileSet.contains(name)) - assert(fileSet.contains(namePrefix)) - } + val userDir = System.getProperty("user.dir") + val filenamePrefix = UUID.randomUUID().toString + + try { + val handler = FileHandler( + filename = filenamePrefix + ".log", // Note relative path! + rollPolicy = Policy.Hourly, + append = true, + rotateCount = 2, + formatter = BareFormatter + ).apply() + + handler.publish(record1) + assert(filesContainingName(userDir, filenamePrefix).length == 1) + handler.roll() + + handler.publish(record1) + assert(filesContainingName(userDir, filenamePrefix).length == 2) + handler.roll() + + handler.publish(record1) + assert(filesContainingName(userDir, filenamePrefix).length == 2) + handler.close() + + } finally { + filesContainingName(userDir, filenamePrefix).foreach(f => new File(f).delete()) } + } - "correctly handles relative paths" in { + test("FileHandler should roll log files based on max size") { + withTempFolder { + // roll the log on the 3rd write. + val maxSize = record1.getMessage.length * 3 - 1 - val userDir = System.getProperty("user.dir") - val filenamePrefix = UUID.randomUUID().toString + assert(new File(folderName).list().length == 0) - try { - val handler = FileHandler( - filename = filenamePrefix + ".log", // Note relative path! - rollPolicy = Policy.Hourly, - append = true, - rotateCount = 2, - formatter = BareFormatter - ).apply() + val handler = FileHandler( + filename = folderName + "/test.log", + rollPolicy = Policy.MaxSize(maxSize.bytes), + append = true, + formatter = BareFormatter + ).apply() + // move time forward so the rotated logfiles will have distinct names. + Time.withCurrentTimeFrozen { time => + time.advance(1.second) handler.publish(record1) - assert(filesContainingName(userDir, filenamePrefix).length == 1) - handler.roll() - + assert(new File(folderName).list().length == 1) + time.advance(1.second) handler.publish(record1) - assert(filesContainingName(userDir, filenamePrefix).length == 2) - handler.roll() + assert(new File(folderName).list().length == 1) + time.advance(1.second) handler.publish(record1) - assert(filesContainingName(userDir, filenamePrefix).length == 2) - handler.close() + assert(new File(folderName).list().length == 2) + time.advance(1.second) + handler.publish(record1) + assert(new File(folderName).list().length == 2) - } finally { - filesContainingName(userDir, filenamePrefix).foreach(f => new File(f).delete()) + time.advance(1.second) + handler.publish(record1) + assert(new File(folderName).list().length == 3) } - } - "roll log files based on max size" in { - withTempFolder { - // roll the log on the 3rd write. - val maxSize = record1.getMessage.length * 3 - 1 - - assert(new File(folderName).list().length == 0) - - val handler = FileHandler( - filename = folderName + "/test.log", - rollPolicy = Policy.MaxSize(maxSize.bytes), - append = true, - formatter = BareFormatter - ).apply() - - // move time forward so the rotated logfiles will have distinct names. - Time.withCurrentTimeFrozen { time => - time.advance(1.second) - handler.publish(record1) - assert(new File(folderName).list().length == 1) - time.advance(1.second) - handler.publish(record1) - assert(new File(folderName).list().length == 1) - - time.advance(1.second) - handler.publish(record1) - assert(new File(folderName).list().length == 2) - time.advance(1.second) - handler.publish(record1) - assert(new File(folderName).list().length == 2) - - time.advance(1.second) - handler.publish(record1) - assert(new File(folderName).list().length == 3) - } - - handler.close() - } + handler.close() } + } - /** - * This test mimics the scenario that, if log file exists, FileHandler should pick up it's size - * and roll over at the set limit, in order to prevent the file size from growing. - * The test succeeds if the roll over takes place as desired else fails. - */ - withTempFolder { - "rollover at set limit when test is restarted, execution" in { - val logLevel = Level.INFO - val fileSizeInMegaBytes: Long = 1 - val record1 = new javalog.LogRecord(logLevel, "Sending bytes to fill up file") - val filename = folderName + "/LogFileDir/testFileSize.log" - val rollPolicy = Policy.MaxSize(fileSizeInMegaBytes.megabytes) - val rotateCount = 8 - val append = true - val formatter = BareFormatter - - val handler = FileHandler(filename, rollPolicy, append, rotateCount, formatter).apply() - for (a <- 1 to 20000) { - handler.publish(record1) - } - handler.close() + /** + * This test mimics the scenario that, if log file exists, FileHandler should pick up it's size + * and roll over at the set limit, in order to prevent the file size from growing. + * The test succeeds if the roll over takes place as desired else fails. + */ + withTempFolder { + test("FileHandler should rollover at set limit when test is restarted, execution") { + val logLevel = Level.INFO + val fileSizeInMegaBytes: Long = 1 + val record1 = new javalog.LogRecord(logLevel, "Sending bytes to fill up file") + val filename = folderName + "/LogFileDir/testFileSize.log" + val rollPolicy = Policy.MaxSize(fileSizeInMegaBytes.megabytes) + val rotateCount = 8 + val append = true + val formatter = BareFormatter + + val handler = FileHandler(filename, rollPolicy, append, rotateCount, formatter).apply() + for (a <- 1 to 20000) { + handler.publish(record1) + } + handler.close() - val handler2 = FileHandler(filename, rollPolicy, append, rotateCount, formatter).apply() - for (a <- 1 to 20000) { - handler2.publish(record1) - } - handler2.close() - - def listLogFiles(dir: String): List[File] = { - val d = new File(dir) - if (d.exists && d.isDirectory) { - d.listFiles.filter(_.isFile).toList - } else { - List[File]() - } + val handler2 = FileHandler(filename, rollPolicy, append, rotateCount, formatter).apply() + for (a <- 1 to 20000) { + handler2.publish(record1) + } + handler2.close() + + def listLogFiles(dir: String): List[File] = { + val d = new File(dir) + if (d.exists && d.isDirectory) { + d.listFiles.filter(_.isFile).toList + } else { + List[File]() } + } - val files = listLogFiles(folderName + "/LogFileDir") - files.foreach { (f: File) => - val len = f.length().bytes - if (len > fileSizeInMegaBytes.megabytes) { - fail("Failed to roll over the log file") - } + val files = listLogFiles(folderName + "/LogFileDir") + files.foreach { (f: File) => + val len = f.length().bytes + if (len > fileSizeInMegaBytes.megabytes) { + fail("Failed to roll over the log file") } } } diff --git a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala index e3f8892322..c82e126ca0 100644 --- a/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/FormatterTest.scala @@ -17,11 +17,10 @@ package com.twitter.logging import java.util.{logging => javalog} - import com.twitter.conversions.StringOps._ -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class FormatterTest extends AnyWordSpec { +class FormatterTest extends AnyFunSuite { val basicFormatter = new Formatter val utcFormatter = new Formatter( @@ -66,183 +65,179 @@ class FormatterTest extends AnyWordSpec { record4.setMillis(1206769996722L) record4.setThrown(noStackTraceException) - "Formatter" should { - "create a prefix" in { - assert( - basicFormatter.formatPrefix(Level.ERROR, "20080329-05:53:16.722", "(root)") == - "ERR [20080329-05:53:16.722] (root): " - ) - assert( - basicFormatter.formatPrefix(Level.DEBUG, "20080329-05:53:16.722", "(root)") == - "DEB [20080329-05:53:16.722] (root): " - ) - assert( - basicFormatter.formatPrefix(Level.WARNING, "20080329-05:53:16.722", "(root)") == - "WAR [20080329-05:53:16.722] (root): " - ) - } + test("Formatter should create a prefix") { + assert( + basicFormatter.formatPrefix(Level.ERROR, "20080329-05:53:16.722", "(root)") == + "ERR [20080329-05:53:16.722] (root): " + ) + assert( + basicFormatter.formatPrefix(Level.DEBUG, "20080329-05:53:16.722", "(root)") == + "DEB [20080329-05:53:16.722] (root): " + ) + assert( + basicFormatter.formatPrefix(Level.WARNING, "20080329-05:53:16.722", "(root)") == + "WAR [20080329-05:53:16.722] (root): " + ) + } - "format a log level name" in { - assert(basicFormatter.formatLevelName(Level.ERROR) == "ERROR") - assert(basicFormatter.formatLevelName(Level.DEBUG) == "DEBUG") - assert(basicFormatter.formatLevelName(Level.WARNING) == "WARNING") - } + test("Formatter should format a log level name") { + assert(basicFormatter.formatLevelName(Level.ERROR) == "ERROR") + assert(basicFormatter.formatLevelName(Level.DEBUG) == "DEBUG") + assert(basicFormatter.formatLevelName(Level.WARNING) == "WARNING") + } - "format text" in { - val record = new LogRecord(Level.ERROR, "error %s") - assert(basicFormatter.formatText(record) == "error %s") - record.setParameters(Array("123")) - assert(basicFormatter.formatText(record) == "error 123") - } + test("Formatter should format text") { + val record = new LogRecord(Level.ERROR, "error %s") + assert(basicFormatter.formatText(record) == "error %s") + record.setParameters(Array("123")) + assert(basicFormatter.formatText(record) == "error 123") + } - "format message lines" in { - val record = new LogRecord(Level.ERROR, "how now brown cow") - assert(basicFormatter.formatMessageLines(record) sameElements Array("how now brown cow")) - } + test("Formatter should format message lines") { + val record = new LogRecord(Level.ERROR, "how now brown cow") + assert(basicFormatter.formatMessageLines(record) sameElements Array("how now brown cow")) + } - "format message lines multiple values" in { - val record = new LogRecord(Level.ERROR, "how now brown cow\nthe quick brown fox") - assert(basicFormatter + test("Formatter should format message lines multiple values") { + val record = new LogRecord(Level.ERROR, "how now brown cow\nthe quick brown fox") + assert( + basicFormatter .formatMessageLines(record) sameElements Array("how now brown cow", "the quick brown fox")) - } + } - "format message lines with message" in { - val record = new LogRecord(Level.ERROR, "how now brown cow") - assert( - basicFormatter.formatMessageLines("how now brown cow", record.getThrown) sameElements Array( - "how now brown cow")) - } + test("Formatter should format message lines with message") { + val record = new LogRecord(Level.ERROR, "how now brown cow") + assert( + basicFormatter.formatMessageLines("how now brown cow", record.getThrown) sameElements Array( + "how now brown cow")) + } - "format message lines with message and multiple values" in { - val record = new LogRecord(Level.ERROR, "how now brown cow\nthe quick brown fox") - // ignores the message from the record - assert( - basicFormatter.formatMessageLines("how now brown cow", record.getThrown) sameElements Array( - "how now brown cow")) - } + test("Formatter should format message lines with message and multiple values") { + val record = new LogRecord(Level.ERROR, "how now brown cow\nthe quick brown fox") + // ignores the message from the record + assert( + basicFormatter.formatMessageLines("how now brown cow", record.getThrown) sameElements Array( + "how now brown cow")) + } - "format message lines with message and multiple values again" in { - val record = new LogRecord(Level.ERROR, "how now brown cow\nthe quick brown fox") - // ignores the message from the record - assert( - basicFormatter - .formatMessageLines( - "how now brown cow\nthe quick brown fox", - record.getThrown) sameElements Array("how now brown cow", "the quick brown fox")) - } + test("Formatter should format message lines with message and multiple values again") { + val record = new LogRecord(Level.ERROR, "how now brown cow\nthe quick brown fox") + // ignores the message from the record + assert( + basicFormatter + .formatMessageLines( + "how now brown cow\nthe quick brown fox", + record.getThrown) sameElements Array("how now brown cow", "the quick brown fox")) + } + + test("Formatter should format a timestamp") { + assert(utcFormatter.format(record1) == "ERR [20080329-05:53:16.722] jobs: boo.\n") + } - "format a timestamp" in { - assert(utcFormatter.format(record1) == "ERR [20080329-05:53:16.722] jobs: boo.\n") + test("Formatter should do lazy message evaluation") { + var callCount = 0 + def getSideEffect = { + callCount += 1 + "ok" } - "do lazy message evaluation" in { - var callCount = 0 - def getSideEffect = { - callCount += 1 - "ok" - } + val record = new LazyLogRecord(Level.DEBUG, "this is " + getSideEffect) - val record = new LazyLogRecord(Level.DEBUG, "this is " + getSideEffect) + assert(callCount == 0) + assert(basicFormatter.formatText(record) == "this is ok") + assert(callCount == 1) + assert(basicFormatter.formatText(record) == "this is ok") + assert(callCount == 1) + } - assert(callCount == 0) - assert(basicFormatter.formatText(record) == "this is ok") - assert(callCount == 1) - assert(basicFormatter.formatText(record) == "this is ok") - assert(callCount == 1) - } + test("Formatter should format package names") { + assert(utcFormatter.format(record1) == "ERR [20080329-05:53:16.722] jobs: boo.\n") + assert( + fullPackageFormatter.format(record1) == + "ERR [20080329-05:53:16.722] com.example.jobs: boo.\n" + ) + } - "format package names" in { - assert(utcFormatter.format(record1) == "ERR [20080329-05:53:16.722] jobs: boo.\n") - assert( - fullPackageFormatter.format(record1) == - "ERR [20080329-05:53:16.722] com.example.jobs: boo.\n" - ) - } + test("Formatter should handle other prefixes") { + assert(prefixFormatter.format(record2) == "jobs 05:53 DEBU useless info.\n") + } - "handle other prefixes" in { - assert(prefixFormatter.format(record2) == "jobs 05:53 DEBU useless info.\n") - } + test("Formatter should truncate line") { + assert( + truncateFormatter.format(record3) == + "CRI [20080329-05:53:16.722] whiskey: Something terrible happened th...\n" + ) + } - "truncate line" in { - assert( - truncateFormatter.format(record3) == - "CRI [20080329-05:53:16.722] whiskey: Something terrible happened th...\n" - ) + object ExceptionLooper { + def cycle(n: Int): Unit = { + if (n == 0) { + throw new Exception("Aie!") + } else { + cycle(n - 1) + throw new Exception("this is just here to fool the tail recursion optimizer") + } } - "write stack traces" should { - object ExceptionLooper { - def cycle(n: Int): Unit = { - if (n == 0) { - throw new Exception("Aie!") - } else { - cycle(n - 1) - throw new Exception("this is just here to fool the tail recursion optimizer") - } - } - - def cycle2(n: Int): Unit = { - try { - cycle(n) - } catch { - case t: Throwable => - throw new Exception("grrrr", t) - } - } + def cycle2(n: Int): Unit = { + try { + cycle(n) + } catch { + case t: Throwable => + throw new Exception("grrrr", t) } + } + } - def scrub(in: String) = { - in.regexSub("""FormatterTest.scala:\d+""".r) { m => "FormatterTest.scala:NNN" } - .regexSub("""FormatterTest\$[\w\\$]+""".r) { m => "FormatterTest$$" } - } + def scrub(in: String) = { + in.regexSub("""FormatterTest.scala:\d+""".r) { m => "FormatterTest.scala:NNN" } + .regexSub("""FormatterTest\$[\w\\$]+""".r) { m => "FormatterTest$$" } + } - "simple" in { - val exception = - try { - ExceptionLooper.cycle(10) - null - } catch { - case t: Throwable => t - } - assert( - Formatter.formatStackTrace(exception, 5).map { scrub(_) } == List( - " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", - " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", - " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", - " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", - " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", - " (...more...)" - ) - ) + test("Formatter should write simple stack traces") { + val exception = + try { + ExceptionLooper.cycle(10) + null + } catch { + case t: Throwable => t } + assert( + Formatter.formatStackTrace(exception, 5).map { scrub(_) } == List( + " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", + " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", + " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", + " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", + " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", + " (...more...)" + ) + ) + } - "nested" in { - val exception = - try { - ExceptionLooper.cycle2(2) - null - } catch { - case t: Throwable => t - } - assert( - Formatter.formatStackTrace(exception, 1).map { scrub(_) } == List( - " at com.twitter.logging.FormatterTest$$.cycle2(FormatterTest.scala:NNN)", - " (...more...)", - "Caused by java.lang.Exception: Aie!", - " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", - " (...more...)" - ) - ) - + test("Formatter should write nested stack traces") { + val exception = + try { + ExceptionLooper.cycle2(2) + null + } catch { + case t: Throwable => t } + assert( + Formatter.formatStackTrace(exception, 1).map { scrub(_) } == List( + " at com.twitter.logging.FormatterTest$$.cycle2(FormatterTest.scala:NNN)", + " (...more...)", + "Caused by java.lang.Exception: Aie!", + " at com.twitter.logging.FormatterTest$$.cycle(FormatterTest.scala:NNN)", + " (...more...)" + ) + ) + } - "log even blank exceptions" in { - assert( - utcFormatter.format(record4) == - "ERR [20080329-05:53:16.722] jobs: with minimal exception\n" + - "ERR [20080329-05:53:16.722] jobs: java.lang.Exception: fast exception no stacktrace\n" - ) - } - } + test("Formatter should log even blank exceptions") { + assert( + utcFormatter.format(record4) == + "ERR [20080329-05:53:16.722] jobs: with minimal exception\n" + + "ERR [20080329-05:53:16.722] jobs: java.lang.Exception: fast exception no stacktrace\n" + ) } } diff --git a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala index 17ee59e186..ab849d7c3e 100644 --- a/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/LoggerTest.scala @@ -26,10 +26,10 @@ import java.util.concurrent.Future import java.util.concurrent.TimeUnit import java.util.{logging => javalog} import org.scalatest.BeforeAndAfter +import org.scalatest.funsuite.AnyFunSuite import scala.collection.mutable -import org.scalatest.wordspec.AnyWordSpec -class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { +class LoggerTest extends AnyFunSuite with TempFolder with BeforeAndAfter { val logLevel = Logger.levelNames(Option[String](System.getenv("log")).getOrElse("FATAL").toUpperCase) @@ -107,358 +107,349 @@ class LoggerTest extends AnyWordSpec with TempFolder with BeforeAndAfter { case class WithLogLevel(logLevel: Level) extends Exception("") with HasLogLevel - "Logger" should { - val h = new LoggerSpecHelper - import h._ + val h = new LoggerSpecHelper + import h._ - def before() = { - Logger.clearHandlers() - timeFrozenHandler.clear() - myHandler = new StringHandler(BareFormatter, None) - log = Logger.get("") - log.setLevel(Level.ERROR) - log.addHandler(myHandler) - } - - "not execute string creation and concatenation" in { - val logger = Logger.get("lazyTest1") - var executed = false - def function() = { - executed = true - "asdf" + executed + " hi there" - } + def before() = { + Logger.clearHandlers() + timeFrozenHandler.clear() + myHandler = new StringHandler(BareFormatter, None) + log = Logger.get("") + log.setLevel(Level.ERROR) + log.addHandler(myHandler) + } - logger.debugLazy(function()) - assert(!executed) + test("Logger should not execute string creation and concatenation") { + val logger = Logger.get("lazyTest1") + var executed = false + def function() = { + executed = true + "asdf" + executed + " hi there" } - "execute string creation and concatenation is FAILING" in { - val logger = Logger.get("lazyTest2") - logger.setLevel(Level.DEBUG) - var executed = false - def function() = { - executed = true - "asdf" + executed + " hi there" - } - logger.debugLazy(function()) - assert(executed) - } + logger.debugLazy(function()) + assert(!executed) + } - "make sure compiles with normal string case" in { - //in some cases, if debugLazy definition was changed, the below would no longer compile - val logger = Logger.get("lazyTest3") - val executed = true - logger.debugLazy("hi there" + executed + "cool") + test("Logger should execute string creation and concatenation is FAILING") { + val logger = Logger.get("lazyTest2") + logger.setLevel(Level.DEBUG) + var executed = false + def function() = { + executed = true + "asdf" + executed + " hi there" } + logger.debugLazy(function()) + assert(executed) + } - "provide level name and value maps" in { - assert( - Logger.levels == Map( - Level.ALL.value -> Level.ALL, - Level.TRACE.value -> Level.TRACE, - Level.DEBUG.value -> Level.DEBUG, - Level.INFO.value -> Level.INFO, - Level.WARNING.value -> Level.WARNING, - Level.ERROR.value -> Level.ERROR, - Level.CRITICAL.value -> Level.CRITICAL, - Level.FATAL.value -> Level.FATAL, - Level.OFF.value -> Level.OFF - ) + test("Logger should make sure compiles with normal string case") { + //in some cases, if debugLazy definition was changed, the below would no longer compile + val logger = Logger.get("lazyTest3") + val executed = true + logger.debugLazy("hi there" + executed + "cool") + } + + test("Logger should provide level name and value maps") { + assert( + Logger.levels == Map( + Level.ALL.value -> Level.ALL, + Level.TRACE.value -> Level.TRACE, + Level.DEBUG.value -> Level.DEBUG, + Level.INFO.value -> Level.INFO, + Level.WARNING.value -> Level.WARNING, + Level.ERROR.value -> Level.ERROR, + Level.CRITICAL.value -> Level.CRITICAL, + Level.FATAL.value -> Level.FATAL, + Level.OFF.value -> Level.OFF ) - assert( - Logger.levelNames == Map( - "ALL" -> Level.ALL, - "TRACE" -> Level.TRACE, - "DEBUG" -> Level.DEBUG, - "INFO" -> Level.INFO, - "WARNING" -> Level.WARNING, - "ERROR" -> Level.ERROR, - "CRITICAL" -> Level.CRITICAL, - "FATAL" -> Level.FATAL, - "OFF" -> Level.OFF - ) + ) + assert( + Logger.levelNames == Map( + "ALL" -> Level.ALL, + "TRACE" -> Level.TRACE, + "DEBUG" -> Level.DEBUG, + "INFO" -> Level.INFO, + "WARNING" -> Level.WARNING, + "ERROR" -> Level.ERROR, + "CRITICAL" -> Level.CRITICAL, + "FATAL" -> Level.FATAL, + "OFF" -> Level.OFF ) - } + ) + } - "figure out package names" in { - val log1 = Logger(this.getClass) - assert(log1.name == "com.twitter.logging.LoggerTest") - } + test("Logger should figure out package names") { + val log1 = Logger(this.getClass) + assert(log1.name == "com.twitter.logging.LoggerTest") + } - "log & trace a message" in { - traceLogger(Level.INFO) - Logger.get("").info("angry duck") - mustLog("duck") - } + test("Logger should log & trace a message") { + traceLogger(Level.INFO) + Logger.get("").info("angry duck") + mustLog("duck") + } - "log & trace a throwable with a log level" in { - traceLogger(Level.WARNING) - Logger.get("").throwable(WithLogLevel(Level.WARNING), "bananas") - Logger.get("").throwable(WithLogLevel(Level.INFO), "apples") - mustLog("bananas") - mustNotLog("apples") - } + test("Logger should log & trace a throwable with a log level") { + traceLogger(Level.WARNING) + Logger.get("").throwable(WithLogLevel(Level.WARNING), "bananas") + Logger.get("").throwable(WithLogLevel(Level.INFO), "apples") + mustLog("bananas") + mustNotLog("apples") + } - "log & trace a throwable with a default log level" in { - traceLogger(Level.ERROR) - Logger.get("").throwable(new Exception(), "angry duck") - Logger.get("").throwable(new Exception(), "invisible", defaultLevel = Level.INFO) - mustLog("duck") - mustNotLog("invisible") - } + test("Logger should log & trace a throwable with a default log level") { + traceLogger(Level.ERROR) + Logger.get("").throwable(new Exception(), "angry duck") + Logger.get("").throwable(new Exception(), "invisible", defaultLevel = Level.INFO) + mustLog("duck") + mustNotLog("invisible") + } - "log & trace a message with percent signs" in { - traceLogger(Level.INFO) - Logger.get("")(Level.INFO, "%i") - Logger.get("")(Level.INFO, new Exception(), "%j") - mustLog("%i") - mustLog("%j") - } + test("Logger should log & trace a message with percent signs") { + traceLogger(Level.INFO) + Logger.get("")(Level.INFO, "%i") + Logger.get("")(Level.INFO, new Exception(), "%j") + mustLog("%i") + mustLog("%j") + } - "log a message, with timestamp" in { - before() + test("Logger should log a message, with timestamp") { + before() - Logger.clearHandlers() - myHandler = timeFrozenHandler - log.addHandler(timeFrozenHandler) - log.error("error!") - assert(parse() == List("ERR [20080329-05:53:16.722] (root): error!")) - } + Logger.clearHandlers() + myHandler = timeFrozenHandler + log.addHandler(timeFrozenHandler) + log.error("error!") + assert(parse() == List("ERR [20080329-05:53:16.722] (root): error!")) + } - "get single-threaded return the same value" in { - val loggerFirst = Logger.get("getTest") - assert(loggerFirst != null) + test("Logger should get single-threaded return the same value") { + val loggerFirst = Logger.get("getTest") + assert(loggerFirst != null) - val loggerSecond = Logger.get("getTest") - assert(loggerSecond == loggerFirst) + val loggerSecond = Logger.get("getTest") + assert(loggerSecond == loggerFirst) + } + + test("Logger should get multi-threaded return the same value") { + val numThreads = 10 + val latch = new CountDownLatch(1) + + // queue up the workers + val executorService = Executors.newFixedThreadPool(numThreads) + val futureResults = new mutable.ListBuffer[Future[Logger]] + for (i <- 0.until(numThreads)) { + val future = executorService.submit(new Callable[Logger]() { + def call(): Logger = { + latch.await(10, TimeUnit.SECONDS) + return Logger.get("concurrencyTest") + } + }) + futureResults += future } + executorService.shutdown + // let them rip, and then wait for em to finish + latch.countDown + assert(executorService.awaitTermination(10, TimeUnit.SECONDS) == true) + + // now make sure they are all the same reference + val expected = futureResults(0).get + for (i <- 1.until(numThreads)) { + val result = futureResults(i).get + assert(result == expected) + } + } - "get multi-threaded return the same value" in { - val numThreads = 10 - val latch = new CountDownLatch(1) - - // queue up the workers - val executorService = Executors.newFixedThreadPool(numThreads) - val futureResults = new mutable.ListBuffer[Future[Logger]] - for (i <- 0.until(numThreads)) { - val future = executorService.submit(new Callable[Logger]() { - def call(): Logger = { - latch.await(10, TimeUnit.SECONDS) - return Logger.get("concurrencyTest") - } - }) - futureResults += future - } - executorService.shutdown - // let them rip, and then wait for em to finish - latch.countDown - assert(executorService.awaitTermination(10, TimeUnit.SECONDS) == true) - - // now make sure they are all the same reference - val expected = futureResults(0).get - for (i <- 1.until(numThreads)) { - val result = futureResults(i).get - assert(result == expected) - } + test( + "Logger should withLoggers applies logger factories, executes a block, and then applies original factories") { + val initialFactories = List(LoggerFactory(node = "", level = Some(Level.DEBUG))) + val otherFactories = List(LoggerFactory(node = "", level = Some(Level.INFO))) + Logger.configure(initialFactories) + + assert(Logger.get("").getLevel() == Level.DEBUG) + Logger.withLoggers(otherFactories) { + assert(Logger.get("").getLevel() == Level.INFO) } + assert(Logger.get("").getLevel() == Level.DEBUG) + } - "withLoggers applies logger factories, executes a block, and then applies original factories" in { - val initialFactories = List(LoggerFactory(node = "", level = Some(Level.DEBUG))) - val otherFactories = List(LoggerFactory(node = "", level = Some(Level.INFO))) - Logger.configure(initialFactories) + test("Logger configure logging should") { + def before(): Unit = { + Logger.clearHandlers() + } + } - assert(Logger.get("").getLevel() == Level.DEBUG) - Logger.withLoggers(otherFactories) { - assert(Logger.get("").getLevel() == Level.INFO) - } - assert(Logger.get("").getLevel() == Level.DEBUG) + test("Logger configure logging with file handler") { + val separator = java.io.File.pathSeparator + withTempFolder { + val log: Logger = LoggerFactory( + node = "com.twitter", + level = Some(Level.DEBUG), + handlers = FileHandler( + filename = folderName + separator + "test.log", + rollPolicy = Policy.Never, + append = false, + level = Some(Level.INFO), + formatter = new Formatter( + useFullPackageNames = true, + truncateAt = 1024, + prefix = "%s %s" + ) + ) :: Nil + ).apply() + + assert(log.getLevel() == Level.DEBUG) + assert(log.getHandlers().length == 1) + val handler = log.getHandlers()(0).asInstanceOf[FileHandler] + val fileName = folderName + separator + "test.log" + assert(handler.filename == fileName) + assert(handler.append == false) + assert(handler.getLevel() == Level.INFO) + val formatter = handler.formatter + assert( + formatter.formatPrefix(javalog.Level.WARNING, "10:55", "hello") == "WARNING 10:55 hello" + ) + assert(log.name == "com.twitter") + assert(formatter.truncateAt == 1024) + assert(formatter.useFullPackageNames == true) } + } - "configure logging" should { - def before(): Unit = { - Logger.clearHandlers() - } + test("Logger configure logging with syslog handler") { + withTempFolder { + val log: Logger = LoggerFactory( + node = "com.twitter", + handlers = SyslogHandler( + formatter = new SyslogFormatter( + serverName = Some("elmo"), + priority = 128 + ), + server = "localhost", + port = 212 + ) :: Nil + ).apply() + + assert(log.getHandlers().length == 1) + val h = log.getHandlers()(0).asInstanceOf[SyslogHandler] + assert(h.dest.asInstanceOf[InetSocketAddress].getHostName == "localhost") + assert(h.dest.asInstanceOf[InetSocketAddress].getPort == 212) + val formatter = h.formatter.asInstanceOf[SyslogFormatter] + assert(formatter.serverName == Some("elmo")) + assert(formatter.priority == 128) + } + } - "file handler" in { - val separator = java.io.File.pathSeparator - withTempFolder { - val log: Logger = LoggerFactory( - node = "com.twitter", - level = Some(Level.DEBUG), - handlers = FileHandler( - filename = folderName + separator + "test.log", - rollPolicy = Policy.Never, - append = false, - level = Some(Level.INFO), - formatter = new Formatter( - useFullPackageNames = true, - truncateAt = 1024, - prefix = "%s %s" - ) - ) :: Nil - ).apply() - - assert(log.getLevel() == Level.DEBUG) - assert(log.getHandlers().length == 1) - val handler = log.getHandlers()(0).asInstanceOf[FileHandler] - val fileName = folderName + separator + "test.log" - assert(handler.filename == fileName) - assert(handler.append == false) - assert(handler.getLevel() == Level.INFO) - val formatter = handler.formatter - assert( - formatter.formatPrefix(javalog.Level.WARNING, "10:55", "hello") == "WARNING 10:55 hello" + test("Logger configure logging with complex config") { + withTempFolder { + val factories = LoggerFactory( + level = Some(Level.INFO), + handlers = ThrottledHandler( + duration = 60.seconds, + maxToDisplay = 10, + handler = FileHandler( + filename = folderName + "/production.log", + rollPolicy = Policy.SigHup, + formatter = new Formatter( + truncateStackTracesAt = 100 + ) ) - assert(log.name == "com.twitter") - assert(formatter.truncateAt == 1024) - assert(formatter.useFullPackageNames == true) - } + ) :: Nil + ) :: LoggerFactory( + node = "w3c", + level = Some(Level.OFF), + useParents = false + ) :: LoggerFactory( + node = "bad_jobs", + level = Some(Level.INFO), + useParents = false, + handlers = FileHandler( + filename = folderName + "/bad_jobs.log", + rollPolicy = Policy.Never + ) :: Nil + ) :: Nil + + Logger.configure(factories) + assert(Logger.get("").getLevel() == Level.INFO) + assert(Logger.get("w3c").getLevel() == Level.OFF) + assert(Logger.get("bad_jobs").getLevel() == Level.INFO) + try { + Logger.get("").getHandlers()(0).asInstanceOf[ThrottledHandler] + } catch { + case _: ClassCastException => fail("not a ThrottledHandler") } - - "syslog handler" in { - withTempFolder { - val log: Logger = LoggerFactory( - node = "com.twitter", - handlers = SyslogHandler( - formatter = new SyslogFormatter( - serverName = Some("elmo"), - priority = 128 - ), - server = "localhost", - port = 212 - ) :: Nil - ).apply() - - assert(log.getHandlers().length == 1) - val h = log.getHandlers()(0).asInstanceOf[SyslogHandler] - assert(h.dest.asInstanceOf[InetSocketAddress].getHostName == "localhost") - assert(h.dest.asInstanceOf[InetSocketAddress].getPort == 212) - val formatter = h.formatter.asInstanceOf[SyslogFormatter] - assert(formatter.serverName == Some("elmo")) - assert(formatter.priority == 128) - } + try { + Logger + .get("") + .getHandlers()(0) + .asInstanceOf[ThrottledHandler] + .handler + .asInstanceOf[FileHandler] + } catch { + case _: ClassCastException => fail("not a FileHandler") } - - "complex config" in { - withTempFolder { - val factories = LoggerFactory( - level = Some(Level.INFO), - handlers = ThrottledHandler( - duration = 60.seconds, - maxToDisplay = 10, - handler = FileHandler( - filename = folderName + "/production.log", - rollPolicy = Policy.SigHup, - formatter = new Formatter( - truncateStackTracesAt = 100 - ) - ) - ) :: Nil - ) :: LoggerFactory( - node = "w3c", - level = Some(Level.OFF), - useParents = false - ) :: LoggerFactory( - node = "bad_jobs", - level = Some(Level.INFO), - useParents = false, - handlers = FileHandler( - filename = folderName + "/bad_jobs.log", - rollPolicy = Policy.Never - ) :: Nil - ) :: Nil - - Logger.configure(factories) - assert(Logger.get("").getLevel() == Level.INFO) - assert(Logger.get("w3c").getLevel() == Level.OFF) - assert(Logger.get("bad_jobs").getLevel() == Level.INFO) - try { - Logger.get("").getHandlers()(0).asInstanceOf[ThrottledHandler] - } catch { - case _: ClassCastException => fail("not a ThrottledHandler") - } - try { - Logger - .get("") - .getHandlers()(0) - .asInstanceOf[ThrottledHandler] - .handler - .asInstanceOf[FileHandler] - } catch { - case _: ClassCastException => fail("not a FileHandler") - } - assert(Logger.get("w3c").getHandlers().size == 0) - try { - Logger.get("bad_jobs").getHandlers()(0).asInstanceOf[FileHandler] - } catch { - case _: ClassCastException => fail("not a FileHandler") - } - - } + assert(Logger.get("w3c").getHandlers().size == 0) + try { + Logger.get("bad_jobs").getHandlers()(0).asInstanceOf[FileHandler] + } catch { + case _: ClassCastException => fail("not a FileHandler") } - } - "java logging" should { - val logger = javalog.Logger.getLogger("") - - def before() = { - traceLogger(Level.INFO) - } + } + } - "single arg calls" in { - before() - logger.log(javalog.Level.INFO, "V1={0}", "A") - mustLog("V1=A") - } + test("java logging single arg calls") { + val logger = javalog.Logger.getLogger("") + traceLogger(Level.INFO) + logger.log(javalog.Level.INFO, "V1={0}", "A") + mustLog("V1=A") + } - "varargs calls" in { - before() - logger.log(javalog.Level.INFO, "V1={0}, V2={1}", Array[AnyRef]("A", "B")) - mustLog("V1=A, V2=B") - } + test("java logging varargs calls") { + val logger = javalog.Logger.getLogger("") + traceLogger(Level.INFO) + logger.log(javalog.Level.INFO, "V1={0}, V2={1}", Array[AnyRef]("A", "B")) + mustLog("V1=A, V2=B") + } - "invalid message format" in { - before() - logger.log(javalog.Level.INFO, "V1=%s", "A") - mustLog("V1=%s") // %s notation is not known in java MessageFormat - } + test("java logging invalid message format") { + val logger = javalog.Logger.getLogger("") + traceLogger(Level.INFO) + logger.log(javalog.Level.INFO, "V1=%s", "A") + mustLog("V1=%s") // %s notation is not known)java MessageFormat + } - // logging in scala uses the %s format and not the Java MessageFormat - "compare scala logging format" in { - before() - Logger.get("").info("V1{0}=%s", "A") - mustLog("V1{0}=A") - } - } + // logging in scala uses the %s format and not the Java MessageFormat + test("java logging compare scala logging format") { + val logger = javalog.Logger.getLogger("") + traceLogger(Level.INFO) + Logger.get("").info("V1{0}=%s", "A") + mustLog("V1{0}=A") } - "Levels.fromJava" should { - "return a corresponding level if it exists" in { - assert(Level.fromJava(javalog.Level.OFF) == Some(Level.OFF)) - assert(Level.fromJava(javalog.Level.SEVERE) == Some(Level.FATAL)) - // No corresponding level for ERROR and CRITICAL in javalog. - assert(Level.fromJava(javalog.Level.WARNING) == Some(Level.WARNING)) - assert(Level.fromJava(javalog.Level.INFO) == Some(Level.INFO)) - assert(Level.fromJava(javalog.Level.FINE) == Some(Level.DEBUG)) - assert(Level.fromJava(javalog.Level.FINER) == Some(Level.TRACE)) - assert(Level.fromJava(javalog.Level.ALL) == Some(Level.ALL)) - } + test("Levels.fromJava should return a corresponding level if it exists") { + assert(Level.fromJava(javalog.Level.OFF) == Some(Level.OFF)) + assert(Level.fromJava(javalog.Level.SEVERE) == Some(Level.FATAL)) + // No corresponding level for ERROR and CRITICAL in javalog. + assert(Level.fromJava(javalog.Level.WARNING) == Some(Level.WARNING)) + assert(Level.fromJava(javalog.Level.INFO) == Some(Level.INFO)) + assert(Level.fromJava(javalog.Level.FINE) == Some(Level.DEBUG)) + assert(Level.fromJava(javalog.Level.FINER) == Some(Level.TRACE)) + assert(Level.fromJava(javalog.Level.ALL) == Some(Level.ALL)) + } - "return None for non corresponding levels" in { - assert(Level.fromJava(javalog.Level.CONFIG) == None) - assert(Level.fromJava(javalog.Level.FINEST) == None) - } + test("Levels.fromJava should return None for non corresponding levels") { + assert(Level.fromJava(javalog.Level.CONFIG) == None) + assert(Level.fromJava(javalog.Level.FINEST) == None) } - "Levels.parse" should { - "return Levels for valid names" in { - assert(Level.parse("INFO") == Some(Level.INFO)) - assert(Level.parse("TRACE") == Some(Level.TRACE)) - } + test("Levels.parse should return Levels for valid names") { + assert(Level.parse("INFO") == Some(Level.INFO)) + assert(Level.parse("TRACE") == Some(Level.TRACE)) + } - "return None for invalid names" in { - assert(Level.parse("") == None) - assert(Level.parse("FOO") == None) - } + test("Levels.parse should return None for invalid names") { + assert(Level.parse("") == None) + assert(Level.parse("FOO") == None) } } diff --git a/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala index 7763c147f0..77e8326f32 100644 --- a/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/QueueingHandlerTest.scala @@ -18,10 +18,11 @@ package com.twitter.logging import com.twitter.util.Local import java.util.{logging => javalog} -import org.scalatest.concurrent.{IntegrationPatience, Eventually} -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.concurrent.IntegrationPatience +import org.scalatest.concurrent.Eventually +import org.scalatest.funsuite.AnyFunSuite -class QueueingHandlerTest extends AnyWordSpec with Eventually with IntegrationPatience { +class QueueingHandlerTest extends AnyFunSuite with Eventually with IntegrationPatience { class MockHandler extends Handler(BareFormatter, None) { def publish(record: javalog.LogRecord) = () @@ -37,159 +38,157 @@ class QueueingHandlerTest extends AnyWordSpec with Eventually with IntegrationPa logger } - "QueueingHandler" should { - "publish" in { - val logger = freshLogger() - val stringHandler = new StringHandler(BareFormatter, Some(Logger.INFO)) - val queueHandler = new QueueingHandler(stringHandler) - logger.addHandler(queueHandler) + test("QueueingHandler should publish") { + val logger = freshLogger() + val stringHandler = new StringHandler(BareFormatter, Some(Logger.INFO)) + val queueHandler = new QueueingHandler(stringHandler) + logger.addHandler(queueHandler) - logger.warning("oh noes!") - eventually { - // let thread log - assert(stringHandler.get == "oh noes!\n") - } + logger.warning("oh noes!") + eventually { + // let thread log + assert(stringHandler.get == "oh noes!\n") } + } - "publish with local context" in { - val logger = freshLogger() - val local = new Local[String] - val formatter = new Formatter { - override def format(record: javalog.LogRecord) = - local().getOrElse("MISSING!") + ":" + formatText(record) + lineTerminator - } - val stringHandler = new StringHandler(formatter, Some(Logger.INFO)) - logger.addHandler(new QueueingHandler(stringHandler)) + test("QueueingHandler should publish with local context") { + val logger = freshLogger() + val local = new Local[String] + val formatter = new Formatter { + override def format(record: javalog.LogRecord) = + local().getOrElse("MISSING!") + ":" + formatText(record) + lineTerminator + } + val stringHandler = new StringHandler(formatter, Some(Logger.INFO)) + logger.addHandler(new QueueingHandler(stringHandler)) - local() = "foo" - logger.warning("oh noes!") + local() = "foo" + logger.warning("oh noes!") - eventually { - assert(stringHandler.get == "foo:oh noes!\n") - } + eventually { + assert(stringHandler.get == "foo:oh noes!\n") } + } - "publish, drop on overflow" in { - val logger = freshLogger() - val blockingHandler = new MockHandler { - override def publish(record: javalog.LogRecord): Unit = { - Thread.sleep(100000) - } + test("QueueingHandler should publish, drop on overflow") { + val logger = freshLogger() + val blockingHandler = new MockHandler { + override def publish(record: javalog.LogRecord): Unit = { + Thread.sleep(100000) } - @volatile var droppedCount = 0 + } + @volatile var droppedCount = 0 - val queueHandler = new QueueingHandler(blockingHandler, 1) { - override protected def onOverflow(record: javalog.LogRecord): Unit = { - droppedCount += 1 - } + val queueHandler = new QueueingHandler(blockingHandler, 1) { + override protected def onOverflow(record: javalog.LogRecord): Unit = { + droppedCount += 1 } - logger.addHandler(queueHandler) + } + logger.addHandler(queueHandler) - logger.warning("1") - logger.warning("2") - logger.warning("3") + logger.warning("1") + logger.warning("2") + logger.warning("3") - eventually { - // let thread log and block - assert(droppedCount >= 1) // either 1 or 2, depending on race - } + eventually { + // let thread log and block + assert(droppedCount >= 1) // either 1 or 2, depending on race } + } - "flush" in { - val logger = freshLogger() - var wasFlushed = false - val handler = new MockHandler { - override def flush(): Unit = { wasFlushed = true } - } - val queueHandler = new QueueingHandler(handler, 1) - logger.addHandler(queueHandler) - - // smoke test: thread might write it, flush might write it - logger.warning("oh noes!") - queueHandler.flush() - assert(wasFlushed == true) + test("QueueingHandler should flush") { + val logger = freshLogger() + var wasFlushed = false + val handler = new MockHandler { + override def flush(): Unit = { wasFlushed = true } } + val queueHandler = new QueueingHandler(handler, 1) + logger.addHandler(queueHandler) - "close" in { - val logger = freshLogger() - var wasClosed = false - val handler = new MockHandler { - override def close(): Unit = { wasClosed = true } - } - val queueHandler = new QueueingHandler(handler) - logger.addHandler(queueHandler) + // smoke test: thread might write it, flush might write it + logger.warning("oh noes!") + queueHandler.flush() + assert(wasFlushed == true) + } - logger.warning("oh noes!") - eventually { - // let thread log - queueHandler.close() - assert(wasClosed == true) - } + test("QueueingHandler should close") { + val logger = freshLogger() + var wasClosed = false + val handler = new MockHandler { + override def close(): Unit = { wasClosed = true } + } + val queueHandler = new QueueingHandler(handler) + logger.addHandler(queueHandler) + + logger.warning("oh noes!") + eventually { + // let thread log + queueHandler.close() + assert(wasClosed == true) } + } - "handle exceptions in the underlying handler" in { - val logger = freshLogger() - @volatile var mustError = true - @volatile var didLog = false - val handler = new MockHandler { - override def publish(record: javalog.LogRecord): Unit = { - if (mustError) { - mustError = false - throw new Exception("Unable to log for whatever reason.") - } else { - didLog = true - } + test("QueueingHandler should handle exceptions in the underlying handler") { + val logger = freshLogger() + @volatile var mustError = true + @volatile var didLog = false + val handler = new MockHandler { + override def publish(record: javalog.LogRecord): Unit = { + if (mustError) { + mustError = false + throw new Exception("Unable to log for whatever reason.") + } else { + didLog = true } } + } - val queueHandler = new QueueingHandler(handler) - logger.addHandler(queueHandler) + val queueHandler = new QueueingHandler(handler) + logger.addHandler(queueHandler) - logger.info("fizz") - logger.info("buzz") + logger.info("fizz") + logger.info("buzz") - eventually { - // let thread log - assert(didLog == true) - } + eventually { + // let thread log + assert(didLog == true) } + } - "forward formatter to the underlying handler" in { - val handler = new MockHandler {} + test("QueueingHandler should forward formatter to the underlying handler") { + val handler = new MockHandler {} - val queueHandler = new QueueingHandler(handler) - val formatter = new Formatter() + val queueHandler = new QueueingHandler(handler) + val formatter = new Formatter() - queueHandler.setFormatter(formatter) + queueHandler.setFormatter(formatter) - assert(handler.getFormatter eq formatter) - } + assert(handler.getFormatter eq formatter) + } - "obey flag to force or not force class and method name inference" in { - val formatter = new Formatter() { - override def format(record: javalog.LogRecord) = { - val prefix = - if (record.getSourceClassName != null) record.getSourceClassName + ": " - else "" + test("QueueingHandler should obey flag to force or not force class and method name inference") { + val formatter = new Formatter() { + override def format(record: javalog.LogRecord) = { + val prefix = + if (record.getSourceClassName != null) record.getSourceClassName + ": " + else "" - prefix + formatText(record) + lineTerminator - } + prefix + formatText(record) + lineTerminator } - for (infer <- Seq(true, false)) { - val logger = freshLogger() - val stringHandler = new StringHandler(formatter, Some(Logger.INFO)) - val queueHandler = new QueueingHandler(stringHandler, Int.MaxValue, infer) - logger.addHandler(queueHandler) - val helper = new QueueingHandlerTestHelper() - helper.logSomething(logger) - - val expectedMessagePrefix = if (infer) helper.getClass.getName + ": " else "" - val expectedMessage = expectedMessagePrefix + helper.message + "\n" - - eventually { - // let thread log - assert(stringHandler.get == expectedMessage) - } + } + for (infer <- Seq(true, false)) { + val logger = freshLogger() + val stringHandler = new StringHandler(formatter, Some(Logger.INFO)) + val queueHandler = new QueueingHandler(stringHandler, Int.MaxValue, infer) + logger.addHandler(queueHandler) + val helper = new QueueingHandlerTestHelper() + helper.logSomething(logger) + + val expectedMessagePrefix = if (infer) helper.getClass.getName + ": " else "" + val expectedMessage = expectedMessagePrefix + helper.message + "\n" + + eventually { + // let thread log + assert(stringHandler.get == expectedMessage) } } } diff --git a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala index 06ca8e2781..655ee18b20 100644 --- a/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/SyslogHandlerTest.scala @@ -20,9 +20,9 @@ import java.net.DatagramPacket import java.net.DatagramSocket import java.util.{logging => javalog} -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class SyslogHandlerTest extends AnyWordSpec { +class SyslogHandlerTest extends AnyFunSuite { val record1 = new javalog.LogRecord(Level.FATAL, "fatal message!") record1.setLoggerName("net.lag.whiskey.Train") record1.setMillis(1206769996722L) @@ -30,90 +30,88 @@ class SyslogHandlerTest extends AnyWordSpec { record2.setLoggerName("net.lag.whiskey.Train") record2.setMillis(1206769996722L) - "SyslogHandler" should { - "write syslog entries" in { - // start up new syslog listener - val serverSocket = new DatagramSocket - val serverPort = serverSocket.getLocalPort + test("SyslogHandler should write syslog entries") { + // start up new syslog listener + val serverSocket = new DatagramSocket + val serverPort = serverSocket.getLocalPort - var syslog = SyslogHandler( - port = serverPort, - formatter = new SyslogFormatter( - timezone = Some("UTC"), - hostname = "raccoon.local" - ) - ).apply() - syslog.publish(record1) - syslog.publish(record2) - - SyslogFuture.sync() - val p = new DatagramPacket(new Array[Byte](1024), 1024) - serverSocket.receive(p) - assert( - new String( - p.getData, - 0, - p.getLength) == "<9>2008-03-29T05:53:16 raccoon.local whiskey: fatal message!" - ) - serverSocket.receive(p) - assert( - new String( - p.getData, - 0, - p.getLength) == "<11>2008-03-29T05:53:16 raccoon.local whiskey: error message!" + var syslog = SyslogHandler( + port = serverPort, + formatter = new SyslogFormatter( + timezone = Some("UTC"), + hostname = "raccoon.local" ) - } + ).apply() + syslog.publish(record1) + syslog.publish(record2) - "with server name" in { - // start up new syslog listener - val serverSocket = new DatagramSocket - val serverPort = serverSocket.getLocalPort + SyslogFuture.sync() + val p = new DatagramPacket(new Array[Byte](1024), 1024) + serverSocket.receive(p) + assert( + new String( + p.getData, + 0, + p.getLength) == "<9>2008-03-29T05:53:16 raccoon.local whiskey: fatal message!" + ) + serverSocket.receive(p) + assert( + new String( + p.getData, + 0, + p.getLength) == "<11>2008-03-29T05:53:16 raccoon.local whiskey: error message!" + ) + } - var syslog = SyslogHandler( - port = serverPort, - formatter = new SyslogFormatter( - serverName = Some("pingd"), - timezone = Some("UTC"), - hostname = "raccoon.local" - ) - ).apply() - syslog.publish(record1) + test("SyslogHandler with with server name") { + // start up new syslog listener + val serverSocket = new DatagramSocket + val serverPort = serverSocket.getLocalPort - SyslogFuture.sync() - val p = new DatagramPacket(new Array[Byte](1024), 1024) - serverSocket.receive(p) - assert( - new String( - p.getData, - 0, - p.getLength) == "<9>2008-03-29T05:53:16 raccoon.local [pingd] whiskey: fatal message!" + var syslog = SyslogHandler( + port = serverPort, + formatter = new SyslogFormatter( + serverName = Some("pingd"), + timezone = Some("UTC"), + hostname = "raccoon.local" ) - } + ).apply() + syslog.publish(record1) - "with BSD time format" in { - // start up new syslog listener - val serverSocket = new DatagramSocket - val serverPort = serverSocket.getLocalPort + SyslogFuture.sync() + val p = new DatagramPacket(new Array[Byte](1024), 1024) + serverSocket.receive(p) + assert( + new String( + p.getData, + 0, + p.getLength) == "<9>2008-03-29T05:53:16 raccoon.local [pingd] whiskey: fatal message!" + ) + } - var syslog = SyslogHandler( - port = serverPort, - formatter = new SyslogFormatter( - useIsoDateFormat = false, - timezone = Some("UTC"), - hostname = "raccoon.local" - ) - ).apply() - syslog.publish(record1) + test("SyslogHandler with BSD time format") { + // start up new syslog listener + val serverSocket = new DatagramSocket + val serverPort = serverSocket.getLocalPort - SyslogFuture.sync() - val p = new DatagramPacket(new Array[Byte](1024), 1024) - serverSocket.receive(p) - assert( - new String( - p.getData, - 0, - p.getLength) == "<9>Mar 29 05:53:16 raccoon.local whiskey: fatal message!" + var syslog = SyslogHandler( + port = serverPort, + formatter = new SyslogFormatter( + useIsoDateFormat = false, + timezone = Some("UTC"), + hostname = "raccoon.local" ) - } + ).apply() + syslog.publish(record1) + + SyslogFuture.sync() + val p = new DatagramPacket(new Array[Byte](1024), 1024) + serverSocket.receive(p) + assert( + new String( + p.getData, + 0, + p.getLength) == "<9>Mar 29 05:53:16 raccoon.local whiskey: fatal message!" + ) } } diff --git a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala index 166b878e40..b5fc3dfde7 100644 --- a/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala +++ b/util-logging/src/test/scala/com/twitter/logging/ThrottledHandlerTest.scala @@ -20,75 +20,73 @@ import com.twitter.conversions.DurationOps._ import com.twitter.io.TempFolder import com.twitter.util.Time import org.scalatest.BeforeAndAfter -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class ThrottledHandlerTest extends AnyWordSpec with BeforeAndAfter with TempFolder { +class ThrottledHandlerTest extends AnyFunSuite with BeforeAndAfter with TempFolder { private var handler: StringHandler = _ - "ThrottledHandler" should { - before { - Logger.clearHandlers() - handler = new StringHandler(BareFormatter, None) - } + before { + Logger.clearHandlers() + handler = new StringHandler(BareFormatter, None) + } - after { - Logger.clearHandlers() - } + after { + Logger.clearHandlers() + } - "throttle keyed log messages" in { - val log = Logger() - val throttledLog = new ThrottledHandler(handler, 1.second, 3) - Time.withCurrentTimeFrozen { timeCtrl => - log.addHandler(throttledLog) + test("ThrottledHandler should throttle keyed log messages") { + val log = Logger() + val throttledLog = new ThrottledHandler(handler, 1.second, 3) + Time.withCurrentTimeFrozen { timeCtrl => + log.addHandler(throttledLog) - log.error("apple: %s", "help!") - log.error("apple: %s", "help 2!") - log.error("orange: %s", "orange!") - log.error("orange: %s", "orange!") - log.error("apple: %s", "help 3!") - log.error("apple: %s", "help 4!") - log.error("apple: %s", "help 5!") - timeCtrl.advance(2.seconds) - log.error("apple: %s", "done.") + log.error("apple: %s", "help!") + log.error("apple: %s", "help 2!") + log.error("orange: %s", "orange!") + log.error("orange: %s", "orange!") + log.error("apple: %s", "help 3!") + log.error("apple: %s", "help 4!") + log.error("apple: %s", "help 5!") + timeCtrl.advance(2.seconds) + log.error("apple: %s", "done.") - assert( - handler.get.split("\n").toList == List( - "apple: help!", - "apple: help 2!", - "orange: orange!", - "orange: orange!", - "apple: help 3!", - "(swallowed 2 repeating messages)", - "apple: done." - ) + assert( + handler.get.split("\n").toList == List( + "apple: help!", + "apple: help 2!", + "orange: orange!", + "orange: orange!", + "apple: help 3!", + "(swallowed 2 repeating messages)", + "apple: done." ) - } + ) } + } - "log the summary even if nothing more is logged with that name" in { - Time.withCurrentTimeFrozen { time => - val log = Logger() - val throttledLog = new ThrottledHandler(handler, 1.second, 3) - log.addHandler(throttledLog) - log.error("apple: %s", "help!") - log.error("apple: %s", "help!") - log.error("apple: %s", "help!") - log.error("apple: %s", "help!") - log.error("apple: %s", "help!") + test("ThrottledHandler should log the summary even if nothing more is logged with that name") { + Time.withCurrentTimeFrozen { time => + val log = Logger() + val throttledLog = new ThrottledHandler(handler, 1.second, 3) + log.addHandler(throttledLog) + log.error("apple: %s", "help!") + log.error("apple: %s", "help!") + log.error("apple: %s", "help!") + log.error("apple: %s", "help!") + log.error("apple: %s", "help!") - time.advance(2.seconds) - log.error("hello.") + time.advance(2.seconds) + log.error("hello.") - assert( - handler.get.split("\n").toList == List( - "apple: help!", - "apple: help!", - "apple: help!", - "(swallowed 2 repeating messages)", - "hello." - ) + assert( + handler.get.split("\n").toList == List( + "apple: help!", + "apple: help!", + "apple: help!", + "(swallowed 2 repeating messages)", + "hello." ) - } + ) } } } From c91c2b9566a91326fdaccb3ba4700470fb406912 Mon Sep 17 00:00:00 2001 From: Joy Bestourous Date: Fri, 7 Oct 2022 14:23:20 +0000 Subject: [PATCH 733/796] [util-codec, util-test, util-thrift]: migrate tests off of WordSpec Problem: Our recommendation for testing is to use AnyFunSuite, but our own testing uses AnyWordSpec Solution: Migrate JIRA Issues: CSL-9463 Differential Revision: https://phabricator.twitter.biz/D991143 --- .../com/twitter/util/StringEncoderTest.scala | 46 +++++++++---------- .../com/twitter/logging/TestLogging.scala | 4 +- .../twitter/util/ThriftSerializerTest.scala | 22 ++++----- 3 files changed, 33 insertions(+), 39 deletions(-) diff --git a/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala b/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala index 774155b0d0..df60791a82 100644 --- a/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala +++ b/util-codec/src/test/scala/com/twitter/util/StringEncoderTest.scala @@ -16,63 +16,59 @@ package com.twitter.util * limitations under the License. */ -import java.lang.StringBuilder +import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.wordspec.AnyWordSpec - -class StringEncoderTest extends AnyWordSpec { +class StringEncoderTest extends AnyFunSuite { val longString = "A string that is really really really really really really long and has more than 76 characters" val result = "QSBzdHJpbmcgdGhhdCBpcyByZWFsbHkgcmVhbGx5IHJlYWxseSByZWFsbHkgcmVhbGx5IHJlYWxseSBsb25nIGFuZCBoYXMgbW9yZSB0aGFuIDc2IGNoYXJhY3RlcnM=" - "strip new lines" in { + test("strip new lines") { assert(Base64StringEncoder.encode(longString.getBytes) == result) } - "decode value with stripped new lines" in { + test("decode value with stripped new lines") { assert(new String(Base64StringEncoder.decode(result)) == longString) } } -class Base64StringEncoderTest extends AnyWordSpec { +class Base64StringEncoderTest extends AnyFunSuite { val urlUnsafeBytes = Array(-1.toByte, -32.toByte) val resultUnsafe = "/+A" val resultSafe = "_-A" - "encode / as _ and encode + as - to maintain url safe strings" in { + test("encode / as _ and encode + as - to maintain url safe strings") { assert(Base64UrlSafeStringEncoder.encode(urlUnsafeBytes) == resultSafe) } - "decode url-safe strings" in { + test("decode url-safe strings") { assert(Base64UrlSafeStringEncoder.decode(resultSafe) === urlUnsafeBytes) } - "decode url-unsafe strings" in { + test("decode url-unsafe strings") { intercept[IllegalArgumentException] { Base64UrlSafeStringEncoder.decode(resultUnsafe) } } } -class GZIPStringEncoderTest extends AnyWordSpec { - "a gzip string encoder" should { +class GZIPStringEncoderTest extends AnyFunSuite { + test("a gzip string encoder should properly encode and decode strings") { val gse = new GZIPStringEncoder {} - "properly encode and decode strings" in { - def testCodec(str: String): Unit = { - assert(str == gse.decodeString(gse.encodeString(str))) - } + def testCodec(str: String): Unit = { + assert(str == gse.decodeString(gse.encodeString(str))) + } - testCodec("a") - testCodec("\n\t\n\t\n\n\n\n\t\n\nt\n\t\n\t\n\tn\t\nt\nt\nt\nt\nt\nt\tn\nt\nt\n\t\nt\n") - testCodec("aosnetuhsaontehusaonethsoantehusaonethusonethusnaotehu") + testCodec("a") + testCodec("\n\t\n\t\n\n\n\n\t\n\nt\n\t\n\t\n\tn\t\nt\nt\nt\nt\nt\nt\tn\nt\nt\n\t\nt\n") + testCodec("aosnetuhsaontehusaonethsoantehusaonethusonethusnaotehu") - // build a huge string - val sb = new StringBuilder - for (_ <- 1 to 10000) { - sb.append("oasnuthoesntihosnteidosentidosentauhsnoetidosentihsoneitdsnuthsin\n") - } - testCodec(sb.toString) + // build a huge string + val sb = new StringBuilder + for (_ <- 1 to 10000) { + sb.append("oasnuthoesntihosnteidosentidosentauhsnoetidosentihsoneitdsnuthsin\n") } + testCodec(sb.toString) } } diff --git a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala index 53cb73414d..7a741bc1d8 100644 --- a/util-test/src/main/scala/com/twitter/logging/TestLogging.scala +++ b/util-test/src/main/scala/com/twitter/logging/TestLogging.scala @@ -19,13 +19,13 @@ package com.twitter.logging import java.util.{logging => jlogging} import org.scalatest.BeforeAndAfter -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite /** * Specify logging during unit tests via system property, defaulting to FATAL only. */ @deprecated("Prefer using slf4j for logging", "2019-11-20") -trait TestLogging extends BeforeAndAfter { self: AnyWordSpec => +trait TestLogging extends BeforeAndAfter { self: AnyFunSuite => val logLevel: Level = Logger.levelNames(Option[String](System.getenv("log")).getOrElse("FATAL").toUpperCase) diff --git a/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala b/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala index 31fabad810..ae1fc63840 100644 --- a/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala +++ b/util-thrift/src/test/scala/com/twitter/util/ThriftSerializerTest.scala @@ -16,9 +16,9 @@ package com.twitter.util -import org.scalatest.wordspec.AnyWordSpec +import org.scalatest.funsuite.AnyFunSuite -class ThriftSerializerTest extends AnyWordSpec { +class ThriftSerializerTest extends AnyFunSuite { val aString = "me gustan los tacos y los burritos" val aNumber = 42 val original = new TestThriftStructure(aString, aNumber) @@ -45,17 +45,15 @@ class ThriftSerializerTest extends AnyWordSpec { true } - "ThriftSerializer" should { - "encode and decode json" in { - testSerializer(new JsonThriftSerializer, Some(json)) - } + test("ThriftSerializer should encode and decode json") { + testSerializer(new JsonThriftSerializer, Some(json)) + } - "encode and decode binary" in { - testSerializer(new BinaryThriftSerializer, encodedBinary) - } + test("ThriftSerializer should encode and decode binary") { + testSerializer(new BinaryThriftSerializer, encodedBinary) + } - "encode and decode compact" in { - testSerializer(new CompactThriftSerializer, encodedCompact) - } + test("ThriftSerializer should encode and decode compact") { + testSerializer(new CompactThriftSerializer, encodedCompact) } } From d852d5f02d128b233486d8727bf8a093a48d0ab4 Mon Sep 17 00:00:00 2001 From: Eitan Adler Date: Fri, 7 Oct 2022 14:25:04 +0000 Subject: [PATCH 734/796] [mockito] bump two more finagle targets to newer mockito; fix sbt build Problem also, finagle is built using both sbt and bazel. The sbt build has less individual control about which dependencies to use. Solution upgrade the remainder of the finagle targets but mark a few tests as `ignored` since they do not pass with newer mockito. bump sbt mockito version to match version used by bazel JIRA Issues: CSL-12361, CSL-12168, DEAD-3 Differential Revision: https://phabricator.twitter.biz/D991504 --- .../main/scala/com/twitter/util/testing/BUILD | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/util-test/src/main/scala/com/twitter/util/testing/BUILD b/util-test/src/main/scala/com/twitter/util/testing/BUILD index 7c7919813c..efd4cb9917 100644 --- a/util-test/src/main/scala/com/twitter/util/testing/BUILD +++ b/util-test/src/main/scala/com/twitter/util/testing/BUILD @@ -18,3 +18,25 @@ scala_library( "3rdparty/jvm/org/scalatest", ], ) + +scala_library( + name = "no-mockito", + compiler_option_sets = ["fatal_warnings"], + platform = "java8", + provides = scala_artifact( + org = "com.twitter", + name = "util-test-testing-no-mockito", + repo = artifactory, + ), + strict_deps = True, + tags = ["bazel-compatible"], + sources = ["*.scala", "!ArgumentCapture.scala"], + dependencies = [ + "3rdparty/jvm/org/scalatest", + "util/util-stats/src/main/scala", + "util/util-stats/src/main/scala/com/twitter/finagle/stats", + ], + exports = [ + "3rdparty/jvm/org/scalatest", + ], +) From 83bcbe6a857a846b29395bfd720c2985bb2602aa Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Fri, 7 Oct 2022 15:16:31 +0000 Subject: [PATCH 735/796] Replace ServiceIdentifier.fromCertificate cache with sslsession info get Problem The change in D968665 to equals and hashCode for Certificates used by SslSessionInfo has caused some issues for existing services. Solution Now that D973030 has merged, we can get the ServiceIdentity without repeatedly parsing the Certificate. Thus, we can remove the ProxyCertificate impl and its usages. Result Services can performantly get the local service identity on the hot path without impacting existing usages of Certificate. JIRA Issues: PSEC-14172, SSTR-7438 Differential Revision: https://phabricator.twitter.biz/D988049 --- .../util/security/ProxyX509Certificate.scala | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 util-security/src/main/scala/com/twitter/util/security/ProxyX509Certificate.scala diff --git a/util-security/src/main/scala/com/twitter/util/security/ProxyX509Certificate.scala b/util-security/src/main/scala/com/twitter/util/security/ProxyX509Certificate.scala deleted file mode 100644 index da5d4e6ba6..0000000000 --- a/util-security/src/main/scala/com/twitter/util/security/ProxyX509Certificate.scala +++ /dev/null @@ -1,39 +0,0 @@ -package com.twitter.util.security - -import java.security.cert.X509Certificate - -/** - * X509Certificate that transparently proxies everything - */ -class ProxyX509Certificate(underlying: X509Certificate) extends X509Certificate { - def getEncoded(): Array[Byte] = underlying.getEncoded() - def getPublicKey(): java.security.PublicKey = underlying.getPublicKey() - def verify(key: java.security.PublicKey, sigProvider: String): Unit = - underlying.verify(key, sigProvider) - def verify(key: java.security.PublicKey): Unit = underlying.verify(key) - def checkValidity(date: java.util.Date): Unit = underlying.checkValidity(date) - def checkValidity(): Unit = underlying.checkValidity() - def getBasicConstraints(): Int = underlying.getBasicConstraints() - def getIssuerDN(): java.security.Principal = underlying.getIssuerDN() - def getIssuerUniqueID(): Array[Boolean] = underlying.getIssuerUniqueID() - def getKeyUsage(): Array[Boolean] = underlying.getKeyUsage() - def getNotAfter(): java.util.Date = underlying.getNotAfter() - def getNotBefore(): java.util.Date = underlying.getNotBefore() - def getSerialNumber(): java.math.BigInteger = underlying.getSerialNumber() - def getSigAlgName(): String = underlying.getSigAlgName() - def getSigAlgOID(): String = underlying.getSigAlgOID() - def getSigAlgParams(): Array[Byte] = underlying.getSigAlgParams() - def getSignature(): Array[Byte] = underlying.getSignature() - def getSubjectDN(): java.security.Principal = underlying.getSubjectDN() - def getSubjectUniqueID(): Array[Boolean] = underlying.getSubjectUniqueID() - def getTBSCertificate(): Array[Byte] = underlying.getTBSCertificate() - def getVersion(): Int = underlying.getVersion() - - def getCriticalExtensionOIDs(): java.util.Set[String] = underlying.getCriticalExtensionOIDs() - def getExtensionValue(x: String): Array[Byte] = underlying.getExtensionValue(x) - def getNonCriticalExtensionOIDs(): java.util.Set[String] = - underlying.getNonCriticalExtensionOIDs() - def hasUnsupportedCriticalExtension(): Boolean = underlying.hasUnsupportedCriticalExtension() - - override def toString: String = underlying.toString -} From 8aff661c451b49d48bdc0fff31fd1f11e1dca46f Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Fri, 7 Oct 2022 19:22:56 +0000 Subject: [PATCH 736/796] [bazel] move more junit_tests to bazel only Differential Revision: https://phabricator.twitter.biz/D991456 --- .../src/test/java/com/twitter/cache/guava/{BUILD => BUILD.bazel} | 0 .../src/test/scala/com/twitter/cache/guava/{BUILD => BUILD.bazel} | 0 .../test/scala/com/twitter/util/jackson/{BUILD => BUILD.bazel} | 0 .../test/java/com/twitter/util/security/{BUILD => BUILD.bazel} | 0 .../test/scala/com/twitter/util/security/{BUILD => BUILD.bazel} | 0 .../src/test/java/com/twitter/util/logging/{BUILD => BUILD.bazel} | 0 .../test/scala/com/twitter/util/tunable/{BUILD => BUILD.bazel} | 0 .../java/com/twitter/util/validation/tests/{BUILD => BUILD.bazel} | 0 .../test/scala/com/twitter/util/validation/{BUILD => BUILD.bazel} | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename util-cache-guava/src/test/java/com/twitter/cache/guava/{BUILD => BUILD.bazel} (100%) rename util-cache-guava/src/test/scala/com/twitter/cache/guava/{BUILD => BUILD.bazel} (100%) rename util-jackson/src/test/scala/com/twitter/util/jackson/{BUILD => BUILD.bazel} (100%) rename util-security/src/test/java/com/twitter/util/security/{BUILD => BUILD.bazel} (100%) rename util-security/src/test/scala/com/twitter/util/security/{BUILD => BUILD.bazel} (100%) rename util-slf4j-api/src/test/java/com/twitter/util/logging/{BUILD => BUILD.bazel} (100%) rename util-tunable/src/test/scala/com/twitter/util/tunable/{BUILD => BUILD.bazel} (100%) rename util-validator/src/test/java/com/twitter/util/validation/tests/{BUILD => BUILD.bazel} (100%) rename util-validator/src/test/scala/com/twitter/util/validation/{BUILD => BUILD.bazel} (100%) diff --git a/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD.bazel similarity index 100% rename from util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD rename to util-cache-guava/src/test/java/com/twitter/cache/guava/BUILD.bazel diff --git a/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD b/util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD.bazel similarity index 100% rename from util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD rename to util-cache-guava/src/test/scala/com/twitter/cache/guava/BUILD.bazel diff --git a/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD b/util-jackson/src/test/scala/com/twitter/util/jackson/BUILD.bazel similarity index 100% rename from util-jackson/src/test/scala/com/twitter/util/jackson/BUILD rename to util-jackson/src/test/scala/com/twitter/util/jackson/BUILD.bazel diff --git a/util-security/src/test/java/com/twitter/util/security/BUILD b/util-security/src/test/java/com/twitter/util/security/BUILD.bazel similarity index 100% rename from util-security/src/test/java/com/twitter/util/security/BUILD rename to util-security/src/test/java/com/twitter/util/security/BUILD.bazel diff --git a/util-security/src/test/scala/com/twitter/util/security/BUILD b/util-security/src/test/scala/com/twitter/util/security/BUILD.bazel similarity index 100% rename from util-security/src/test/scala/com/twitter/util/security/BUILD rename to util-security/src/test/scala/com/twitter/util/security/BUILD.bazel diff --git a/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD b/util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD.bazel similarity index 100% rename from util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD rename to util-slf4j-api/src/test/java/com/twitter/util/logging/BUILD.bazel diff --git a/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD b/util-tunable/src/test/scala/com/twitter/util/tunable/BUILD.bazel similarity index 100% rename from util-tunable/src/test/scala/com/twitter/util/tunable/BUILD rename to util-tunable/src/test/scala/com/twitter/util/tunable/BUILD.bazel diff --git a/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD b/util-validator/src/test/java/com/twitter/util/validation/tests/BUILD.bazel similarity index 100% rename from util-validator/src/test/java/com/twitter/util/validation/tests/BUILD rename to util-validator/src/test/java/com/twitter/util/validation/tests/BUILD.bazel diff --git a/util-validator/src/test/scala/com/twitter/util/validation/BUILD b/util-validator/src/test/scala/com/twitter/util/validation/BUILD.bazel similarity index 100% rename from util-validator/src/test/scala/com/twitter/util/validation/BUILD rename to util-validator/src/test/scala/com/twitter/util/validation/BUILD.bazel From a9c09f489d83967a73fa4580bf9aac81aa3f3adb Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Fri, 7 Oct 2022 22:33:48 +0000 Subject: [PATCH 737/796] [util-core] Take care of thread flushing in Fiber Problem: When we start using the work queue based fiber we're going to need a way to call flush() for its tasks. Solution: Transfer the responsibility for flushing to the local fiber. This fiber will perform any necessary flushing of its own queue and the underlying scheduler. For example: - in the default case, Fiber.Global, only the global scheduler needs to be flushed - in the case of a WorkQueueFiber backed by a thread local scheduler, we need to flush both the fiber's thread local queue and the current thread's queue in the LocalScheduler JIRA Issues: CSL-12359 Differential Revision: https://phabricator.twitter.biz/D991279 --- .../src/main/scala/com/twitter/util/Fiber.scala | 17 ++++++++++++++++- .../main/scala/com/twitter/util/Promise.scala | 3 +-- .../scala/com/twitter/util/WorkQueueFiber.scala | 15 ++++++++++----- .../com/twitter/util/WorkQueueFiberTest.scala | 3 +++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Fiber.scala b/util-core/src/main/scala/com/twitter/util/Fiber.scala index aaf53b7d8d..ed2db6275d 100644 --- a/util-core/src/main/scala/com/twitter/util/Fiber.scala +++ b/util-core/src/main/scala/com/twitter/util/Fiber.scala @@ -9,15 +9,21 @@ private[twitter] abstract class Fiber { /** Submit work to the Fiber */ def submitTask(r: FiberTask): Unit + + /** Flush outstanding tasks */ + def flush(): Unit } -private[twitter] object Fiber { +private[twitter] object Fiber extends Fiber { // Global default fiber which solely submits tasks to the global Scheduler val Global: Fiber = new Fiber { override def submitTask(r: FiberTask): Unit = { Scheduler.submit(r) } + override def flush(): Unit = { + Scheduler.flush() + } } // Create fiber that captures the current Scheduler to avoid the volatile lookup on each @@ -28,6 +34,9 @@ private[twitter] object Fiber { def submitTask(r: FiberTask): Unit = { scheduler.submit(r) } + override def flush(): Unit = { + scheduler.flush() + } } def let[T](fiber: Fiber)(f: => T): T = { @@ -37,4 +46,10 @@ private[twitter] object Fiber { try f finally Local.restore(oldCtx) } + + // Submit task to the fiber stored in the Local Context + def submitTask(r: FiberTask): Unit = Local.save().fiber.submitTask(r) + + // Flush fiber stored in the Local Context + def flush(): Unit = Local.save().fiber.flush() } diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index d2a9aae995..f7ef58e720 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -1,6 +1,5 @@ package com.twitter.util -import com.twitter.concurrent.Scheduler import scala.annotation.tailrec import scala.runtime.NonLocalReturnControl import scala.util.control.NonFatal @@ -675,7 +674,7 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { // Await.result(Future.Done.map(Predef.identity)) // } // - Scheduler.flush() + Fiber.flush() if (condition.await(timeout.inNanoseconds, java.util.concurrent.TimeUnit.NANOSECONDS)) this else throw new TimeoutException(timeout.toString) diff --git a/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala index 57bfb3c126..bb2da139f2 100644 --- a/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala +++ b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala @@ -69,6 +69,11 @@ private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.Fibe } else multiThreadedSubmit(r) } + /** + * If necessary, execute any scheduler specific flushing. + */ + protected def schedulerFlush(): Unit + /** * If necessary, drain any thread specific state and resubmit the work queue. * @@ -100,13 +105,13 @@ private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.Fibe if (!multiThreadedQueue.isEmpty) { // There is fiber work that needs to be executed. Attempt to reschedule. - - // TODO: if we end up using the ThreadLocalScheduler as our underlying execution engine - // we're going to need to be able to flush it as well. Special consideration will need - // to be taken for CPU usage tracking if that is the case. - // We can work all this in the future. tryScheduleFiber() } + + // Depending on the underlying execution engine, we may need to flush the scheduler as well. + // Special consideration will need to be taken for CPU usage tracking if that is the case. + // We can work this all out in the future. + schedulerFlush() } } diff --git a/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala b/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala index f69b3457b1..a7c1b6829e 100644 --- a/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala @@ -38,6 +38,8 @@ class WorkQueueFiberTest extends AnyFunSuite { assert(runnable == null) runnable = r } + + override protected def schedulerFlush(): Unit = () } test("fiber creations are tracked") { @@ -244,6 +246,7 @@ class WorkQueueFiberTest extends AnyFunSuite { override protected def schedulerSubmit(r: Runnable): Unit = { r.run() } + override protected def schedulerFlush(): Unit = () } f.submitTask { () => From d1dbb109e2ed5cf959cfc77f9cb942ce907c52bc Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sat, 8 Oct 2022 19:59:44 +0000 Subject: [PATCH 738/796] [bazel] convert applicable java_library and scala_library to bazel only Differential Revision: https://phabricator.twitter.biz/D992114 --- .../src/test/java/com/twitter/util/jackson/{BUILD => BUILD.bazel} | 0 .../test/java/com/twitter/util/validation/{BUILD => BUILD.bazel} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename util-jackson/src/test/java/com/twitter/util/jackson/{BUILD => BUILD.bazel} (100%) rename util-validator/src/test/java/com/twitter/util/validation/{BUILD => BUILD.bazel} (100%) diff --git a/util-jackson/src/test/java/com/twitter/util/jackson/BUILD b/util-jackson/src/test/java/com/twitter/util/jackson/BUILD.bazel similarity index 100% rename from util-jackson/src/test/java/com/twitter/util/jackson/BUILD rename to util-jackson/src/test/java/com/twitter/util/jackson/BUILD.bazel diff --git a/util-validator/src/test/java/com/twitter/util/validation/BUILD b/util-validator/src/test/java/com/twitter/util/validation/BUILD.bazel similarity index 100% rename from util-validator/src/test/java/com/twitter/util/validation/BUILD rename to util-validator/src/test/java/com/twitter/util/validation/BUILD.bazel From e9d1bc9e91910a5f94ae5ab23405b3cfce4549cd Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Mon, 10 Oct 2022 14:26:12 +0000 Subject: [PATCH 739/796] util-core: Introduce TimeFormatter as replacement for TimeFormat Problem ======= `c.t.util.TimeFormat` uses SimpleDateFormat, which requires synchronization and is part of the legacy time API. This synchronization causes contention when multiple threads are formatting dates using the same instance. Solution ======== Introduce a new class, `c.t.util.TimeFormatter` that uses Java 8 `DateTimeFormatter`. It is an immutable and thus threadsafe replacement that doesn't need synchronization. JIRA Issues: CSL-12338 Differential Revision: https://phabricator.twitter.biz/D987464 --- CHANGELOG.rst | 20 ++++ .../main/scala/com/twitter/util/Time.scala | 103 +++++++++++++++--- .../com/twitter/util/TimeFormatter.scala | 81 ++++++++++++++ .../com/twitter/util/TwitterDateFormat.scala | 61 ++++++----- .../scala/com/twitter/util/TimeTest.scala | 95 ++++++++++++++++ .../twitter/util/TwitterDateFormatTest.scala | 18 --- 6 files changed, 320 insertions(+), 58 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/util/TimeFormatter.scala diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 18a73342d7..717945d7e5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,9 +12,29 @@ New Features * util-core: Introduce Future.fromCompletableFuture ``PHAB_ID=D940686`` +* util-core: Introduce `Time.fromInstant` ``PHAB_ID=D987464`` + +* util-core: Introduce `TimeFormatter`, a replacement for `TimeFormat`. `TimeFormatter` uses + the modern `java.time.format.DateTimeFormatter` API instead of the `java.text.SimpleDateFormat` + with the main advantage being that it doesn't require synchronization on either formatting or + parsing, and is thus more performant and causes no contention. ``PHAB_ID=D987464`` + * util-stats: `MetricBuilder` now has API to configure a format in which histograms are exported. Use `.withHistogramFormat` to access it. ``PHAB_ID=D908189`` +Deprecations +~~~~~~~~~~~~ +* util-core: `TimeFormat`, as well as the `Time.format` methods are deprecated in favor of using + `TimeFormatter`. ``PHAB_ID=D987464`` + +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-core: `Time.at`, `Time.toString`, and `Time.fromRss` now use `TimeFormatter` internally and + thus have better performance and do not cause contention through synchronization. `toString` always + uses `TimeFormatter` while `at` and `fromRss` only use it when run with Java 9 or later due to a + compatibility bug in Java 1.8. ``PHAB_ID=D987464`` + 22.7.0 ------ diff --git a/util-core/src/main/scala/com/twitter/util/Time.scala b/util-core/src/main/scala/com/twitter/util/Time.scala index e94edb88ca..cedde8064b 100644 --- a/util-core/src/main/scala/com/twitter/util/Time.scala +++ b/util-core/src/main/scala/com/twitter/util/Time.scala @@ -17,11 +17,13 @@ package com.twitter.util import java.io.Serializable +import java.text.ParseException import java.time.ZonedDateTime import java.time.Clock import java.time.Instant import java.time.OffsetDateTime import java.time.ZoneOffset +import java.time.format.DateTimeParseException import java.util.concurrent.TimeUnit import java.util.Date import java.util.Locale @@ -294,11 +296,14 @@ private[util] object TimeBox { * * @see [[Duration]] for intervals between two points in time. * @see [[Stopwatch]] for measuring elapsed time. - * @see [[TimeFormat]] for converting to and from `String` representations. + * @see [[TimeFormatter]] for converting to and from `String` representations. */ object Time extends TimeLikeOps[Time] { def fromNanoseconds(nanoseconds: Long): Time = new Time(nanoseconds) + def fromInstant(instant: Instant): Time = new Time( + instant.getEpochSecond * Duration.NanosPerSecond + instant.getNano) + // This is needed for Java compatibility. override def fromFractionalSeconds(seconds: Double): Time = super.fromFractionalSeconds(seconds) override def fromSeconds(seconds: Int): Time = super.fromSeconds(seconds) @@ -333,7 +338,7 @@ object Time extends TimeLikeOps[Time] { override def diff(that: Time): Duration = that match { case Top | Undefined => Duration.Undefined - case other => Duration.Top + case _ => Duration.Top } override def isFinite: Boolean = false @@ -363,7 +368,7 @@ object Time extends TimeLikeOps[Time] { override def diff(that: Time): Duration = that match { case Bottom | Undefined => Duration.Undefined - case other => Duration.Bottom + case _ => Duration.Bottom } override def isFinite: Boolean = false @@ -436,13 +441,53 @@ object Time extends TimeLikeOps[Time] { */ val epoch: Time = fromNanoseconds(0L) - private val defaultFormat = new TimeFormat("yyyy-MM-dd HH:mm:ss Z") - private val rssFormat = new TimeFormat("E, dd MMM yyyy HH:mm:ss Z") + private val defaultFormat = TimeFormatter("yyyy-MM-dd HH:mm:ss Z") + + // The default new parsing pattern is different from the default formatting pattern. There + // are two reasons for this, both of them designed to make parsing backwards-compatible with + // TimeFormat's SimpleDateFormat-based solution. One is that DateTimeFormatter pattern Z doesn't + // recognize textual zone IDs and z recognizes mostly just textual zone IDs, so we add both of + // them as optional. (Unfortunately, we can't enforce exactly one to be accepted, so it is + // possible to parse a string ending with a space and no time zone, or two time zones in which + // case last zone wins.) See https://bugs.openjdk.org/browse/JDK-8294710 + // Another reason is that old SimpleDateFormat accepts single-digits for months, days, hours, + // minutes, and seconds with two-character patterns, while DateTimeFormatter doesn't. So for + // formatting we output canonical two-digit forms, but for parsing we accept single-digit forms. + private[this] val defaultParser = + versionedParser(oldPattern = "yyyy-MM-dd HH:mm:ss Z", newPattern = "yyyy-M-d H:m:s [Z][z]") + + // The new RSS datetime pattern leverages the optional elements feature to correctly specify that + // day-of-week and second-of-minute are optional in RFC-1123. It also uses the same [Z][z] pattern + // as default parse. + private[this] val rssParser = versionedParser( + oldPattern = "E, d MMM yyyy HH:mm:ss Z", + newPattern = "[E, ]d MMM yyyy HH:mm[:ss] [Z][z]") + + // This creates a parser function that either uses the deprecated TimeFormat or the new + // TimeFormatter, depending on whether the system executes on a 1.8 Java runtime or not. This is + // We still use the old SimpleDateFormat-based format for parsing Time.at/fromRss when running on + // Java 1.8 as a workaround for a bug with correct timezone parsing with the new java.time API + // that is fixed in Java 11 and later, see https://bugs.openjdk.org/browse/JDK-8294853. + private def versionedParser(oldPattern: String, newPattern: String): String => Time = { + if (System.getProperty("java.version").startsWith("1.8")) { + val tf = new TimeFormat(oldPattern) + tf.parse + } else { + // Setting Locale.US will provide the expected textual values for day-of-week and month-of-year. + val tf = TimeFormatter(newPattern, Locale.US) + s => + try { + tf.parse(s) + } catch { + case e: DateTimeParseException => throw translateException(e) + } + } + } /** * Note, this should only ever be updated by methods used for testing. */ - private[util] val localGetTime: Local[scala.Function0[Time]] = new Local[() => Time] + private[util] val localGetTime: Local[() => Time] = new Local[() => Time] private[util] val localGetTimer: Local[MockTimer] = new Local[MockTimer] /** @@ -450,6 +495,14 @@ object Time extends TimeLikeOps[Time] { */ def apply(date: Date): Time = fromMilliseconds(date.getTime) + private[this] def translateException(e: DateTimeParseException): ParseException = { + // Methods calling this used to throw java.text.ParseException before, + // and some callers might rely on that. + val pe = new ParseException(e.getMessage, e.getErrorIndex) + pe.initCause(e) + pe + } + /** * Creates a [[Time]] instance at the given `datetime` string in the * "yyyy-MM-dd HH:mm:ss Z" format. @@ -459,7 +512,8 @@ object Time extends TimeLikeOps[Time] { * time.toString // 2019-05-12 08:00:00 +0000 * }}} */ - def at(datetime: String): Time = defaultFormat.parse(datetime) + def at(datetime: String): Time = + defaultParser(datetime) /** * Execute body with the time function replaced by `timeFunction` @@ -535,7 +589,8 @@ object Time extends TimeLikeOps[Time] { /** * Returns the Time parsed from a string in RSS format. Eg: "Wed, 15 Jun 2005 19:00:00 GMT" */ - def fromRss(rss: String): Time = rssFormat.parse(rss) + def fromRss(rss: String): Time = + rssParser(rss) } trait TimeControl { @@ -546,8 +601,25 @@ trait TimeControl { /** * A thread-safe wrapper around a SimpleDateFormat object. * + * This class is deprecated in favor of [[TimeFormatter]]. When moving from this class to + * [[TimeFormatter]], be mindful that there are some subtle differences in patterns between + * [[java.text.SimpleDateFormat]] used by this class and [[DateTimeFormatter]] used by [[TimeFormatter]]. + * Pattern character 'u' has a different meaning. Character 'a' cannot be repeated, and numerals for + * day of week produced by 'e' and 'c' can count from either Sunday or Monday depending on locale. + * Time zones might be parsed differently, and [[DateTimeFormatter]] defines several different time + * zone pattern characters with various behaviors. Consult the Javadoc for [[java.text.SimpleDateFormat]] + * and [[DateTimeFormatter]] for details. Finally, the parse method in this class throws a + * [[java.text.ParseException]] while the one in [[TimeFormatter]] throws [[DateTimeParseException]]. + * If you rely on catching exceptions from parsing, you will need to change the type of the exception. + * Note that built in [[Time.at]] and [[Time.fromRss]] keep throwing [[java.text.ParseException]] for + * backwards compatibility. + * * The default timezone is UTC. */ +@deprecated( + message = + "Use TimeFormatter instead; it uses modern time APIs and requires no synchronization. " + + "See TimeFormat documentation for information on pattern differences.") class TimeFormat( pattern: String, locale: Option[Locale], @@ -593,7 +665,7 @@ class TimeFormat( * * @see [[Duration]] for intervals between two points in time. * @see [[Stopwatch]] for measuring elapsed time. - * @see [[TimeFormat]] for converting to and from `String` representations. + * @see [[TimeFormatter]] for converting to and from `String` representations. */ sealed class Time private[util] (protected val nanos: Long) extends TimeLike[Time] @@ -627,11 +699,17 @@ sealed class Time private[util] (protected val nanos: Long) /** * Formats this Time according to the given SimpleDateFormat pattern. */ + @deprecated( + message = + "Use TimeFormatter instead. See TimeFormat documentation for information on pattern differences.") def format(pattern: String): String = new TimeFormat(pattern).format(this) /** * Formats this Time according to the given SimpleDateFormat pattern and locale. */ + @deprecated( + message = + "Use TimeFormatter instead. See TimeFormat documentation for information on pattern differences.") def format(pattern: String, locale: Locale): String = new TimeFormat(pattern, Some(locale)).format(this) @@ -697,11 +775,8 @@ sealed class Time private[util] (protected val nanos: Long) /** * Converts this Time object to a java.time.Instant */ - def toInstant: Instant = { - val millis = inMilliseconds - val nanos = inNanoseconds - (millis * Duration.NanosPerMillisecond) - Instant.ofEpochMilli(millis).plusNanos(nanos) - } + def toInstant: Instant = + Instant.ofEpochSecond(0L, nanos) /** * Converts this Time object to a java.time.ZonedDateTime with ZoneId of UTC diff --git a/util-core/src/main/scala/com/twitter/util/TimeFormatter.scala b/util-core/src/main/scala/com/twitter/util/TimeFormatter.scala new file mode 100644 index 0000000000..3db7dc5add --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/TimeFormatter.scala @@ -0,0 +1,81 @@ +/* + * Copyright 2022 Twitter Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.twitter.util + +import java.time.Instant +import java.time.ZoneId +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import java.time.format.DateTimeFormatterBuilder +import java.time.temporal.ChronoField +import java.util.Locale + +object TimeFormatter { + + /** + * Creates a new [[TimeFormatter]] based on a pattern and optionally a locale and a time zone. + * @param pattern the [[DateTimeFormatter]] compatible pattern + * @param locale locale, defaults to platform default locale for formatting + * @param zoneId time zone, defaults to UTC + * @return a new [[TimeFormatter]] for the specified pattern, locale, and time zone. + */ + def apply( + pattern: String, + locale: Locale = Locale.getDefault(Locale.Category.FORMAT), + zoneId: ZoneId = ZoneOffset.UTC + ): TimeFormatter = { + val (has_y, has_M, has_d, has_H) = TwitterDateFormat.validatePattern(pattern, newFormat = true) + val builder = new DateTimeFormatterBuilder() + .parseLenient() + .parseCaseInsensitive() + .appendPattern(pattern) + // These defaults match those that SimpleDateFormat would use during + // parsing when these fields are not specified. [[Instant.from]] can + // produce an Instant from parse results if at least all the fields up to + // HOUR_OF_DAY are present. + if (!has_y) builder.parseDefaulting(ChronoField.YEAR_OF_ERA, 1970) + if (!has_M) builder.parseDefaulting(ChronoField.MONTH_OF_YEAR, 1) + if (!has_d) builder.parseDefaulting(ChronoField.DAY_OF_MONTH, 1) + if (!has_H) builder.parseDefaulting(ChronoField.HOUR_OF_DAY, 0) + new TimeFormatter(builder.toFormatter(locale).withZone(zoneId)) + } +} + +/** + * A parser and formatter for Time instances. Instances of this class are immutable and threadsafe. + * Use [[TimeFormatter$.apply]] to construct instances from [[DateTimeFormatter]]-compatible pattern + * strings. + */ +class TimeFormatter private[util] (format: DateTimeFormatter) { + + /** + * Parses a string into a [[Time]]. + * @param str the string to parse + * @return the parsed time object + * @throws java.time.format.DateTimeParseException if the string can not be parsed + */ + def parse(str: String): Time = + Time.fromInstant(Instant.from(format.parse(str))) + + /** + * Formats a [[Time]] object into a string. + * @param time the time object to format + * @return a string with formatted representation of the time object + */ + def format(time: Time): String = + format.format(time.toInstant) +} diff --git a/util-core/src/main/scala/com/twitter/util/TwitterDateFormat.scala b/util-core/src/main/scala/com/twitter/util/TwitterDateFormat.scala index b878e33130..6626557a3b 100644 --- a/util-core/src/main/scala/com/twitter/util/TwitterDateFormat.scala +++ b/util-core/src/main/scala/com/twitter/util/TwitterDateFormat.scala @@ -18,35 +18,44 @@ object TwitterDateFormat { new SimpleDateFormat(pattern, locale) } - def validatePattern(pattern: String): Unit = { - val stripped = stripSingleQuoted(pattern) - if (stripped.contains('Y') && !(stripped.contains('w'))) { + // Returns a quadruple of booleans, each signifying whether the pattern has + // a symbol for year-of-era (y), month-of-year (M), day-of-month (d), and + // hour-of-day (H), respectively. Throws an exception if week-year (Y) + // is used without week-in-year (w). + def validatePattern( + pattern: String, + newFormat: Boolean = false + ): (Boolean, Boolean, Boolean, Boolean) = { + var quoteOff = true + var optionalLevel = 0 + var has_Y = false + var has_w = false + var has_y = false + var has_M = false + var has_d = false + var has_H = false + val chars = pattern.toCharArray + var i = 0 + while (i < chars.length) { + chars(i) match { + case '\'' => quoteOff = !quoteOff + case '[' if newFormat && quoteOff => optionalLevel += 1 + case ']' if newFormat && quoteOff => optionalLevel -= 1 + case 'y' if !has_y && quoteOff && optionalLevel == 0 => has_y = true + case 'M' if !has_M && quoteOff && optionalLevel == 0 => has_M = true + case 'd' if !has_d && quoteOff && optionalLevel == 0 => has_d = true + case 'H' if !has_H && quoteOff && optionalLevel == 0 => has_H = true + case 'Y' if !has_Y && quoteOff && optionalLevel == 0 => has_Y = true + case 'w' if !has_w && quoteOff && optionalLevel == 0 => has_w = true + case _ => () + } + i += 1 + } + if (has_Y && !has_w) { throw new IllegalArgumentException( "Invalid date format uses 'Y' for week-of-year without 'w': %s".format(pattern) ) } - } - - /** Patterns can contain quoted strings 'foo' which we should ignore in checking the pattern */ - private[util] def stripSingleQuoted(pattern: String): String = { - val buf = new StringBuilder(pattern.size) - var startIndex = 0 - var endIndex = 0 - while (startIndex >= 0 && startIndex < pattern.size) { - endIndex = pattern.indexOf('\'', startIndex) - if (endIndex < 0) { - buf.append(pattern.substring(startIndex)) - startIndex = -1 - } else { - buf.append(pattern.substring(startIndex, endIndex)) - startIndex = endIndex + 1 - endIndex = pattern.indexOf('\'', startIndex) - if (endIndex < 0) { - throw new IllegalArgumentException("Unmatched quote in date format: %s".format(pattern)) - } - startIndex = endIndex + 1 - } - } - buf.toString + (has_y, has_M, has_d, has_H) } } diff --git a/util-core/src/test/scala/com/twitter/util/TimeTest.scala b/util-core/src/test/scala/com/twitter/util/TimeTest.scala index bc2e02b993..b757e0d795 100644 --- a/util-core/src/test/scala/com/twitter/util/TimeTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TimeTest.scala @@ -7,6 +7,7 @@ import java.io.ObjectInputStream import java.io.ObjectOutputStream import java.time.Instant import java.time.OffsetDateTime +import java.time.ZoneId import java.time.ZoneOffset import java.time.ZonedDateTime import java.util.Locale @@ -380,6 +381,84 @@ class TimeFormatTest extends AnyFunSuite { } } +class TimeFormatterTest extends AnyFunSuite { + def assertParsesTo( + fmt: String, + str: String, + year: Int, + month: Int, + day: Int, + hour: Int, + minute: Int, + second: Int + ) = { + assert( + TimeFormatter(fmt).parse(str) == Time.fromInstant( + Instant.from(OffsetDateTime.of(year, month, day, hour, minute, second, 0, ZoneOffset.UTC)))) + + } + + test("format correctly with non US locale") { + val locale = Locale.GERMAN + val format = "EEEE" + val timeFormat = TimeFormatter(format, locale) + val day = "Donnerstag" + assert(timeFormat.format(timeFormat.parse(day)) == day) + } + + test("set UTC timezone as default") { + val format = "HH:mm" + val timeFormat = TimeFormatter(format) + val utcFormat = TimeFormatter(format, zoneId = ZoneOffset.UTC) + assert(timeFormat.format(utcFormat.parse("16:04")) == "16:04") + } + + test("set non-UTC timezone correctly") { + val format = "HH:mm" + val estFormat = TimeFormatter(format, zoneId = ZoneId.of("America/New_York")) + val utcFormat = TimeFormatter(format, zoneId = ZoneOffset.UTC) + assert(utcFormat.format(estFormat.parse("16:04")) == "21:04") + } + + test("parse dates with explicit offset timezone") { + // Known bug in Java 1.8.0 until at least _345, see https://bugs.openjdk.org/browse/JDK-8294853 + // We're conditionally executing this test because pants still runs tests with Java 8. + // Alas, java.lang.Runtime.Version also doesn't exist until Java 9, so we're stuck with this + // somewhat pedestrian way of checking the runtime version... + if (!System.getProperty("java.version").startsWith("1.8.0")) { + assertParsesTo("yyyy-MM-dd HH:mm:ss Z", "2022-09-29 13:33:37 +0100", 2022, 9, 29, 12, 33, 37) + } + } + + test("parse dates with explicit named timezone") { + assertParsesTo("yyyy-MM-dd HH:mm:ss z", "2022-09-29 13:33:37 EST", 2022, 9, 29, 17, 33, 37) + } + + test("parse dates with no timezone") { + assertParsesTo("yyyy-MM-dd HH:mm:ss", "2022-09-29 13:33:37", 2022, 9, 29, 13, 33, 37) + } + + test("parse dates only") { + assertParsesTo("yyyy-MM-dd", "2022-09-29", 2022, 9, 29, 0, 0, 0) + } + + test("parse year-month only") { + assertParsesTo("yyyy-MM", "2022-09", 2022, 9, 1, 0, 0, 0) + } + + test("parse month-day only") { + assertParsesTo("MM-dd", "09-29", 1970, 9, 29, 0, 0, 0) + } + + test("parse hour-minute only") { + assertParsesTo("HH-mm", "13-37", 1970, 1, 1, 13, 37, 0) + } + + test("parse hour only") { + assertParsesTo("HH", "13", 1970, 1, 1, 13, 0, 0) + } +} + trait TimeOps { val ops: Time.type = Time } class TimeTest extends AnyFunSuite @@ -727,4 +806,20 @@ class TimeTest assert(offsetDateTime.getNano == Time.now.inNanoseconds % Duration.NanosPerSecond) } } + + test("at") { + assert(Time.at("1970-01-01 00:00:00 +0100").inLongSeconds == -3600) + assert(Time.at("1970-01-01 00:00:00 +0000").inLongSeconds == 0) + assert(Time.at("1970-01-01 00:00:00 UTC").inLongSeconds == 0) + assert(Time.at("1970-01-01 00:00:00 GMT").inLongSeconds == 0) + assert(Time.at("1970-01-01 00:00:00 EST").inLongSeconds == 18000) + } + + test("fromRss") { + assert(Time.fromRss("Thu, 1 Jan 1970 00:00:00 +0100").inLongSeconds == -3600) + assert(Time.fromRss("Thu, 1 Jan 1970 00:00:00 +0000").inLongSeconds == 0) + assert(Time.fromRss("Thu, 1 Jan 1970 00:00:00 UTC").inLongSeconds == 0) + assert(Time.fromRss("Thu, 1 Jan 1970 00:00:00 GMT").inLongSeconds == 0) + assert(Time.fromRss("Thu, 1 Jan 1970 00:00:00 EST").inLongSeconds == 18000) + } } diff --git a/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala b/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala index 84bbb5c19f..c3fccc5088 100644 --- a/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala +++ b/util-core/src/test/scala/com/twitter/util/TwitterDateFormatTest.scala @@ -20,22 +20,4 @@ class TwitterDateFormatTest extends AnyFunSuite { test("TwitterDateFormat should allow Y when quoted") { TwitterDateFormat("yyyy 'Year'") } - - test("TwitterDateFormat should stripSingleQuoted") { - import TwitterDateFormat._ - assert(stripSingleQuoted("") == "") - assert(stripSingleQuoted("YYYY") == "YYYY") - assert(stripSingleQuoted("''") == "") - assert(stripSingleQuoted("'abc'") == "") - assert(stripSingleQuoted("x'abc'") == "x") - assert(stripSingleQuoted("'abc'x") == "x") - assert(stripSingleQuoted("'abc'def'ghi'") == "def") - - intercept[IllegalArgumentException] { - stripSingleQuoted("'abc") - } - intercept[IllegalArgumentException] { - stripSingleQuoted("'abc'def'ghi") - } - } } From 49659ca280cce020e4dafdb0954e641baaca851e Mon Sep 17 00:00:00 2001 From: Ruben Oanta Date: Tue, 11 Oct 2022 20:00:29 +0000 Subject: [PATCH 740/796] csl: bump jackson to 2.13.4 Problem / Solution Upgrade jackson across source. Differential Revision: https://phabricator.twitter.biz/D991965 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index c8ecd4bed1..e483451629 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "22.10.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.13.3" +val jacksonVersion = "2.13.4" val json4sVersion = "4.0.3" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" From 16ff5c3db423488507cd9b4c6c647d41c1938fd3 Mon Sep 17 00:00:00 2001 From: Yi Cheng Date: Sun, 16 Oct 2022 23:02:52 +0000 Subject: [PATCH 741/796] [bazel] convert applicable resources target to bazel only Differential Revision: https://phabricator.twitter.biz/D999259 --- util-jackson/src/test/resources/{BUILD => BUILD.bazel} | 0 .../src/main/resources/{BUILD => BUILD.bazel} | 0 util-tunable/src/test/resources/{BUILD => BUILD.bazel} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename util-jackson/src/test/resources/{BUILD => BUILD.bazel} (100%) rename util-security-test-certs/src/main/resources/{BUILD => BUILD.bazel} (100%) rename util-tunable/src/test/resources/{BUILD => BUILD.bazel} (100%) diff --git a/util-jackson/src/test/resources/BUILD b/util-jackson/src/test/resources/BUILD.bazel similarity index 100% rename from util-jackson/src/test/resources/BUILD rename to util-jackson/src/test/resources/BUILD.bazel diff --git a/util-security-test-certs/src/main/resources/BUILD b/util-security-test-certs/src/main/resources/BUILD.bazel similarity index 100% rename from util-security-test-certs/src/main/resources/BUILD rename to util-security-test-certs/src/main/resources/BUILD.bazel diff --git a/util-tunable/src/test/resources/BUILD b/util-tunable/src/test/resources/BUILD.bazel similarity index 100% rename from util-tunable/src/test/resources/BUILD rename to util-tunable/src/test/resources/BUILD.bazel From 49a2e031982c4babcb66aaa0f5a0cbdc809ab86b Mon Sep 17 00:00:00 2001 From: Jing Yan Date: Tue, 18 Oct 2022 23:57:03 +0000 Subject: [PATCH 742/796] twitter-server: Expose namespaces in admin/namespaces endpoint Problem/Solution Expose namespaces carried in `ExpressionSchema` via a new Twitter Server Admin endpoint as `admin/namespaces` JIRA Issues: CSL-11983 Differential Revision: https://phabricator.twitter.biz/D986107 --- .../main/scala/com/twitter/finagle/stats/SchemaRegistry.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala index 615e426c13..df63b1e2c7 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/SchemaRegistry.scala @@ -1,6 +1,7 @@ package com.twitter.finagle.stats -import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey} +import com.twitter.finagle.stats.exp.ExpressionSchema +import com.twitter.finagle.stats.exp.ExpressionSchemaKey /** * Interface used via the LoadService mechanism to obtain an From 64c63bc1587e25ab409410010263934443c485ed Mon Sep 17 00:00:00 2001 From: Jared Crawford Date: Thu, 20 Oct 2022 18:37:30 +0000 Subject: [PATCH 743/796] Remove ex-employees from PROJECT files / sandboxes JIRA Issues: PSEC-14296 Differential Revision: https://phabricator.twitter.biz/D1000832 --- util-security/PROJECT | 1 - 1 file changed, 1 deletion(-) diff --git a/util-security/PROJECT b/util-security/PROJECT index 66ca946cfc..828e37f74e 100644 --- a/util-security/PROJECT +++ b/util-security/PROJECT @@ -4,7 +4,6 @@ owners: # Individual users are not responsible for providing a blocking review, but they # are capable of doing so in the event that members of the groups cannot. - banderson - - bdimmick - roanta - vkostyukov - yufang From d26189b7d8bddb54ee40cfa84dc4c76e67a1aa95 Mon Sep 17 00:00:00 2001 From: Christopher Coco Date: Fri, 21 Oct 2022 18:38:34 +0000 Subject: [PATCH 744/796] util/util-inject - Remove experimental code Problem/Solution Remove the experimental code that is not used and not included in the sbt build but causes some confusion with the actual injection library in Finatra. Result Less confusion. Differential Revision: https://phabricator.twitter.biz/D1002578 --- util-inject/PROJECT | 9 - util-inject/README.md | 7 - .../com/twitter/util/inject/flag/BUILD.bazel | 14 - .../com/twitter/util/inject/flag/Flagged.java | 24 - .../twitter/util/inject/flag/FlaggedImpl.java | 49 -- .../com/twitter/util/inject/flag/Flags.java | 30 - .../scala/com/twitter/util/inject/BUILD.bazel | 28 - .../scala/com/twitter/util/inject/Flags.scala | 17 - .../com/twitter/util/inject/Injector.scala | 142 ----- .../twitter/util/inject/InjectorModule.scala | 16 - .../util/inject/TwitterBaseModule.scala | 24 - .../twitter/util/inject/TwitterModule.scala | 368 ------------ .../util/inject/TwitterModuleFlags.scala | 112 ---- .../util/inject/TwitterModuleLifecycle.scala | 83 --- .../com/twitter/util/inject/TypeUtils.scala | 76 --- .../twitter/util/inject/internal/BUILD.bazel | 25 - .../util/inject/internal/FlagsModule.scala | 69 --- .../inject/internal/InstalledModules.scala | 82 --- .../util/inject/internal/Modules.scala | 113 ---- .../TwitterTypeConvertersModule.scala | 46 -- .../util/inject/internal/iterable.scala | 37 -- .../util/inject/internal/package.scala | 5 - .../java/com/twitter/util/inject/BUILD.bazel | 10 - .../util/inject/TestBindingAnnotation.java | 19 - .../scala/com/twitter/util/inject/BUILD.bazel | 22 - .../twitter/util/inject/InjectorTest.scala | 195 ------- .../util/inject/TwitterModuleTest.scala | 543 ------------------ .../twitter/util/inject/TypeUtilsTest.scala | 46 -- .../com/twitter/util/inject/classes.scala | 104 ---- .../inject/module/SeqAbstractModule.scala | 12 - .../util/inject/module/TestModule.scala | 26 - .../inject/module/TestTwitterModule.scala | 136 ----- .../twitter/util/inject/servicefactory.scala | 32 -- 33 files changed, 2521 deletions(-) delete mode 100644 util-inject/PROJECT delete mode 100644 util-inject/README.md delete mode 100644 util-inject/src/main/java/com/twitter/util/inject/flag/BUILD.bazel delete mode 100644 util-inject/src/main/java/com/twitter/util/inject/flag/Flagged.java delete mode 100644 util-inject/src/main/java/com/twitter/util/inject/flag/FlaggedImpl.java delete mode 100644 util-inject/src/main/java/com/twitter/util/inject/flag/Flags.java delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/BUILD.bazel delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/Flags.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/Injector.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/InjectorModule.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/TwitterBaseModule.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/TwitterModule.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleLifecycle.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD.bazel delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/internal/FlagsModule.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/internal/InstalledModules.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/internal/Modules.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/internal/TwitterTypeConvertersModule.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/internal/iterable.scala delete mode 100644 util-inject/src/main/scala/com/twitter/util/inject/internal/package.scala delete mode 100644 util-inject/src/test/java/com/twitter/util/inject/BUILD.bazel delete mode 100644 util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java delete mode 100644 util-inject/src/test/scala/com/twitter/util/inject/BUILD.bazel delete mode 100644 util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala delete mode 100644 util-inject/src/test/scala/com/twitter/util/inject/TwitterModuleTest.scala delete mode 100644 util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala delete mode 100644 util-inject/src/test/scala/com/twitter/util/inject/classes.scala delete mode 100644 util-inject/src/test/scala/com/twitter/util/inject/module/SeqAbstractModule.scala delete mode 100644 util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala delete mode 100644 util-inject/src/test/scala/com/twitter/util/inject/module/TestTwitterModule.scala delete mode 100644 util-inject/src/test/scala/com/twitter/util/inject/servicefactory.scala diff --git a/util-inject/PROJECT b/util-inject/PROJECT deleted file mode 100644 index fbed8cad0c..0000000000 --- a/util-inject/PROJECT +++ /dev/null @@ -1,9 +0,0 @@ -owners: - - banderson - - ibennett - - mnakamura - - roanta - - vkostyukov - - yufang -watchers: - - csl-team:ldap diff --git a/util-inject/README.md b/util-inject/README.md deleted file mode 100644 index c62f2a3072..0000000000 --- a/util-inject/README.md +++ /dev/null @@ -1,7 +0,0 @@ -util-inject -=========== - -2020-08-31 - -This library is currently in development and experimental. We do -not recommend users attempt to adopt this code at this time. diff --git a/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD.bazel b/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD.bazel deleted file mode 100644 index abf6148959..0000000000 --- a/util-inject/src/main/java/com/twitter/util/inject/flag/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -java_library( - sources = ["*.java"], - compiler_option_sets = ["fatal_warnings"], - provides = artifact( - org = "com.twitter", - name = "util-inject-java", - repo = artifactory, - ), - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - ], -) diff --git a/util-inject/src/main/java/com/twitter/util/inject/flag/Flagged.java b/util-inject/src/main/java/com/twitter/util/inject/flag/Flagged.java deleted file mode 100644 index 0c2c596e01..0000000000 --- a/util-inject/src/main/java/com/twitter/util/inject/flag/Flagged.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.util.inject.flag; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import com.google.inject.BindingAnnotation; - -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -/** - * Annotates parsed flags. - * - * @see - * @see - */ -@Retention(RUNTIME) -@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) -@BindingAnnotation -public @interface Flagged { - - /** Name of the flag */ - String value(); -} diff --git a/util-inject/src/main/java/com/twitter/util/inject/flag/FlaggedImpl.java b/util-inject/src/main/java/com/twitter/util/inject/flag/FlaggedImpl.java deleted file mode 100644 index 591629148b..0000000000 --- a/util-inject/src/main/java/com/twitter/util/inject/flag/FlaggedImpl.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.twitter.util.inject.flag; - -import java.io.Serializable; -import java.lang.annotation.Annotation; - -/** - * Do not use this class directly. Use {@link Flags#named(String)}. - * - * Pattern copied from com.google.inject.name.NamedImpl. - * @see - * @see Flags - * @see Flagged - */ -class FlaggedImpl implements Flagged, Serializable { - private final String value; - - FlaggedImpl(String value) { - this.value = value; - } - - public String value() { - return this.value; - } - - public int hashCode() { - // This is specified in java.lang.Annotation. - return (127 * "value".hashCode()) ^ value.hashCode(); - } - - /** Flag specific equals */ - public boolean equals(Object o) { - if (!(o instanceof Flagged)) { - return false; - } - - Flagged other = (Flagged) o; - return value.equals(other.value()); - } - - public String toString() { - return "@" + Flagged.class.getName() + "(value=" + value + ")"; - } - - public Class annotationType() { - return Flagged.class; - } - - private static final long serialVersionUID = 0; -} diff --git a/util-inject/src/main/java/com/twitter/util/inject/flag/Flags.java b/util-inject/src/main/java/com/twitter/util/inject/flag/Flags.java deleted file mode 100644 index 0695caa89f..0000000000 --- a/util-inject/src/main/java/com/twitter/util/inject/flag/Flags.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.twitter.util.inject.flag; - -import com.google.inject.Key; - -/** - * Utility methods for use with {@code @}{@link Flagged}. Pattern copied from - * {@code com.google.inject.name.Names}. - * - * @see - * @see Flagged - */ -public final class Flags { - private Flags() { - } - - /** Creates a {@link Flagged} annotation with {@code name} as the value. */ - public static Flagged named(String name) { - return new FlaggedImpl(name); - } - - /** - * Builds a {@code com.google.inject.Key} over a String type annotated - * with {@code @}{@link Flagged}(named) - * - * @see - */ - public static Key key(String name) { - return Key.get(String.class, Flags.named(name)); - } -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/BUILD.bazel b/util-inject/src/main/scala/com/twitter/util/inject/BUILD.bazel deleted file mode 100644 index 22028bbb23..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - provides = scala_artifact( - org = "com.twitter", - name = "util-inject", - repo = artifactory, - ), - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "//:scala-reflect", - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/net/codingwell:scala-guice", - "util/util-app/src/main/scala", - "util/util-slf4j-api/src/main/scala", - ], - exports = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-assistedinject", - "3rdparty/jvm/net/codingwell:scala-guice", - "util/util-app/src/main/scala", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/util-inject/src/main/scala/com/twitter/util/inject/Flags.scala b/util-inject/src/main/scala/com/twitter/util/inject/Flags.scala deleted file mode 100644 index e562f443f3..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/Flags.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.twitter.inject - -import com.twitter.app.Flag - -/** Transforms a [[com.twitter.app.Flags]] collection into a map look up */ -case class Flags private[inject] (private[inject] val flgs: com.twitter.app.Flags) { - private[this] val underlying: Map[String, Flag[_]] = { - val flags: Seq[Flag[_]] = flgs.getAll(includeGlobal = false).toSeq - flags.map(f => f.name -> f).toMap - } - - /** gets Flag from [com.twitter.app.Flags] map look up */ - def get(name: String): Option[Flag[_]] = underlying.get(name) - - /** gets set of defined Flags */ - def getAll: Seq[Flag[_]] = underlying.values.toSeq -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/Injector.scala b/util-inject/src/main/scala/com/twitter/util/inject/Injector.scala deleted file mode 100644 index d39107684e..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/Injector.scala +++ /dev/null @@ -1,142 +0,0 @@ -package com.twitter.inject - -import com.google.inject.{Key, Injector => UnderlyingInjector} -import java.lang.annotation.Annotation -import net.codingwell.scalaguice.KeyExtensions._ -import net.codingwell.scalaguice.typeLiteral -import scala.reflect.ClassTag -import scala.reflect.runtime.universe._ - -final case class Injector private[inject] (underlying: UnderlyingInjector) { - - /** - * Returns the appropriate instance for the given key constructed from the - * passed type [[T]]. - * - * When feasible, avoid using this method in favor of having the Injector - * inject your dependencies ahead of time via annotating your constructor - * with `@Inject`. - * - * @tparam T type of the bound instance to return from the object graph. - * - * @return bound instance of type [[T]]. - */ - def instance[T: TypeTag]: T = - underlying.getInstance(typeLiteral[T].toKey) - - /** - * Returns the appropriate instance for the given key constructed from the - * passed type [[T]] and given [[java.lang.annotation.Annotation]] type [[Ann]]. - * - * When feasible, avoid using this method in favor of having the Injector - * inject your dependencies ahead of time via annotating your constructor - * with `@Inject`. - * - * @tparam T type of the bound instance to return from the object graph. - * @tparam Ann type of the annotation used to disambiguate the bound type [[T]]. - * - * @return bound instance of type [[T]] annotated with annotation type [[Ann]]. - */ - def instance[T: TypeTag, Ann <: Annotation: ClassTag]: T = - underlying.getInstance(typeLiteral[T].annotatedWith[Ann]) - - /** - * Returns the appropriate instance for the given key constructed from the - * passed type [[T]] and given [[java.lang.annotation.Annotation]] annotation. - * - * When feasible, avoid using this method in favor of having the Injector - * inject your dependencies ahead of time via annotating your constructor - * with `@Inject`. - * - * @param annotation [[java.lang.annotation.Annotation]] instance used to - * disambiguate the bound type [[T]]. - * @tparam T type of the bound instance to return from the object graph. - * - * @return bound instance of type [[T]] annotated with annotation. - */ - def instance[T: TypeTag](annotation: Annotation): T = - underlying.getInstance(typeLiteral[T].annotatedWith(annotation)) - - /** - * Returns the appropriate instance for the given key constructed from the - * passed type [[T]] and given [[java.lang.annotation.Annotation]] class. - * - * When feasible, avoid using this method in favor of having the Injector - * inject your dependencies ahead of time via annotating your constructor - * with `@Inject`. - * - * @param annotationClazz class of [[java.lang.annotation.Annotation]] used - * to disambiguate the bound type [[T]]. - * @tparam T type of the bound instance to return from the object graph. - * @return bound instance of type [[T]] annotated with annotation class. - */ - def instance[T: TypeTag](annotationClazz: Class[_ <: Annotation]): T = - underlying.getInstance(typeLiteral[T].annotatedWith(annotationClazz)) - - /** - * Returns the appropriate instance for the given injection type. - * - * When feasible, avoid using this method, in favor of having the Injector - * inject your dependencies ahead of time via annotating your constructor - * with `@Inject`. - * - * @param clazz the class of type [[T]] of the bound instance to return - * from the object graph. - * @tparam T type of the bound instance to return from the object graph. - * - * @return bound instance of type [[T]]. - */ - def instance[T](clazz: Class[T]): T = underlying.getInstance(clazz) - - /** - * Returns the appropriate instance for the given key constructed from the - * passed class and given [[java.lang.annotation.Annotation]] annotation. - * - * When feasible, avoid using this method, in favor of having the Injector - * inject your dependencies ahead of time via annotating your constructor - * with `@Inject`. - * - * @param clazz the class of type [[T]] of the bound instance to return from - * the object graph. - * @param annotation [[java.lang.annotation.Annotation]] instance used to - * disambiguate the bound type [[T]]. - * @tparam T type of the bound instance to return from the object graph. - * - * @return bound instance of type [[T]]. - */ - def instance[T](clazz: Class[T], annotation: Annotation): T = - underlying.getInstance(Key.get(clazz, annotation)) - - /** - * Returns the appropriate instance for the given key constructed from the - * passed class and given [[java.lang.annotation.Annotation]] class. - * - * @param clazz the class of type [[T]] of the bound instance to return from - * the object graph. - * @param annotationClazz [[java.lang.annotation.Annotation]] class used to - * disambiguate the bound type [[T]]. - * @tparam T type of the bound instance to return from the object graph. - * @tparam Ann type of the annotation class used to disambiguate the bound type [[T]]. - * - * @return bound instance of type [[T]]. - */ - def instance[T, Ann <: Annotation](clazz: Class[T], annotationClazz: Class[Ann]): T = - underlying.getInstance(Key.get(clazz, annotationClazz)) - - /** - * Returns the appropriate instance for the given injection key. - * - * When feasible, avoid using this method in favor of having the Injector - * inject your dependencies ahead of time via annotating your constructor - * with `@Inject`. - * - * @param key [[com.google.inject.Key]] binding key of the bound instance to return - * from the object graph. - * @tparam T type of the bound instance to return from the object graph. - * - * @return bound instance of type [[T]] represented by [[com.google.inject.Key]] key. - * - * @see [[https://google.github.io/guice/api-docs/latest/javadoc/com/google/inject/Key.html com.google.inject.Key]] - */ - def instance[T](key: Key[T]): T = underlying.getInstance(key) -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/InjectorModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/InjectorModule.scala deleted file mode 100644 index 72f741c64f..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/InjectorModule.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.twitter.inject - -import com.google.inject.{Provides, Injector => GuiceInjector} -import javax.inject.Singleton - -object InjectorModule extends TwitterModule { - - @Provides - @Singleton - private def providesInjector(injector: GuiceInjector): Injector = { - Injector(injector) - } - - /** Java-friendly way to access this module as a singleton instance */ - def get(): this.type = this -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/TwitterBaseModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/TwitterBaseModule.scala deleted file mode 100644 index 5938beefde..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/TwitterBaseModule.scala +++ /dev/null @@ -1,24 +0,0 @@ -package com.twitter.inject - -import com.google.inject.Module - -private[inject] trait TwitterBaseModule extends TwitterModuleFlags with TwitterModuleLifecycle { - - /** - * Additional modules to be composed into this module. This list of modules is generally used - * ''instead'' of the [[com.twitter.inject.TwitterModule.install]] method to properly support the - * [[TwitterModuleLifecycle]] for [[com.twitter.inject.TwitterModule]] instances. - * - * @note [[com.twitter.inject.TwitterModule.install(module: Module)]] can still be used for non-[[TwitterModule]] 1 - * instances, and is sometimes preferred due to `install` being deferred until after flag parsing occurs. - * @note Java users should prefer [[javaModules]]. - */ - protected[inject] def modules: Seq[Module] = Seq.empty - - /** Additional modules to be composed into this module from Java */ - protected[inject] def javaModules: java.util.Collection[Module] = - java.util.Collections.emptyList[Module]() - - /** Additional framework modules to be composed into this module. */ - protected[inject] def frameworkModules: Seq[Module] = Seq.empty -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/TwitterModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/TwitterModule.scala deleted file mode 100644 index 8e5f6459ac..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/TwitterModule.scala +++ /dev/null @@ -1,368 +0,0 @@ -package com.twitter.inject - -import com.google.inject.assistedinject.FactoryModuleBuilder -import com.google.inject.matcher.{Matcher, Matchers} -import com.google.inject.spi.TypeConverter -import com.google.inject.{Module, _} -import com.twitter.app.Flaggable -import com.twitter.util.logging.Logging -import java.lang.annotation.Annotation -import net.codingwell.scalaguice.{ScalaModule, ScalaMultibinder, ScalaOptionBinder, typeLiteral} - -/** - * =Overview= - * A support class for [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/Module.html Module]] - * implementations which exposes a DSL for binding via type parameters. Extend this class, - * override the `configure` method and call the `bind` methods, or define custom `@Provides` - * annotated methods. This class also provides an integration with [[com.twitter.app.Flag]] types - * which allows for passing external configuration to aid in the creation of bound types. Lastly, it - * is also possible to define a list of other [[TwitterModule]] instances which this [[TwitterModule]] - * "depends" on by setting the [[TwitterBaseModule.modules]] (or [[TwitterBaseModule.javaModules]]) - * to a non-empty list. This will ensure that when only this [[TwitterModule]] instance is used to - * compose an [[Injector]] the "dependent" list of modules will also be installed. - * - * =Lifecycle= - * A [[TwitterModule]] has a lifecycle. Executing this lifecycle is particularly important if the - * [[TwitterModule]] creates any [[com.twitter.app.Flag]] instances. - * - * @note Attempting to bind the same type multiple times with no discriminating - * [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/BindingAnnotation.html com.google.inject.BindingAnnotation]] - * will result in an exception during injector construction. - * @example - * {{{ - * object MyModule extends TwitterModule { - * flag[String](name = "card.gateway", help = "The processing gateway to use for credit cards.") - * - * override protected def configure(): Unit = { - * bind[Service].to[ServiceImpl].in[Singleton] - * bind[CreditCardPaymentService] - * bind[Bar[Foo]].to[FooBarImpl] - * bind[PaymentService].to[CreditCardPaymentService] - * } - * - * @Singleton - * @Provides - * def provideCreditCardServiceProcessor( - * @Flag("card.gateway") gateway: String - * ): CreditCardProcessor = { - * new CreditCardProcessor(gateway) - * } - * } - * }}} - * @see [[https://static.javadoc.io/com.google.inject/guice/4.1.0/com/google/inject/AbstractModule.html com.google.inject.AbstractModule]] - * @see [[https://twitter.github.io/finatra/user-guide/getting-started/modules.html Writing Modules in Finatra]] - * @define multiBinding - * An API to bind multiple values separately, only to later inject them as a complete collection. - * "Multibinding" is intended for use in your application's module: - * {{{ - * class SnacksModule extends TwitterModule { - * override protected def configure(): Unit = { - * bindMultiple[Snack].addBinding.toInstance(new Twix()) - * bindMultiple[Snack].addBinding.toProvider[SnickersProvider] - * bindMultiple[Snack].addBinding.to[Skittles] - * } - * } - * }}} - * - * With this binding, a `Set[Snack]` can now be injected: - * {{{ - * class SnackMachine @Inject()(Set[Snack] snacks) - * }}} - * - * Contributing multibindings from different modules is also supported. For example, it is okay - * for both `CandyModule` and `ChipsModule` to create their own `Multibinder[Snack]`, and to each - * contribute bindings to the set of snacks. When that set is injected, it will contain elements - * from both modules. - * - * The set's iteration order is consistent with the binding order. This is convenient when multiple - * elements are contributed by the same module because that module can order its bindings - * appropriately. Avoid relying on the iteration order of elements contributed by different modules, - * since there is no equivalent mechanism to order modules. - * - * The set is unmodifiable. Elements can only be added to the set by configuring the multibinder. - * Elements can never be removed from the set. - * - * Elements are resolved at set injection time. If an element is bound to a provider, that - * provider's get method will be called each time the set is injected (unless the binding is - * also scoped). - * - * Annotations can be used to create different sets of the same element type. Each distinct - * annotation gets its own independent collection of elements. - * - * Elements '''MUST''' be distinct. If multiple bound elements have the same value, set injection will fail. - * - * Elements '''MUST''' be non-null. If any set element is null, set injection will fail. - * @define optionalBinding - * Calling this method will always supply the bindings: `Option[T]` and `Option[Provider[T]]`. If - * `setBinding` or `setDefault` are called, it will also bind `T`. - * - * `setDefault` is intended for use by frameworks that need a default value. User code can call - * `setBinding` to override the default. - * - * '''Warning:''' even if `setBinding` is called, the default binding will still exist in the - * object graph. If it is a singleton, it will be instantiated in `Stage.PRODUCTION`. - * - * If `setDefault` or `setBinding` are linked to Providers, the Provider may return null. If it - * does, the Option bindings will be a None. Binding `setBinding` to a Provider that returns - * null will not cause `OptionalBinder` to fall back to the `setDefault` binding. - * - * If neither `setDefault` nor `setBinding` are called, it will try to link to a user-supplied - * binding of the same type. If no binding exists, the options will be absent. Otherwise, if a - * user-supplied binding of that type exists, or if `setBinding` or `setDefault` are called, the - * options will return `Some(T)` if they are bound to a non-null value, otherwise `None`. - * - * Values are resolved at injection time. If a value is bound to a provider, that provider's - * get method will be called each time the option is injected (unless the binding is also - * scoped, or an option of provider is injected). - * - * Annotations are used to create different options of the same key/value type. Each distinct - * annotation gets its own independent binding. For example: - * {{{ - * class FrameworkModule extends TwitterModule { - * override protected def configure(): Unit = { - * bindOption[Renamer] - * } - * } - * }}} - * - * With this module, an `Option[Renamer]` can now be injected. With no other bindings, the - * option will be None. Users can specify bindings in one of two ways: - * - * Option 1: - * {{{ - * class UserRenamerModule extends TwitterModule { - * override protected def configure(): Unit = { - * bind[Renamer].to[ReplacingRenamer] - * } - * } - * }}} - * or Option 2: - * {{{ - * class UserRenamerModule extends TwitterModule { - * override protected def configure(): Unit = { - * bindOption[Renamer].setBinding.to[ReplacingRenamer] - * } - * } - * }}} - * - * With both options, the `Option[Renamer]` will be present and supply the `ReplacingRenamer`. - * - * Default values can be supplied using: - * {{{ - * class FrameworkModule extends TwitterModule { - * override protected def configure(): Unit = { - * bindOption[String, LookupUrl].setDefault.toInstance(DefaultLookupUrl) - * } - * } - * }}} - * - * With the above module, code can inject an `@LookupUrl`-annotated String and it will supply - * the `DefaultLookupUrl`. A user can change this value by binding: - * {{{ - * class UserLookupModule extends TwitterModule { - * override protected def configure(): Unit = { - * bindOption[String, LookupUrl].setBinding.toInstance(CustomLookupUrl) - * } - * } - * }}} - * - * which will override the default value. - * - * If one module uses `setDefault` the only way to override the default is to use `setBinding`. - * It is an error for a user to specify the binding without using - * [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] - * if `setDefault` or `setBinding` are called. For example: - * {{{ - * class FrameworkModule extends TwitterModule { - * override protected def configure(): Unit = { - * bindOption[String, LookupUrl].setDefault.toInstance(DefaultLookupUrl) - * } - * } - * class UserLookupModule extends TwitterModule { - * override protected def configure(): Unit = { - * bind[String, LookupUrl].toInstance(CustomLookupUrl); - * } - * } - * }}} - * - * would generate an error, because both the framework and the user are trying to bind `@LookupUrl` String. - */ -abstract class TwitterModule - extends AbstractModule - with TwitterBaseModule - with ScalaModule - with Logging { - - /** - * Uses the given [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/Module.html com.google.inject.Module]] - * to configure more bindings. This is not supported for instances of type [[TwitterModule]] and - * will throw an [[UnsupportedOperationException]] if attempted on an instance of [[TwitterModule]]. - * This is to properly support the [[TwitterModuleLifecycle]] for [[TwitterModule]] instances. - * - * @note [[TwitterModule.install(module: Module)]] can still be used for non-[[TwitterModule]] instances, and is - * sometimes preferred due to `install` being deferred until after flag parsing occurs. - */ - @throws[UnsupportedOperationException] - override protected def install(module: Module): Unit = { - module match { - case _: TwitterModule => - throw new UnsupportedOperationException( - "Install not supported for TwitterModules. Please use 'override val modules = Seq(module1, module2, ...)'" - ) - case _ => - super.install(module) - } - } - - /* Protected */ - - /** - * Provides for installing and building a factory that combines the caller's arguments with - * injector-supplied values to construct objects. This is preferable to calling `install` - * on the [[TwitterModule]] which provides the factory as install is not supported for - * [[TwitterModule]] types. - * - * @tparam T type of the assisted injection factory - * - * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/assistedinject/AssistedInject.html com.google.inject.assistedinject.AssistedInject]] - * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/assistedinject/FactoryModuleBuilder.html com.google.inject.assistedinject.FactoryModuleBuilder]] - * @see [[https://github.com/google/guice/wiki/AssistedInject Assisted Injection]] - */ - protected def bindAssistedFactory[T: Manifest](): Unit = { - super.install(new FactoryModuleBuilder().build(manifest[T].runtimeClass)) - } - - /** - * Binds a type converter. The injector will use the given converter to - * convert string constants to matching types as needed. - * - * @param converter converts values - * @tparam T type to match that the converter can handle - */ - protected def addTypeConverter[T: Manifest](converter: TypeConverter): Unit = { - convertToTypes(Matchers.only(typeLiteral[T]), converter) - } - - /** - * Binds a type converter derived from a [[Flaggable]], making it possible to inject flags of all - * kinds. The injector will use a provided `Flaggable` to perform type conversion during an - * injection. - * - * For example (in Scala): - * - * {{{ - * addFlagConverter[List[(Int, Int)]] // support injecting flags of type List[(Int, Int)] - * }}} - * - * @see [[addFlagConverter(Matcher,Flaggable)]] for variant that's more suitable for Java. - */ - protected def addFlagConverter[T <: AnyRef: Manifest](implicit F: Flaggable[T]): Unit = { - addTypeConverter[T](new TypeConverter { - def convert(s: String, typeLiteral: TypeLiteral[_]): AnyRef = F.parse(s) - }) - } - - /** - * Binds a type converter derived from a [[Flaggable]], making it possible to inject flags of all - * kinds. The injector will use a provided `Flaggable` to perform type conversion during an - * injection. - * - * For example (in Java): - * - * {{{ - * import java.util.List; - * import com.google.inject.TypeLiteral; - * import com.google.inject.matcher.Matchers; - * import com.twitter.app.Flaggable; - * - * addFlagConverter( - * Matchers.only(new TypeLiteral>>() {}), - * Flaggable.ofJavaList(Flaggable.ofTuple(Flaggable.ofJavaInteger(), Flaggable.ofJavaInteger()) - * ); - * }}} - * - * @see [[addFlagConverter(Manifest,Flaggable]] for a variant that's more suitable for Scala. - */ - protected def addFlagConverter[T <: AnyRef]( - typeMatcher: Matcher[_ >: TypeLiteral[_]], - F: Flaggable[T] - ): Unit = { - convertToTypes( - typeMatcher, - new TypeConverter { - def convert(s: String, typeLiteral: TypeLiteral[_]): AnyRef = F.parse(s) - } - ) - } - - /** - * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] - * that collects instances of type [[T]] in a [[scala.collection.immutable.Set]] that is itself - * bound with no binding annotation. - * - * $multiBinding - * - * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] - */ - protected def bindMultiple[T: Manifest]: ScalaMultibinder[T] = - ScalaMultibinder.newSetBinder[T](binderAccess) - - /** - * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] - * that collects instances of type [[T]] in a [[scala.collection.immutable.Set]] that is itself bound with a binding - * annotation [[A]]. - * - * $multiBinding - * - * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] - */ - protected def bindMultiple[T: Manifest, A <: Annotation: Manifest]: ScalaMultibinder[T] = - ScalaMultibinder.newSetBinder[T, A](binderAccess) - - /** - * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] - * that collects instances of type [[T]] in a [[scala.collection.immutable.Set]] that is itself - * bound with a binding annotation. - * - * $multiBinding - * - * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/Multibinder.html com.google.inject.multibindings.Multibinder]] - */ - protected def bindMultiple[T: Manifest](annotation: Annotation): ScalaMultibinder[T] = - ScalaMultibinder.newSetBinder[T](binderAccess, annotation) - - /** - * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] - * that binds an instance of [[T]] in a [[scala.Option]]. - * - * $optionalBinding - * - * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] - */ - protected def bindOption[T: Manifest]: ScalaOptionBinder[T] = - ScalaOptionBinder.newOptionBinder[T](binderAccess) - - /** - * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] - * that binds an instance of [[T]] in a [[scala.Option]] that is itself bound with a binding - * annotation [[A]]. - * - * $optionalBinding - * - * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] - */ - protected def bindOption[T: Manifest, A <: Annotation: Manifest]: ScalaOptionBinder[T] = - ScalaOptionBinder.newOptionBinder[T, A](binderAccess) - - /** - * Returns a new [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] - * that binds an instance of [[T]] in a [[scala.Option]] that is itself bound with a - * binding annotation. - * - * $optionalBinding - * - * @see [[https://google.github.io/guice/api-docs/4.1/javadoc/com/google/inject/multibindings/OptionalBinder.html com.google.inject.multibindings.OptionalBinder]] - */ - protected def bindOption[T: Manifest](annotation: Annotation): ScalaOptionBinder[T] = - ScalaOptionBinder.newOptionBinder[T](binderAccess, annotation) -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala b/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala deleted file mode 100644 index 1511fa5556..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleFlags.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.twitter.inject - -import com.twitter.app.{Flag, Flaggable} -import scala.collection.mutable.ArrayBuffer - -/** [[com.twitter.app.Flag]] integration for a [[TwitterModule]]. */ -private[inject] trait TwitterModuleFlags { - - /* Mutable State */ - - protected[inject] lazy val flags: ArrayBuffer[Flag[_]] = ArrayBuffer[Flag[_]]() - - /* Protected */ - - /** - * This is akin to the `c.t.inject.app.App#failfastOnFlagsNotParsed` and serves - * a similar purpose but for new [[com.twitter.app.Flag]] instances created in this - * [[TwitterModule]]. The value is 'true' by default. This is to ensure that the value of - * a [[com.twitter.app.Flag]] instance created in this [[TwitterModule]] cannot be incorrectly - * accessed before the application has parsed any passed command line input. This mirrors the - * framework default for `com.twitter.inject.app.App#failfastOnFlagsNotParsed` for Flag - * instances created within the application container. - * - * @return a [[Boolean]] indicating if [[com.twitter.app.Flag]] instances created in - * this [[TwitterModule]] should be set with [[com.twitter.app.Flag.failFastUntilParsed]] - * set to 'true' or 'false'. Default: 'true'. - * @note This value SHOULD NOT be changed to 'false' without a very good reason. - */ - protected[this] def failfastOnFlagsNotParsed: Boolean = true - - /** - * A Java-friendly method for creating a named [[Flag]]. - * - * @param name the name of the [[Flag]]. - * @param default a default value for the [[Flag]] when no value is given as an application - * argument. - * @param help the help text explaining the purpose of the [[Flag]]. - * @return the created [[Flag]]. - */ - final def createFlag[T]( - name: String, - default: T, - help: String, - flaggable: Flaggable[T] - ): Flag[T] = { - val flag = - new Flag[T](name, help, default, failFastUntilParsed = failfastOnFlagsNotParsed)(flaggable) - flags += flag - flag - } - - /** - * A Java-friendly way to create a "mandatory" [[Flag]]. "Mandatory" flags MUST have a value - * provided as an application argument (as they have no default value to be used). - * - * @param name the name of the [[Flag]]. - * @param help the help text explaining the purpose of the [[Flag]]. - * @param usage a string describing the type of the [[Flag]], i.e.: Integer. - * @return the created [[Flag]]. - */ - final def createMandatoryFlag[T]( - name: String, - help: String, - usage: String, - flaggable: Flaggable[T] - ): Flag[T] = { - val flag = - new Flag[T](name, help, usage, failFastUntilParsed = failfastOnFlagsNotParsed)(flaggable) - flags += flag - flag - } - - /** - * Create a [[Flag]] and add it to this Module's flags list. - * - * @note Java users: see the more Java-friendly [[createFlag]] or [[createMandatoryFlag]]. - * - * @param name the name of the [[Flag]]. - * @param default a default value for the [[Flag]] when no value is given as an application - * argument. - * @param help the help text explaining the purpose of the [[Flag]]. - * @tparam T must be a [[Flaggable]] type. - * @return the created [[Flag]]. - */ - final def flag[T: Flaggable](name: String, default: T, help: String): Flag[T] = { - val flag = new Flag[T](name, help, default, failFastUntilParsed = failfastOnFlagsNotParsed) - flags += flag - flag - } - - /** - * Create a "mandatory" flag and add it to this Module's flags list."Mandatory" flags MUST have - * a value provided as an application argument (as they have no default value to be used). - * - * @note Java users: see the more Java-friendly [[createFlag]] or [[createMandatoryFlag]]. - * - * @param name the name of the [[Flag]]. - * @param help the help text explaining the purpose of the [[Flag]]. - * @tparam T must be a [[Flaggable]] type. - * @return the created [[Flag]]. - */ - final def flag[T: Flaggable: Manifest](name: String, help: String): Flag[T] = { - val flag = - new Flag[T]( - name, - help, - manifest[T].runtimeClass.toString, - failFastUntilParsed = failfastOnFlagsNotParsed) - flags += flag - flag - } -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleLifecycle.scala b/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleLifecycle.scala deleted file mode 100644 index a1c5a1bf55..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/TwitterModuleLifecycle.scala +++ /dev/null @@ -1,83 +0,0 @@ -package com.twitter.inject - -import com.twitter.util.logging.Logging -import scala.collection.mutable -import scala.util.control.NonFatal - -/** - * [[com.twitter.inject.TwitterModule]] to com.twitter.inject.app.App lifecycle integration. - */ -private[inject] trait TwitterModuleLifecycle extends Logging { - - /* Mutable State */ - private lazy val closableFunctions = mutable.Buffer[() => Unit]() - - /* Protected */ - - /** - * Invoked after the injector is started. - * - * This method should only get singleton instances from the injector. - */ - protected[inject] def singletonStartup(injector: Injector): Unit = {} - - /** - * Invoke after external ports are bound and any clients are resolved - * - * This method should only get singleton instances from the injector. - */ - protected[inject] def singletonPostWarmupComplete(injector: Injector): Unit = {} - - /** - * Invoked on graceful shutdown of the application. - * - * This method should only get singleton instances from the injector. - */ - protected[inject] def singletonShutdown(injector: Injector): Unit = {} - - /** - * Collects functions over a [[com.twitter.util.Closable]]s. These functions will be passed to - * the application `onExit` function to be executed on graceful shutdown of the application. - * - * @note It is expected that the passed function is a function over a [[com.twitter.util.Closable]]. - * - * @param f A Function0 which returns Unit. It is expected that this function encapsulates awaiting - * on a [[com.twitter.util.Closable]] that the application would like to ensure is closed - * upon graceful shutdown. - * - * {{{ - * closeOnExit { - * val closable = ... - * Await.result( - * closable.close(after: Duration), timeout: Duration) - * } - * }}} - * - * @see [[com.twitter.app.App.onExit(f: => Unit)]] - * @see [[com.twitter.util.Awaitable]] - * @see [[com.twitter.util.Closable]] - */ - protected def closeOnExit(f: => Unit): Unit = { - // `c.t.app.App#onExit` will swallow any exception from a given function, thus we try to ensure - // some visibility into any potential exception resulting from executing the passed function. - def tryCatchFn(fn: => Unit): () => Unit = { () => - { - try { - fn - } catch { - case NonFatal(e) => - warn(e.getMessage) - debug(s"An error occurred in ${getClass.getSimpleName}#closeOnExit", e) - } - } - } - - closableFunctions += tryCatchFn(f) - } - - /* Private */ - - private[inject] def close(): Unit = { - closableFunctions.foreach(_()) - } -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala b/util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala deleted file mode 100644 index 83af0d05b3..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/TypeUtils.scala +++ /dev/null @@ -1,76 +0,0 @@ -package com.twitter.inject - -import com.google.inject.TypeLiteral -import com.google.inject.internal.MoreTypes.ParameterizedTypeImpl -import scala.reflect.api.TypeCreator -import scala.reflect.runtime.universe._ -import scala.reflect.{ClassTag, ManifestFactory} - -private[inject] object TypeUtils { - - def singleTypeParam[T](objType: java.lang.reflect.Type): java.lang.reflect.Type = objType match { - case parametricType: ParameterizedTypeImpl => - parametricType.getActualTypeArguments.head - } - - def superTypeFromClass(clazz: Class[_], superClazz: Class[_]): java.lang.reflect.Type = { - TypeLiteral.get(clazz).getSupertype(superClazz).getType - } - - /** - * Convert a [[TypeTag]] to a [[Manifest]]. Recursively attempts to - * convert any type arguments from the given [[TypeTag]] to use for - * creating a Manifest[T]. - * - * @tparam T - the [[TypeTag]] to convert - * @return a Manifest[T] representation from the given type [T]. - */ - def asManifest[T: TypeTag]: Manifest[T] = { - val t = typeTag[T] - val mirror = t.mirror - def manifestFromType(t: Type): Manifest[_] = { - t match { - case n if n =:= typeOf[Nothing] => ManifestFactory.Nothing - case n if n =:= typeOf[Null] => ManifestFactory.Null - case n if n =:= typeOf[Any] => ManifestFactory.Any - case n if n =:= typeOf[AnyVal] => ManifestFactory.AnyVal - case _ => - try { - val clazz = ClassTag[T](mirror.runtimeClass(t)).runtimeClass - if (t.typeArgs.length == 1) { - val arg = manifestFromType(t.typeArgs.head) - ManifestFactory.classType(clazz, arg) - } else if (t.typeArgs.length > 1) { - // recursively walk each type arg to create a Manifest - val args = t.typeArgs.map(manifestFromType) - ManifestFactory.classType(clazz, args.head, args.tail: _*) - } else { - ManifestFactory.classType(clazz) - } - } catch { - case _: NoClassDefFoundError => ManifestFactory.Any - } - } - } - manifestFromType(t.tpe).asInstanceOf[Manifest[T]] - } - - /** - * Convert from the given Class[T] to a TypeTag[T] - * @param clazz the class for which to build the resultant TypeTag - * @return a TypeTag[T] representing the given Class[T] - */ - def asTypeTag[T](clazz: Class[_ <: T]): TypeTag[T] = { - val clazzMirror = runtimeMirror(clazz.getClassLoader) - val tpe = clazzMirror.classSymbol(clazz).toType - val typeCreator = new TypeCreator() { - def apply[U <: scala.reflect.api.Universe with scala.Singleton]( - m: scala.reflect.api.Mirror[U] - ): U#Type = { - if (clazzMirror != m) throw new RuntimeException("wrong mirror") - else tpe.asInstanceOf[U#Type] - } - } - TypeTag[T](clazzMirror, typeCreator) - } -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD.bazel b/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD.bazel deleted file mode 100644 index 828882d060..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/internal/BUILD.bazel +++ /dev/null @@ -1,25 +0,0 @@ -scala_library( - sources = ["*.scala"], - compiler_option_sets = ["fatal_warnings"], - provides = scala_artifact( - org = "com.twitter", - name = "util-inject-internal", - repo = artifactory, - ), - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/javax/inject:javax.inject", - "util/util-app/src/main/scala", - "util/util-core:util-core-util", - "util/util-inject/src/main/java/com/twitter/util/inject/flag", - "util/util-inject/src/main/scala/com/twitter/util/inject", - "util/util-slf4j-api/src/main/scala", - ], - exports = [ - "util/util-app/src/main/scala", - "util/util-inject/src/main/java/com/twitter/util/inject/flag", - "util/util-slf4j-api/src/main/scala", - ], -) diff --git a/util-inject/src/main/scala/com/twitter/util/inject/internal/FlagsModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/FlagsModule.scala deleted file mode 100644 index e7d2057883..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/internal/FlagsModule.scala +++ /dev/null @@ -1,69 +0,0 @@ -package com.twitter.inject.internal - -import com.google.inject.{AbstractModule, Key} -import com.twitter.util.inject.flag.Flags -import com.twitter.util.logging.Logging -import javax.inject.Provider - -/** - * Note this is purposely *not* an instance of a [[com.twitter.inject.TwitterModule]] - * as we use no lifecycle callbacks nor does this define any [[com.twitter.app.Flag]]s. - * Additionally, we do not want mistakenly use or bind the [[com.twitter.inject.TwitterModule#flags]] - * reference thus we extend [[AbstractModule]] directly. - * - * This module is solely intended to create object graph bindings for the parsed value of - * every non-global [[com.twitter.app.Flag]] contained in the passed [[com.twitter.app.Flags]] instance. - * If the [[com.twitter.app.Flag]] instance does not have a parsed value a `Provider[Nothing]` - * is provided which when de-referenced will throw an [[IllegalArgumentException]]. - */ -private[inject] class FlagsModule(flags: com.twitter.app.Flags) - extends AbstractModule - with Logging { - - /** - * Currently, we are unable to bind to anything other than a [[String]] type as Flags do not carry - * enough type information due to type erasure. We would need to update [[com.twitter.app.Flag]] - * or [[com.twitter.app.Flaggable]] to carry Manifest or TypeTag information which we could use - * here to determine the correct type for building an instance Key. Or re-create the - * [[com.twitter.app.Flaggable]] logic in a registered TypeConverter. - * - * In the interim, we install the Guice default type conversions along with the [[TwitterTypeConvertersModule]] - * to provide a limited set of conversions by the injector from [[String]] to another type. Thus - * you can inject a [[com.twitter.app.Flag]] of some non-[[String]] types. But this is not all of - * the same types supported by [[com.twitter.app.Flaggable]] and [[com.twitter.app.Flag]]. Some - * mismatch in the functionality is expected as the integration is done manually. That said, - * defining a custom `Flaggable` instance for type X doesn't guarantee it can be injected without - * registering a flag converter. - * - * @see [[TwitterTypeConvertersModule]] - * @see [[https://github.com/google/guice/blob/master/core/src/com/google/inject/internal/TypeConverterBindingProcessor.java Guice Default Type Conversions]] - */ - override def configure(): Unit = { - // bind the c.t.inject.Flags instance to the object graph - binder.bind(classOf[com.twitter.inject.Flags]).toInstance(com.twitter.inject.Flags(flags)) - - // bind every parsed (non-global) flag that has a value keyed by flag name - val asList = flags.getAll(includeGlobal = false).toSeq - val flagsMap = (for (flag <- asList) yield { - flag.name -> flag.getWithDefaultUnparsed - }).toMap - - for ((flagName, valueOpt) <- flagsMap) { - val key: Key[String] = Flags.key(flagName) - valueOpt match { - case Some(value) => - debug("Binding flag: " + flagName + " = " + value) - binder.bind(key).toInstance(value) - case None => - binder - .bind(key) - .toProvider(new Provider[Nothing] { - override def get() = - throw new IllegalArgumentException( - "flag: " + flagName + " has an unspecified value and is not eligible for @Flag injection" - ) - }) - } - } - } -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/internal/InstalledModules.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/InstalledModules.scala deleted file mode 100644 index 0faaf1cba2..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/internal/InstalledModules.scala +++ /dev/null @@ -1,82 +0,0 @@ -package com.twitter.inject.internal - -import com.google.inject.Module -import com.twitter.inject.{Injector, TwitterModuleLifecycle} -import com.twitter.util.logging.Logging -import scala.PartialFunction.condOpt - -/** - * A representation of the "installed" [[com.google.inject.Module]]s, that is - * the created [[com.twitter.inject.Injector]] and the [[com.google.inject.Module]] of which it - * is comprised. Creating the [[com.twitter.inject.Injector]] loses the [[com.google.inject.Module]]s - * over which it was created, but we need to hold onto the list in order to apply the application lifecycle - * to the [[com.twitter.inject.TwitterModuleLifecycle]] versions of the contained [[com.google.inject.Module]]s. - * - * @param injector the [[com.twitter.inject.Injector]] built from the [[com.google.inject.Module]]s - * @param modules the list of [[com.google.inject.Module]]s from which the [[com.twitter.inject.Injector]] was created. - */ -private[inject] case class InstalledModules(injector: Injector, modules: Seq[Module]) - extends Logging { - - def postInjectorStartup(): Unit = { - modules.foreach { - case injectModule: TwitterModuleLifecycle => - try { - injectModule.singletonStartup(injector) - } catch { - case e: Throwable => - error("Startup method error in " + injectModule, e) - throw e - } - case _ => - } - } - - def postWarmupComplete(): Unit = { - modules.foreach { - case injectModule: TwitterModuleLifecycle => - try { - injectModule.singletonPostWarmupComplete(injector) - } catch { - case e: Throwable => - error("Post warmup complete method error in " + injectModule, e) - throw e - } - case _ => - } - } - - /** - * Collect shutdown `ExitFunctions` for [[com.google.inject.Module]] instances - * which implement the [[TwitterModuleLifecycle]] - */ - def shutdown(): Seq[ExitFunction] = { - condOptModules(modules)(_.singletonShutdown(injector)) - } - - /** - * Collect close `ExitFunctions` for [[com.google.inject.Module]] instances which - * implement the [[TwitterModuleLifecycle]] - */ - def close(): Seq[ExitFunction] = { - condOptModules(modules)(_.close()) - } - - /* Private */ - - /** - * Iterates through the list of Modules to match only instances of TwitterModuleLifecycle - * on which to create an `ExitFunction` over the passed in TwitterModuleLifecycle function. - * @see [[scala.PartialFunction.condOpt]] - */ - private[this] def condOptModules( - modules: Seq[Module] - )( - fn: TwitterModuleLifecycle => Unit - ): Seq[ExitFunction] = modules.flatMap { module => - condOpt(module) { - case injectModule: TwitterModuleLifecycle => - () => fn(injectModule) - } - } -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/internal/Modules.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/Modules.scala deleted file mode 100644 index 21944b0d18..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/internal/Modules.scala +++ /dev/null @@ -1,113 +0,0 @@ -package com.twitter.inject.internal - -import com.google.inject.{Guice, Module, Stage} -import com.twitter.app.{Flag, Flags} -import com.twitter.inject.{Injector, TwitterBaseModule} -import com.twitter.inject.internal.iterable._ -import scala.collection.JavaConverters._ - -private[inject] object Modules { - - /** - * De-duplicate a given sequence of [[com.google.inject.Module]]. - * Exposed for testing. - */ - def distinctModules( - modules: Seq[com.google.inject.Module] - ): Seq[com.google.inject.Module] = { - // De-dupe using a `java.util.IdentityHashMap` which uses reference equality instead - // of object equality to check if both objects point to the same memory location - // and filter out modules already seen. We use `filter` because it's stable, and - // the order of module initialization may be important. - val identityHashMap = new java.util.IdentityHashMap[com.google.inject.Module, Boolean]() - modules.filter { module => - if (identityHashMap.containsKey(module)) false - else { - identityHashMap.put(module, true) - true - } - } - } - - /** Capture all flags in the [[com.google.inject.Module]] object hierarchy. */ - private def findModuleFlags(modules: Seq[com.google.inject.Module]): Seq[Flag[_]] = { - // Flags are stored in the App `com.twitter.app.Flags` member variable which is a - // Map[String, Flag[_]], where the key is the Flag#name. Thus, to ensure we correctly account - // for the "override" behavior of Flag#add (the last Flag with the same name added, wins), - // we need to ensure that we "reverse" our Sequence before performing a distinct operation - // discriminated by Flag#name. - modules - .collect { - case injectModule: TwitterBaseModule => injectModule.flags - }.flatten.reverse.distinctBy(_.name) - } - - /** Recursively find all 'composed' modules */ - private def findInstalledModules( - module: com.google.inject.Module - ): Seq[com.google.inject.Module] = module match { - case injectModule: TwitterBaseModule => - injectModule.modules ++ - injectModule.modules.flatMap(findInstalledModules) ++ - injectModule.javaModules.asScala.toSeq ++ - injectModule.javaModules.asScala.toSeq.flatMap(findInstalledModules) ++ - injectModule.frameworkModules ++ - injectModule.frameworkModules.flatMap(findInstalledModules) - case _ => - Seq.empty - } - -} - -/** - * Provides a flattened view of the given lists of [[Module]]s. That is if the [[Module]] is - * a [[[com.twitter.inject.TwitterBaseModule]] type it may reference other [[Module]]s to - * be included when creating the injector. This class will perform the recursive lookup of - * all the referenced [[Module]]s from the give input lists and store each as a flattened - * sequence to be used for injector creation. - * - * We only want to compute all of the composed modules and the flags contained therein - * once as calling the [[com.twitter.inject.TwitterBaseModule#modules]] or - * [[com.twitter.inject.TwitterBaseModule#javaModules]] or - * [[com.twitter.inject.TwitterBaseModule#frameworkModules]] methods is not guaranteed - * to be idempotent. This class is a holder of the computed modules and flags for - * the containing [[com.twitter.inject.app.App]]. - * - * @see [[com.twitter.inject.TwitterBaseModule]] - * @see [[com.twitter.inject.TwitterBaseModule#modules]] - * @see [[com.twitter.inject.TwitterBaseModule#javaModules]] - * @see [[com.twitter.inject.TwitterBaseModule#frameworkModules]] - */ -private[inject] class Modules(required: Seq[Module], overrides: Seq[Module]) { - import Modules._ - - val modules: Seq[Module] = { - val composedModules = required.flatMap(findInstalledModules) - required ++ composedModules - } - - val overrideModules: Seq[Module] = { - val composedOverrideModules = overrides.flatMap(findInstalledModules) - overrides ++ composedOverrideModules - } - - val moduleFlags: Seq[Flag[_]] = - findModuleFlags(distinctModules(modules ++ overrideModules)) - - def addFlags(flag: Flags): Unit = moduleFlags.foreach(flag.add) - - def install( - flags: Flags, - stage: Stage - ): InstalledModules = { - // ensure we add the FlagsModule and the TwitterTypeConvertersModule to the list to build the injector. - val requiredModules = modules ++ Seq(new FlagsModule(flags), TwitterTypeConvertersModule) - val combinedModule = com.google.inject.util.Modules - .`override`(requiredModules.asJava).`with`(overrideModules.asJava) - - InstalledModules( - injector = Injector(Guice.createInjector(stage, combinedModule)), - modules = distinctModules(requiredModules ++ overrideModules) - ) - } -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/internal/TwitterTypeConvertersModule.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/TwitterTypeConvertersModule.scala deleted file mode 100644 index 641d2a2c58..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/internal/TwitterTypeConvertersModule.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.inject.internal - -import com.twitter.inject.TwitterModule -import com.twitter.util.{Duration, StorageUnit, Time} -import java.io.File -import java.net.InetSocketAddress -import java.time.LocalTime - -private[inject] object TwitterTypeConvertersModule extends TwitterModule { - - override def configure(): Unit = { - // Single value type converters - addFlagConverter[LocalTime] - addFlagConverter[Time] - addFlagConverter[Duration] - addFlagConverter[StorageUnit] - addFlagConverter[InetSocketAddress] - addFlagConverter[File] - - // Multi-value type converters (comma-separated) for scala.Seq - addFlagConverter[Seq[String]] - addFlagConverter[Seq[Int]] - addFlagConverter[Seq[Long]] - addFlagConverter[Seq[Double]] - addFlagConverter[Seq[Float]] - addFlagConverter[Seq[LocalTime]] - addFlagConverter[Seq[Time]] - addFlagConverter[Seq[Duration]] - addFlagConverter[Seq[StorageUnit]] - addFlagConverter[Seq[InetSocketAddress]] - addFlagConverter[Seq[File]] - - // Multi-value type converters (comma-separated) for java.util.List - addFlagConverter[java.util.List[String]] - addFlagConverter[java.util.List[Int]] - addFlagConverter[java.util.List[Long]] - addFlagConverter[java.util.List[Double]] - addFlagConverter[java.util.List[Float]] - addFlagConverter[java.util.List[LocalTime]] - addFlagConverter[java.util.List[Time]] - addFlagConverter[java.util.List[Duration]] - addFlagConverter[java.util.List[StorageUnit]] - addFlagConverter[java.util.List[InetSocketAddress]] - addFlagConverter[java.util.List[File]] - } -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/internal/iterable.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/iterable.scala deleted file mode 100644 index c5128d8156..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/internal/iterable.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.twitter.inject.internal - -import scala.collection.generic.CanBuildFrom -import scala.collection.mutable.{HashSet => MutableHashSet} -import scala.language.higherKinds - -private[inject] object iterable { - - implicit class RichIterable[Elem, From[Elem] <: Iterable[Elem]](val self: From[Elem]) - extends AnyVal { - - /** - * Distinct 'iterable' elements using the passed in 'hash' function - * @param hash Hash function to determine unique elements - * @return Distinct elements - */ - def distinctBy[HashCodeType]( - hash: Elem => HashCodeType - )( - implicit cbf: CanBuildFrom[From[Elem], Elem, From[Elem]] - ): From[Elem] = { - val builder = cbf() - val seen = MutableHashSet[HashCodeType]() - - for (elem <- self) { - if (!seen(hash(elem))) { - seen += hash(elem) - builder += elem - } - } - - builder.result() - } - - } - -} diff --git a/util-inject/src/main/scala/com/twitter/util/inject/internal/package.scala b/util-inject/src/main/scala/com/twitter/util/inject/internal/package.scala deleted file mode 100644 index d121a18bec..0000000000 --- a/util-inject/src/main/scala/com/twitter/util/inject/internal/package.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.twitter.inject - -package object internal { - type ExitFunction = () => Unit -} diff --git a/util-inject/src/test/java/com/twitter/util/inject/BUILD.bazel b/util-inject/src/test/java/com/twitter/util/inject/BUILD.bazel deleted file mode 100644 index f745fa1628..0000000000 --- a/util-inject/src/test/java/com/twitter/util/inject/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -java_library( - sources = ["*.java"], - compiler_option_sets = [], - platform = "java8", - strict_deps = True, - tags = ["bazel-compatible"], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - ], -) diff --git a/util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java b/util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java deleted file mode 100644 index ba1219d18b..0000000000 --- a/util-inject/src/test/java/com/twitter/util/inject/TestBindingAnnotation.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.twitter.inject; - -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import com.google.inject.BindingAnnotation; - -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - -/** - * HELPER ANNOTATION FOR TESTS. - *

    - * Note: This is not to be included in any published test:jar. - */ -@BindingAnnotation -@Target(PARAMETER) -@Retention(RUNTIME) -public @interface TestBindingAnnotation {} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/BUILD.bazel b/util-inject/src/test/scala/com/twitter/util/inject/BUILD.bazel deleted file mode 100644 index f89fd93c60..0000000000 --- a/util-inject/src/test/scala/com/twitter/util/inject/BUILD.bazel +++ /dev/null @@ -1,22 +0,0 @@ -junit_tests( - sources = ["**/*.scala"], - compiler_option_sets = ["fatal_warnings"], - platform = "java8", - strict_deps = True, - tags = [ - "bazel-compatible", - "non-exclusive", - ], - dependencies = [ - "3rdparty/jvm/com/google/inject:guice", - "3rdparty/jvm/com/google/inject/extensions:guice-testlib", - "3rdparty/jvm/javax/inject:javax.inject", - "3rdparty/jvm/junit", - "3rdparty/jvm/net/codingwell:scala-guice", - "3rdparty/jvm/org/scalatest", - "3rdparty/jvm/org/scalatestplus:junit", - "3rdparty/jvm/org/scalatestplus:scalacheck", - "util/util-inject/src/main/scala/com/twitter/util/inject", - "util/util-inject/src/test/java/com/twitter/util/inject", - ], -) diff --git a/util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala b/util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala deleted file mode 100644 index b58bb9ee6b..0000000000 --- a/util-inject/src/test/scala/com/twitter/util/inject/InjectorTest.scala +++ /dev/null @@ -1,195 +0,0 @@ -package com.twitter.inject - -import com.google.inject._ -import com.google.inject.name.Names -import com.twitter.inject.classes.{TypeProvider, _} -import com.twitter.inject.module.{SeqAbstractModule, TestModule} -import javax.inject.{Named, Singleton} -import net.codingwell.scalaguice.ScalaModule -import org.junit.runner.RunWith -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers -import org.scalatestplus.junit.JUnitRunner - -object InjectorTest { - type AugmentedSomeClient = SomeClient with Augmentation -} - -@RunWith(classOf[JUnitRunner]) -class InjectorTest extends AnyFunSuite with Matchers { - import InjectorTest._ - - private case class W[T](t: T) - // Users should prefer the `c.t.inject.app.TestInjector` but it is not in scope here. - private val injector: Injector = Injector(Guice.createInjector(new TestModule {})) - - test("byte array generics") { - import net.codingwell.scalaguice.InjectorExtensions._ - val i = Guice.createInjector(SeqAbstractModule) - - i.instance[Seq[Array[Byte]]] should equal(Seq.empty[Array[Byte]]) - } - - test("can't bind class with trait and class") { - intercept[CreationException] { - Guice.createInjector( - new AbstractModule with ScalaModule { - @Provides - @Singleton - def provideSomeClient: SomeClient with Augmentation = { - new SomeClient with Augmentation - } - }, - new AbstractModule with ScalaModule { - @Provides - @Singleton - def provideSomeClient: SomeClient = { - new SomeClient - } - } - ) - } - } - - test("class with trait type") { - import net.codingwell.scalaguice.InjectorExtensions._ - val i = Guice.createInjector( - Stage.PRODUCTION, - new AbstractModule with ScalaModule { - @Provides - @Singleton - def provideSomeClient: SomeClient with Augmentation = { - new SomeClient with Augmentation - } - }) - - val r1 = i.instance[SomeClient with Augmentation] - r1.get should equal("Hello, Alice. Welcome to Wonderland.") - - val r2 = i.instance[AugmentedSomeClient] - r2.get should equal("Hello, Alice. Welcome to Wonderland.") - } - - test("class only type") { - import net.codingwell.scalaguice.InjectorExtensions._ - val i = Guice.createInjector( - Stage.PRODUCTION, - new AbstractModule with ScalaModule { - @Provides - @Singleton - def provideSomeClient: SomeClient = { - new SomeClient - } - }) - - // fails since only SomeClient is bound - val e = intercept[ProvisionException] { - i.instance[Baz] - } - e.getCause.isInstanceOf[ClassCastException] should be(true) - } - - test("get additional instances from injector") { - injector.instance[String](Names.named("str1")) should be("string1") - injector.instance[String, TestBindingAnnotation] should be("prod string") - injector.instance[String](Names.named("name")) should equal("Steve") - } - - test("allow binding target type using a type parameter") { - val module = new TestModule { - override protected def configure(): Unit = { - bind[X].to[Y] - } - } - - Injector(Guice.createInjector(module)).instance[X] - } - - test("allow binding target provider type using a type parameter") { - val module = new TestModule { - override protected def configure(): Unit = { - bind[X].toProvider[YProvider] - } - } - - Injector(Guice.createInjector(module)).instance[X] - } - - test("allow binding to provider of subtype using type parameter") { - val module = new TestModule { - override protected def configure(): Unit = { - bind[Gen[String]].toProvider[ZProvider] - } - } - - Injector(Guice.createInjector(module)).instance(new Key[Gen[String]] {}) - } - - test("allow binding to provider with injected type literal") { - val module = new TestModule { - override protected def configure(): Unit = { - bind[String].toProvider[TypeProvider[B]] - } - } - - Injector(Guice.createInjector(module)).instance(new Key[String] {}) - } - - test("allow binding in scope using a type parameter") { - val module = new TestModule { - override protected def configure(): Unit = { - bind[X].to[Y].in[Singleton] - } - } - - Injector(Guice.createInjector(module)).instance[X] - } - - test("allow binding a container with a generic singleton type") { - val module = new TestModule { - override protected def configure(): Unit = { - bind[SealedTraitContainer[FinalSealedTrait.type]] - .toProvider[SealedTraitContainerFinalSealedTraitProvider] - } - } - - Injector(Guice.createInjector(module)) - .instance(new Key[SealedTraitContainer[FinalSealedTrait.type]] {}) - } - - test("allow binding with annotation using a type parameter") { - val module = new TestModule { - override protected def configure(): Unit = { - bind[X].annotatedWith[Named].to[Y] - } - } - - Injector(Guice.createInjector(module)).instance(Key.get(classOf[X], classOf[Named])) - } - - test("allow use provider form javax.inject.Provider") { - val module = new TestModule { - override def configure(): Unit = { - bind[Foo].toProvider[FooProviderWithJavax] - } - } - - Injector(Guice.createInjector(module)).instance[Foo] - } - - test("provide helpful error when bound to self") { - val module = new TestModule { - override def configure(): Unit = { - bind[X].to[X] - } - } - val e = intercept[CreationException] { - Injector(Guice.createInjector(module)).instance[X] - } - val messages = e.getErrorMessages - messages.size should equal(1) - e.getStackTrace.exists(_.getClassName == this.getClass.getName) - val sources = messages.iterator.next.getSource - sources.contains(this.getClass.getName) should be(true) - } -} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/TwitterModuleTest.scala b/util-inject/src/test/scala/com/twitter/util/inject/TwitterModuleTest.scala deleted file mode 100644 index ef3159ee37..0000000000 --- a/util-inject/src/test/scala/com/twitter/util/inject/TwitterModuleTest.scala +++ /dev/null @@ -1,543 +0,0 @@ -package com.twitter.inject - -import com.google.inject._ -import com.google.inject.name.Names -import com.twitter.inject.servicefactory.ComplexServiceFactory -import com.twitter.inject.classes._ -import com.twitter.inject.module.{ClassToConvert, SeqAbstractModule, TestTwitterModule} -import java.lang.annotation.Annotation -import javax.inject.{Named, Provider, Singleton} -import net.codingwell.scalaguice.ScalaModule -import org.scalatest.funsuite.AnyFunSuite -import org.scalatest.matchers.should.Matchers -import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} -import scala.collection.immutable -import scala.util.Random - -object TwitterModuleTest { - type AugmentedSomeClient = SomeClient with Augmentation -} - -class TwitterModuleTest - extends AnyFunSuite - with Matchers - with BeforeAndAfterAll - with BeforeAndAfterEach { - import TwitterModuleTest._ - - private case class W[T](t: T) - private val annotation = Names.named(Random.alphanumeric.take(12).mkString.toLowerCase()) - - // Users should prefer the `c.t.inject.app.TestInjector` but it is not in scope here. - private val injector: Injector = Injector(Guice.createInjector(TestTwitterModule)) - - override def beforeAll(): Unit = { - super.beforeAll() - TestTwitterModule.singletonStartup(injector) - } - - override def afterAll(): Unit = { - TestTwitterModule.singletonShutdown(injector) - super.afterAll() - } - - test("byte array generics") { - import net.codingwell.scalaguice.InjectorExtensions._ - val i = Guice.createInjector(SeqAbstractModule) - - i.instance[Seq[Array[Byte]]] should equal(Seq.empty[Array[Byte]]) - } - - test("can't bind class with trait and class") { - intercept[CreationException] { - Guice.createInjector( - new AbstractModule with ScalaModule { - @Provides - @Singleton - def provideSomeClient: SomeClient with Augmentation = { - new SomeClient with Augmentation - } - }, - new AbstractModule with ScalaModule { - @Provides - @Singleton - def provideSomeClient: SomeClient = { - new SomeClient - } - } - ) - } - } - - test("class with trait type") { - import net.codingwell.scalaguice.InjectorExtensions._ - val i = Guice.createInjector( - Stage.PRODUCTION, - new AbstractModule with ScalaModule { - @Provides - @Singleton - def provideSomeClient: SomeClient with Augmentation = { - new SomeClient with Augmentation - } - }) - - val r1 = i.instance[SomeClient with Augmentation] - r1.get should equal("Hello, Alice. Welcome to Wonderland.") - - val r2 = i.instance[AugmentedSomeClient] - r2.get should equal("Hello, Alice. Welcome to Wonderland.") - } - - test("class only type") { - import net.codingwell.scalaguice.InjectorExtensions._ - val i = Guice.createInjector( - Stage.PRODUCTION, - new AbstractModule with ScalaModule { - @Provides - @Singleton - def provideSomeClient: SomeClient = { - new SomeClient - } - }) - - // fails since only SomeClient is bound - val e = intercept[ProvisionException] { - i.instance[Baz] - } - e.getCause.isInstanceOf[ClassCastException] should be(true) - } - - test("get assisted factory instance from injector") { - assertServiceFactory(injector.instance[ComplexServiceFactory]) - assertServiceFactory(injector.instance(classOf[ComplexServiceFactory])) - assertServiceFactory(injector.instance(Key.get(classOf[ComplexServiceFactory]))) - } - - test("get additional instances from injector") { - injector.instance[String](Names.named("str1")) should be("string1") - injector.instance[String, TestBindingAnnotation] should be("prod string") - } - - test("type conversion") { - injector.instance[ClassToConvert](Names.named("name")) should be(ClassToConvert("Steve")) - } - - test("allow binding target type using a type parameter") { - val module = new TwitterModule { - override protected def configure(): Unit = { - bind[X].to[Y] - } - } - - Injector(Guice.createInjector(module)).instance[X] - } - - test("allow binding target provider type using a type parameter") { - val module = new TwitterModule { - override protected def configure(): Unit = { - bind[X].toProvider[YProvider] - } - } - - Injector(Guice.createInjector(module)).instance[X] - } - - test("allow binding to provider of subtype using type parameter") { - val module = new TwitterModule { - override protected def configure(): Unit = { - bind[Gen[String]].toProvider[ZProvider] - } - } - - Injector(Guice.createInjector(module)).instance(new Key[Gen[String]] {}) - } - - test("allow binding to provider with injected type literal") { - val module = new TwitterModule { - override protected def configure(): Unit = { - bind[String].toProvider[TypeProvider[B]] - } - } - - Injector(Guice.createInjector(module)).instance(new Key[String] {}) - } - - test("allow binding in scope using a type parameter") { - val module = new TwitterModule { - override protected def configure(): Unit = { - bind[X].to[Y].in[Singleton] - } - } - - Injector(Guice.createInjector(module)).instance[X] - } - - test("allow binding a container with a generic singleton type") { - val module = new TwitterModule { - override protected def configure(): Unit = { - bind[SealedTraitContainer[FinalSealedTrait.type]] - .toProvider[SealedTraitContainerFinalSealedTraitProvider] - } - } - - Injector(Guice.createInjector(module)) - .instance(new Key[SealedTraitContainer[FinalSealedTrait.type]] {}) - } - - test("allow binding with annotation using a type parameter") { - val module = new TwitterModule { - override protected def configure(): Unit = { - bind[X].annotatedWith[Named].to[Y] - } - } - - Injector(Guice.createInjector(module)).instance(Key.get(classOf[X], classOf[Named])) - } - - test("allow use provider form javax.inject.Provider") { - val module = new TwitterModule { - override def configure(): Unit = { - bind[Foo].toProvider[FooProviderWithJavax] - } - } - - Injector(Guice.createInjector(module)).instance[Foo] - } - - test("provide helpful error when bound to self") { - val module = new TwitterModule { - override def configure(): Unit = { - bind[X].to[X] - } - } - val e = intercept[CreationException] { - Injector(Guice.createInjector(module)).instance[X] - } - val messages = e.getErrorMessages - messages.size should equal(1) - e.getStackTrace.exists(_.getClassName == this.getClass.getName) - val sources = messages.iterator.next.getSource - sources.contains(this.getClass.getName) should be(true) - } - - test("multibinder#bind empty [T]") { - val module = new TwitterModule { - override def configure(): Unit = { - bindMultiple[String] - } - } - validate(Injector(Guice.createInjector(module)).instance[immutable.Set[String]]) - } - - test("multibinder#bind [T]") { - val module = new TwitterModule { - override def configure(): Unit = { - bindMultiple[String].addBinding.toInstance("A") - bindMultiple[String].addBinding.toInstance("B") - } - } - validate(Injector(Guice.createInjector(module)).instance[immutable.Set[String]], "A", "B") - } - - test("multibinder#bind [T, Ann]") { - val module = new TwitterModule { - override def configure(): Unit = { - val multi = bindMultiple[String, TestBindingAnnotation] - multi.addBinding.toInstance("A") - multi.addBinding.toInstance("B") - } - } - validate( - Injector(Guice.createInjector(module)).instance[immutable.Set[String], TestBindingAnnotation], - "A", - "B") - } - - test("multibinder#bind [T, Annotation]") { - val module = new TwitterModule { - override def configure(): Unit = { - val multi = bindMultiple[String](annotation) - multi.addBinding.toInstance("A") - multi.addBinding.toInstance("B") - } - } - validate( - Injector(Guice.createInjector(module)).instance[immutable.Set[String]](annotation), - "A", - "B") - } - - test("multibinder#deduplicate") { - val module = new TwitterModule { - override def configure(): Unit = { - bindMultiple[Symbol].addBinding.toInstance('A) - bindMultiple[Symbol].addBinding.toInstance('A) - } - } - validate(Injector(Guice.createInjector(module)).instance[immutable.Set[Symbol]], 'A) - } - - test("multibinder#permit duplicates") { - val module = new TwitterModule { - override def configure(): Unit = { - val multi = bindMultiple[Symbol].permitDuplicates() // silently discards duplicates - multi.addBinding.toInstance('A) - multi.addBinding.toInstance('A) - } - } - validate(Injector(Guice.createInjector(module)).instance[immutable.Set[Symbol]], 'A) - } - - test("multibinder#bind from multiple modules") { - def newModule(i: Int) = new TwitterModule { - override def configure(): Unit = { - val multi = bindMultiple[Int] - multi.addBinding.toInstance(i) - } - } - - validate( - Injector(Guice.createInjector(newModule(1), newModule(2))).instance[immutable.Set[Int]], - 1, - 2) - } - - test("multibinder#bind parameterized [T]") { - val module = new TwitterModule { - override def configure(): Unit = { - val mbStrings = bindMultiple[W[String]] - mbStrings.addBinding.toInstance(W("A")) - val mbInts = bindMultiple[W[Int]] - mbInts.addBinding.toInstance(W(1)) - } - } - - val injector = Injector(Guice.createInjector(module)) - validate(injector.instance[immutable.Set[W[String]]], W("A")) - validate(injector.instance[immutable.Set[W[Int]]], W(1)) - } - - test("multibinder#bind parameterized [T, Ann]") { - val module = new TwitterModule { - override def configure(): Unit = { - val mbStrings = bindMultiple[W[String], TestBindingAnnotation] - mbStrings.addBinding.toInstance(W("A")) - val mbInts = bindMultiple[W[Int], TestBindingAnnotation] - mbInts.addBinding.toInstance(W(1)) - } - } - - val injector = Injector(Guice.createInjector(module)) - validate(injector.instance[immutable.Set[W[String]], TestBindingAnnotation], W("A")) - validate(injector.instance[immutable.Set[W[Int]], TestBindingAnnotation], W(1)) - } - - test("multibinder#bind parameterized [T, Annotation]") { - val module = new TwitterModule { - override def configure(): Unit = { - val mbStrings = bindMultiple[W[String]](annotation) - mbStrings.addBinding.toInstance(W("A")) - val mbInts = bindMultiple[W[Int]](annotation) - mbInts.addBinding.toInstance(W(1)) - } - } - - val injector = Injector(Guice.createInjector(module)) - validate(injector.instance[immutable.Set[W[String]]](annotation), W("A")) - validate(injector.instance[immutable.Set[W[Int]]](annotation), W(1)) - } - - test("optionalbinder#bind [T]") { - val module = new TwitterModule { - override def configure(): Unit = { - bindOption[String].setDefault.toInstance("A") - } - } - - validate(module) - } - - test("optionalbinder#bind [T, Ann]") { - val module = new TwitterModule { - override def configure(): Unit = { - bindOption[String, Named].setDefault.toInstance("A") - } - } - - validateWithAnn[String, Named](module) - } - - test("optionalbinder#bind [T, Annotation]") { - val module = new TwitterModule { - override def configure(): Unit = { - val opt = bindOption[String](annotation) - opt.setDefault.toInstance("A") - } - } - - validateWithAnnotation(module, annotation, expected = "A") - } - - test("optionalbinder#bind optional without default") { - val module = new TwitterModule { - override def configure(): Unit = { - bindOption[String] - } - } - - validateAbsent(module) - } - - test("optionalbinder#override default value") { - val module = new TwitterModule { - override def configure(): Unit = { - val opt = bindOption[String] - opt.setDefault.toInstance("B") - opt.setBinding.toInstance("A") - } - } - - validate(module) - } - - test("optionalbinder#override default value with second binder") { - val module = new TwitterModule { - override def configure(): Unit = { - bindOption[String].setDefault.toInstance("B") - bindOption[String].setBinding.toInstance("A") - } - } - - validate(module) - } - - test("optionalbinder#allow separately annotated bindings for the same [T]") { - val module = new TwitterModule { - override def configure(): Unit = { - val aOpt = bindOption[String](Names.named("A")) - aOpt.setBinding.toInstance("A") - val bOpt = bindOption[String](Names.named("B")) - bOpt.setDefault.toInstance("B") - } - } - - validateWithAnnotation(module, Names.named("A"), expected = "A") - validateWithAnnotation(module, Names.named("B"), expected = "B") - } - - test("optionalbinder#bind parameterized [T]") { - val module = new TwitterModule { - override def configure(): Unit = { - val sOpt = bindOption[W[String]] - sOpt.setDefault.toInstance(W("A")) - val nOpt = bindOption[W[Int]] - nOpt.setDefault.toInstance(W(1)) - } - } - - validate(module, expected = W("A")) - validate(module, expected = W(1)) - } - - test("optionalbinder#bind parameterized [T, Ann]") { - val module = new TwitterModule { - override def configure(): Unit = { - val sOpt = bindOption[W[String], TestBindingAnnotation] - sOpt.setDefault.toInstance(W("A")) - val nOpt = bindOption[W[Int], TestBindingAnnotation] - nOpt.setDefault.toInstance(W(1)) - } - } - - validateWithAnn[W[String], TestBindingAnnotation](module, expected = W("A")) - validateWithAnn[W[Int], TestBindingAnnotation](module, expected = W(1)) - } - - test("optionalbinder#bind parameterized [T, Annotation]") { - val module = new TwitterModule { - override def configure(): Unit = { - val sOpt = bindOption[W[String]](annotation) - sOpt.setDefault.toInstance(W("A")) - val nOpt = bindOption[W[Int]](annotation) - nOpt.setDefault.toInstance(W(1)) - } - } - - validateWithAnnotation(module, annotation, expected = W("A")) - validateWithAnnotation(module, annotation, expected = W(1)) - } - - private[this] def assertServiceFactory(complexServiceFactory: ComplexServiceFactory): Unit = { - complexServiceFactory.create("foo").execute should equal("done foo") - } - - private[this] def validate[T](set: Set[T], expected: T*): Unit = { - set should have size expected.length - for (e <- expected) { - set should contain(e) - } - } - - private[this] def validate[T: Manifest](module: Module, expected: T = "A"): Unit = { - val injector = Injector(Guice.createInjector(module)) - - // Check Option - injector.instance[Option[T]] should contain(expected) - injector.instance[Option[Provider[T]]].get.get() should equal(expected) - injector.instance[Option[javax.inject.Provider[T]]].get.get() should equal(expected) - - // Check Option - injector.instance[Option[T]].get should equal(expected) - injector.instance[Option[Provider[T]]].get.get() should equal(expected) - injector.instance[Option[javax.inject.Provider[T]]].get.get() should equal(expected) - } - - private def validateWithAnn[T: Manifest, Ann <: Annotation: Manifest]( - module: Module, - expected: T = "A" - ): Unit = { - val injector = Injector(Guice.createInjector(module)) - - // Check Option - injector.instance[Option[T], Ann] should contain(expected) - injector.instance[Option[Provider[T]], Ann].get.get() should equal(expected) - injector.instance[Option[javax.inject.Provider[T]], Ann].get.get() should equal(expected) - - // Check Option - injector.instance[Option[T], Ann].get should equal(expected) - injector.instance[Option[Provider[T]], Ann].get.get() should equal(expected) - injector.instance[Option[javax.inject.Provider[T]], Ann].get.get() should equal(expected) - } - - private def validateWithAnnotation[T: Manifest]( - module: Module, - annotation: Annotation, - expected: T - ): Unit = { - val injector = Injector(Guice.createInjector(module)) - - // Check Option - injector.instance[Option[T]](annotation) should contain(expected) - injector.instance[Option[Provider[T]]](annotation).get.get() should equal(expected) - injector.instance[Option[javax.inject.Provider[T]]](annotation).get.get() should equal(expected) - - // Check Option - injector.instance[Option[T]](annotation).get should equal(expected) - injector.instance[Option[Provider[T]]](annotation).get.get() should equal(expected) - injector.instance[Option[javax.inject.Provider[T]]](annotation).get.get() should equal(expected) - } - - private def validateAbsent[T: Manifest](module: Module, expected: T = "A"): Unit = { - val injector = Injector(Guice.createInjector(module)) - - // Check Option - injector.instance[Option[T]] should be(None) - injector.instance[Option[Provider[T]]] should be(None) - injector.instance[Option[javax.inject.Provider[T]]] should be(None) - - // Check Option - injector.instance[Option[T]].isDefined should be(false) - injector.instance[Option[Provider[T]]].isDefined should be(false) - injector.instance[Option[javax.inject.Provider[T]]].isDefined should be(false) - } -} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala b/util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala deleted file mode 100644 index b2567b9b06..0000000000 --- a/util-inject/src/test/scala/com/twitter/util/inject/TypeUtilsTest.scala +++ /dev/null @@ -1,46 +0,0 @@ -package com.twitter.inject - -import org.scalacheck.Arbitrary -import org.scalatest.funsuite.AnyFunSuite -import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks -import scala.reflect.runtime.universe._ - -trait A -trait B - -class TypeUtilsTest extends AnyFunSuite with ScalaCheckDrivenPropertyChecks { - - test("asManifest handles AnyVal/Any/Null/Nothing") { - forAll(Arbitrary.arbAnyVal.arbitrary) { anyVal => - assert(manifestedTypesEquals(anyVal)) - } - assert(TypeUtils.asManifest[AnyVal] == manifest[AnyVal]) - assert(TypeUtils.asManifest[Any] == manifest[Any]) - assert(TypeUtils.asManifest[Null] == manifest[Null]) - assert(TypeUtils.asManifest[Nothing] == manifest[Nothing]) - } - - test("asManifest handles NoClassDefFound exceptions") { - val t = typeTag[A with B] - intercept[NoClassDefFoundError] { - t.mirror.runtimeClass(t.tpe) - } - assert(TypeUtils.asManifest[A with B] == manifest[Any]) - } - - test("asTypeTag handles classes") { - typeTagEquals(classOf[String], TypeUtils.asTypeTag(classOf[String])) - typeTagEquals(classOf[AnyVal], TypeUtils.asTypeTag(classOf[AnyVal])) - typeTagEquals(classOf[Any], TypeUtils.asTypeTag(classOf[Any])) - typeTagEquals(classOf[Null], TypeUtils.asTypeTag(classOf[Null])) - typeTagEquals(classOf[Nothing], TypeUtils.asTypeTag(classOf[Nothing])) - } - - def manifestedTypesEquals[T: TypeTag: Manifest](a: T): Boolean = { - TypeUtils.asManifest[T] == manifest[T] - } - - def typeTagEquals[T](clazz: Class[T], typeTag: TypeTag[T]): Boolean = { - clazz.isAssignableFrom(typeTag.mirror.runtimeClass(typeTag.tpe.typeSymbol.asClass)) - } -} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/classes.scala b/util-inject/src/test/scala/com/twitter/util/inject/classes.scala deleted file mode 100644 index 059bdac24a..0000000000 --- a/util-inject/src/test/scala/com/twitter/util/inject/classes.scala +++ /dev/null @@ -1,104 +0,0 @@ -package com.twitter.inject - -import com.google.inject.{Provider, TypeLiteral} -import javax.inject.{Inject, Named} - -class SomeClient { - protected val underlying: String = "Hello, Alice." - def get: String = underlying -} - -trait Augmentation { self: SomeClient => - override def get: String = s"${self.underlying} Welcome to Wonderland." - def onlyHere: String = "Only on augmented classes." -} - -class Baz @Inject() (client: SomeClient with Augmentation) { - client.onlyHere -} -trait MyServiceInterface { - - def add2(int: Int): Int -} - -class MyServiceImpl extends MyServiceInterface { - override def add2(int: Int): Int = { - int + 2 - } -} -trait MultiService { - val name: String -} - -class OneMultiService extends MultiService { - override val name = "one" -} - -class TwoMultiService extends MultiService { - override val name = "two" -} -object classes { - object Outer { - trait X - class Y extends X - trait Gen[T] { - def get: T - } - - class C extends Gen[String] { - def get = "String" - } - } - - trait X - class Y extends X - - class YProvider extends Provider[Y] { - def get = new Y - } - - trait Gen[T] { - def get: T - } - - class TypeProvider[T] @Inject() (typ: TypeLiteral[T]) extends Provider[String] { - def get = typ.toString - } - - class Z extends Gen[String] { - def get = "String" - } - - class GenStringProvider extends Provider[Gen[String]] { - def get = new Z - } - - class ZProvider extends Provider[Z] { - def get = new Z - } - - trait Foo { - def foo(): String - } - - class FooProviderWithJavax extends javax.inject.Provider[Foo] { - def get(): Foo = new Foo { - def foo() = "foo" - } - } - - case class TwoStrings @Inject() (@Named("first") first: String, @Named("second") second: String) - - trait D - - sealed trait SealedTrait - - final case object FinalSealedTrait extends SealedTrait - - case class SealedTraitContainer[T <: SealedTrait](inner: T) - - class SealedTraitContainerFinalSealedTraitProvider - extends javax.inject.Provider[SealedTraitContainer[FinalSealedTrait.type]] { - def get(): SealedTraitContainer[FinalSealedTrait.type] = SealedTraitContainer(FinalSealedTrait) - } -} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/module/SeqAbstractModule.scala b/util-inject/src/test/scala/com/twitter/util/inject/module/SeqAbstractModule.scala deleted file mode 100644 index 0b9f52d7c1..0000000000 --- a/util-inject/src/test/scala/com/twitter/util/inject/module/SeqAbstractModule.scala +++ /dev/null @@ -1,12 +0,0 @@ -package com.twitter.inject.module - -import com.google.inject.{AbstractModule, Provides} -import javax.inject.Singleton -import net.codingwell.scalaguice.ScalaModule - -object SeqAbstractModule extends AbstractModule with ScalaModule { - @Singleton - @Provides - def provideSeq: Seq[Array[Byte]] = - Seq.empty[Array[Byte]] -} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala b/util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala deleted file mode 100644 index 7e13802ac6..0000000000 --- a/util-inject/src/test/scala/com/twitter/util/inject/module/TestModule.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.twitter.inject.module - -import com.google.inject.AbstractModule -import com.google.inject.name.Names -import com.twitter.inject.TestBindingAnnotation -import com.twitter.inject.{MyServiceImpl, MyServiceInterface} -import java.util.Properties -import javax.inject.Singleton -import net.codingwell.scalaguice.ScalaModule - -trait TestModule extends AbstractModule with ScalaModule { - - override protected def configure(): Unit = { - bind[String].annotatedWith(Names.named("str1")).toInstance("string1") - bind[String].annotatedWith(Names.named("str2")).toInstance("string2") - - bind[Int].toInstance(11) - bind[String].toInstance("default string") - bind[String].annotatedWith[TestBindingAnnotation].toInstance("prod string") - - bind[MyServiceInterface].to[MyServiceImpl].in[Singleton] - val properties = new Properties() - properties.setProperty("name", "Steve") - Names.bindProperties(TestModule.super.binder(), properties) - } -} diff --git a/util-inject/src/test/scala/com/twitter/util/inject/module/TestTwitterModule.scala b/util-inject/src/test/scala/com/twitter/util/inject/module/TestTwitterModule.scala deleted file mode 100644 index e195050ccc..0000000000 --- a/util-inject/src/test/scala/com/twitter/util/inject/module/TestTwitterModule.scala +++ /dev/null @@ -1,136 +0,0 @@ -package com.twitter.inject.module - -import com.google.inject.name.{Named, Names} -import com.google.inject.spi.TypeConverter -import com.google.inject.{Provides, TypeLiteral} -import com.twitter.app.Flag -import com.twitter.conversions.DurationOps._ -import com.twitter.inject.servicefactory.ComplexServiceFactory -import com.twitter.inject.{ - Injector, - MultiService, - MyServiceImpl, - MyServiceInterface, - OneMultiService, - TestBindingAnnotation, - TwitterModule, - TwoMultiService -} -import javax.inject.Singleton -import java.util.Properties -import org.scalatest.Assertions -import scala.util.control.NonFatal - -// for testing the function/API of a TwitterModule -- not for general testing -object TestTwitterModule extends TwitterModule with Assertions { - - // Note: The following flag values are not used in this module, but are @Flag injected elsewhere - flag("moduleMagicNum", "30", "Module Magic number") - flag("moduleDuration", 5.seconds, "Module duration") - - val localModuleFlag: Flag[String] = flag("moduleString", "default", "A passed string") - - override protected def configure(): Unit = { - bind[String].annotatedWith(Names.named("str1")).toInstance("string1") - bind[String].annotatedWith(Names.named("str2")).toInstance("string2") - - bind[Int].toInstance(11) - bind[String].toInstance("default string") - bind[String].annotatedWith[TestBindingAnnotation].toInstance("prod string") - - bind[MyServiceInterface].to[MyServiceImpl].in[Singleton] - - bindAssistedFactory[ComplexServiceFactory]() - getProvider[Int] - - bindMultiple[MultiService].addBinding.to[OneMultiService] - bindMultiple[MultiService].addBinding.to[TwoMultiService] - - bindOption[String](Names.named("SomeString")) - bindOption[Double](Names.named("OptionalDouble")).setDefault.toInstance(142d) - bindOption[Float].setBinding.toInstance(137f) - bindOption[String](Names.named("DoEverythingString")).setBinding.toInstance("NOTDEFAULT") - - addTypeConverter[ClassToConvert](new TypeConverter { - override def convert(s: String, typeLiteral: TypeLiteral[_]): AnyRef = { - ClassToConvert(s) - } - }) - - val properties = new Properties() - properties.setProperty("name", "Steve") - Names.bindProperties(binder(), properties) - } - - override def singletonStartup(injector: Injector): Unit = { - assert( - !localModuleFlag.isDefined - ) // isDefined is only true when a value has been parsed, will be false if the flag still has the default - try { - localModuleFlag() - fail() - } catch { - case NonFatal(_) => - // expected: // no flag parsing happens (since we are creating the injector manually) so the flag read fails - } - assert(injector.instance[String, TestBindingAnnotation] == "prod string") - assert(injector.instance[String, TestBindingAnnotation] == "prod string") - - // check that the Set was bound with two elements - val services = injector.instance[Set[MultiService]] - assert(services.size == 2) - services.foreach { - case _: OneMultiService => assert(true) - case _: TwoMultiService => assert(true) - case _ => assert(false) - } - - // binder created with no binding, lookup of Option[String] should return None - assert(injector.instance[Option[String]](Names.named("SomeString")).isEmpty) - - // binds the Option[Double] - assert(injector.instance[Option[Double]](Names.named("OptionalDouble")).isDefined) - assert(injector.instance[Option[Double]](Names.named("OptionalDouble")).get == 142d) - // should also bind the Double, since we called ".setDefault" - assert(injector.instance[Double](Names.named("OptionalDouble")) == 142d) - - assert(injector.instance[Option[Float]].isDefined) - assert(injector.instance[Option[Float]].get == 137f) - - // no binder was ever created here - try { - injector.instance[Option[String]](Names.named("MyString")) - fail() - } catch { - case NonFatal(e) => - // ensure that the DoEverythingModule is in the stack trace - assert(e.getStackTrace.exists(_.getClassName == TestTwitterModule.getClass.getName)) - } - - // binds the Option[String] - assert(injector.instance[Option[String]](Names.named("DoEverythingString")).isDefined) - assert(injector.instance[Option[String]](Names.named("DoEverythingString")).get == "NOTDEFAULT") - // should also bind the String, since we called ".setBinding" - assert(injector.instance[String](Names.named("DoEverythingString")) == "NOTDEFAULT") - - val myService = injector.instance[MyServiceInterface] - val myService2 = injector.instance[MyServiceInterface] - assert(myService.toString == myService2.toString) - } - - override def singletonPostWarmupComplete(injector: Injector): Unit = { - info("module post warmup complete") - } - - override def singletonShutdown(injector: Injector): Unit = { - info("shutdown") - } - - @Provides - @Named("example") - def provideNamedString: String = { - "named" - } -} - -case class ClassToConvert(name: String) diff --git a/util-inject/src/test/scala/com/twitter/util/inject/servicefactory.scala b/util-inject/src/test/scala/com/twitter/util/inject/servicefactory.scala deleted file mode 100644 index 6d1821e21d..0000000000 --- a/util-inject/src/test/scala/com/twitter/util/inject/servicefactory.scala +++ /dev/null @@ -1,32 +0,0 @@ -package com.twitter.inject - -import com.google.inject.assistedinject.Assisted -import javax.inject.{Inject, Named, Singleton} - -object servicefactory { - - @Singleton - class DoEverythingService { - - def doit = { - "done" - } - } - - class ComplexService @Inject() ( - exampleService: DoEverythingService, - defaultString: String, - @Named("str1") string1: String, - @Named("str2") string2: String, - defaultInt: Int, - @Assisted name: String) { - - def execute = { - exampleService.doit + " " + name - } - } - - trait ComplexServiceFactory { - def create(name: String): ComplexService - } -} From 92572c0257833407a65ac275c30d8feb69a624e2 Mon Sep 17 00:00:00 2001 From: Vladimir Kostyukov Date: Sat, 22 Oct 2022 03:09:40 +0000 Subject: [PATCH 745/796] util-stats: Java-friendlify InMemoryStatsReceiver Problem / Solution InMemoryStatsReceiver isn't pleasant to use in Java. Let's make the UX better. Differential Revision: https://phabricator.twitter.biz/D999001 --- .../finagle/stats/InMemoryStatsReceiver.scala | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala index a8163d93d3..587a4b90c1 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/InMemoryStatsReceiver.scala @@ -11,6 +11,7 @@ import com.twitter.util.Try import java.io.PrintStream import java.util.Locale import java.util.concurrent.ConcurrentHashMap +import scala.annotation.varargs import scala.collection.compat._ import scala.collection.SortedMap import scala.collection.mutable @@ -95,13 +96,9 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } } - override def counter(name: String*): ReadableCounter = { - val metricBuilder = this - .metricBuilder(CounterType) - .withName(name: _*) - - counter(metricBuilder) - } + @varargs + override def counter(name: String*): ReadableCounter = + counter(MetricBuilder.forCounter.withName(name: _*)) /** * Creates a [[ReadableCounter]] of the given `name`. @@ -136,13 +133,9 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { } } - override def stat(name: String*): ReadableStat = { - val metricBuilder = this - .metricBuilder(HistogramType) - .withName(name: _*) - - stat(metricBuilder) - } + @varargs + override def stat(name: String*): ReadableStat = + stat(MetricBuilder.forStat.withName(name: _*)) /** * Creates a [[ReadableStat]] of the given `name`. @@ -181,7 +174,7 @@ class InMemoryStatsReceiver extends StatsReceiver with WithHistogramDetails { /** * Creates a [[Gauge]] of the given `name`. */ - def addGauge(metricBuilder: MetricBuilder)(f: => Float) = { + def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = { validateMetricType(metricBuilder, GaugeType) new Gauge { From a2d711a6190ebaa0d0602e3a97921f548f2932b1 Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Mon, 24 Oct 2022 21:19:13 +0000 Subject: [PATCH 746/796] [util-core] Implement scheduler and executor backed WorkQueueFibers Problem: We want to use a work queue based fiber, but we don't have any concrete implementations. Solution: Implement scheduler-backed and executor-backed versions of WorkQueueFiber. The scheduler-backed WorkQueueFiber needs to flush the scheduler along with the fiber in the case of a thread local scheduler that queues tasks per thread. JIRA Issues: CSL-12357 Differential Revision: https://phabricator.twitter.biz/D992193 --- .../twitter/util/ExecutorWorkQueueFiber.scala | 29 ++++++++ .../util/SchedulerWorkQueueFiber.scala | 28 ++++++++ .../com/twitter/util/WorkQueueFiber.scala | 9 ++- .../util/ExecutorWorkQueueFiberTest.scala | 67 +++++++++++++++++++ .../util/SchedulerWorkQueueFiberTest.scala | 60 +++++++++++++++++ .../com/twitter/util/WorkQueueFiberTest.scala | 26 ++++++- 6 files changed, 213 insertions(+), 6 deletions(-) create mode 100644 util-core/src/main/scala/com/twitter/util/ExecutorWorkQueueFiber.scala create mode 100644 util-core/src/main/scala/com/twitter/util/SchedulerWorkQueueFiber.scala create mode 100644 util-core/src/test/scala/com/twitter/util/ExecutorWorkQueueFiberTest.scala create mode 100644 util-core/src/test/scala/com/twitter/util/SchedulerWorkQueueFiberTest.scala diff --git a/util-core/src/main/scala/com/twitter/util/ExecutorWorkQueueFiber.scala b/util-core/src/main/scala/com/twitter/util/ExecutorWorkQueueFiber.scala new file mode 100644 index 0000000000..abc04766a4 --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/ExecutorWorkQueueFiber.scala @@ -0,0 +1,29 @@ +package com.twitter.util + +import java.util.concurrent.Executor + +/** + * A [[WorkQueueFiber]] implementation that submits its work queue to the underlying + * executor. + * + * This is intended to be used with a thread pool executor that manages task execution + * across threads. The ExecutorWorkQueueFiber pairs up a thread pool and a WorkQueueFiber + * in order to act as a scheduler and handle all task scheduling responsibilities + * for a server. + */ +private[twitter] final class ExecutorWorkQueueFiber( + pool: Executor, + metrics: WorkQueueFiber.FiberMetrics) + extends WorkQueueFiber(metrics) { + + /** + * Submit the work to the underlying executor. + */ + override protected def schedulerSubmit(r: Runnable): Unit = pool.execute(r) + + /** + * No need to flush. Thread pools usually do not have per thread queues, and any that do + * should have some sort of work stealing mechanism to pass tasks around between threads. + */ + override protected def schedulerFlush(): Unit = () +} diff --git a/util-core/src/main/scala/com/twitter/util/SchedulerWorkQueueFiber.scala b/util-core/src/main/scala/com/twitter/util/SchedulerWorkQueueFiber.scala new file mode 100644 index 0000000000..f22a24ebfb --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/SchedulerWorkQueueFiber.scala @@ -0,0 +1,28 @@ +package com.twitter.util + +import com.twitter.concurrent.Scheduler + +/** + * A [[WorkQueueFiber]] implementation that submits its work queue to the global + * scheduler. + * + * This allows us to gain the benefits of the WorkQueueFiber, namely the bundling of work + * into Runnables of multiple tasks, while still utilizing the global scheduler. This + * implementation of the WorkQueueFiber is meant for servers that are not well suited + * for offloading tasks into a thread pool for execution. + */ +private[twitter] final class SchedulerWorkQueueFiber( + scheduler: Scheduler, + fiberMetrics: WorkQueueFiber.FiberMetrics) + extends WorkQueueFiber(fiberMetrics) { + + /** + * Submit the work to the underlying scheduler for execution. + */ + override protected def schedulerSubmit(r: Runnable): Unit = scheduler.submit(r) + + /** + * Flush the underlying scheduler. + */ + override protected def schedulerFlush(): Unit = scheduler.flush() +} diff --git a/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala index bb2da139f2..d56468d617 100644 --- a/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala +++ b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala @@ -107,12 +107,11 @@ private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.Fibe // There is fiber work that needs to be executed. Attempt to reschedule. tryScheduleFiber() } - - // Depending on the underlying execution engine, we may need to flush the scheduler as well. - // Special consideration will need to be taken for CPU usage tracking if that is the case. - // We can work this all out in the future. - schedulerFlush() } + // Depending on the underlying execution engine, we may need to flush the scheduler as well. + // Special consideration will need to be taken for CPU usage tracking if that is the case. + // We can work this all out in the future. + schedulerFlush() } private[this] def threadLocalSubmit(r: FiberTask): Unit = { diff --git a/util-core/src/test/scala/com/twitter/util/ExecutorWorkQueueFiberTest.scala b/util-core/src/test/scala/com/twitter/util/ExecutorWorkQueueFiberTest.scala new file mode 100644 index 0000000000..a561c98b23 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/util/ExecutorWorkQueueFiberTest.scala @@ -0,0 +1,67 @@ +package com.twitter.util + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executor +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import org.scalatest.funsuite.AnyFunSuite + +class ExecutorWorkQueueFiberTest extends AnyFunSuite { + + private class NoOpMetrics extends WorkQueueFiber.FiberMetrics { + def flushIncrement(): Unit = () + def fiberCreated(): Unit = () + def threadLocalSubmitIncrement(): Unit = () + def taskSubmissionIncrement(): Unit = () + def schedulerSubmissionIncrement(): Unit = () + } + + private class TestPool(exec: ExecutorService) extends Executor { + val poolSubmitCount = new AtomicInteger(0) + override def execute(command: Runnable): Unit = { + poolSubmitCount.incrementAndGet() + exec.execute(command) + } + + def shutdown() = exec.shutdown() + } + + test("fiber submits tasks to underlying future pool in single runnable") { + val pool = new TestPool(Executors.newFixedThreadPool(1)) + val fiber = new ExecutorWorkQueueFiber(pool, new NoOpMetrics) + + val latch = new CountDownLatch(1) + val N = 5 + for (_ <- 0 until N) { + fiber.submitTask { () => + // wait to finish task execution so that all task submissions queue up in + // the fiber and are submitted to the pool as a single runnable + latch.await() + } + } + latch.countDown() + assert(pool.poolSubmitCount.get() == 1) + + pool.shutdown() + } + + test("fiber submits tasks individually to underlying future pool") { + val pool = new TestPool(Executors.newFixedThreadPool(1)) + val fiber = new ExecutorWorkQueueFiber(pool, new NoOpMetrics) + + val N = 5 + for (_ <- 0 until N) { + val latch = new CountDownLatch(1) + fiber.submitTask { () => + latch.countDown() + } + // wait for the task to finish executing so that the next task gets its own + // submission to the pool + latch.await() + } + assert(pool.poolSubmitCount.get() == N) + + pool.shutdown() + } +} diff --git a/util-core/src/test/scala/com/twitter/util/SchedulerWorkQueueFiberTest.scala b/util-core/src/test/scala/com/twitter/util/SchedulerWorkQueueFiberTest.scala new file mode 100644 index 0000000000..e56a49cc93 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/util/SchedulerWorkQueueFiberTest.scala @@ -0,0 +1,60 @@ +package com.twitter.util + +import com.twitter.concurrent.LocalScheduler +import java.util.concurrent.CountDownLatch +import org.scalatest.concurrent.Eventually.eventually +import org.scalatest.concurrent.PatienceConfiguration.Timeout +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.time.Seconds +import org.scalatest.time.Span + +class SchedulerWorkQueueFiberTest extends AnyFunSuite { + + private class NoOpMetrics extends WorkQueueFiber.FiberMetrics { + def flushIncrement(): Unit = () + def fiberCreated(): Unit = () + def threadLocalSubmitIncrement(): Unit = () + def taskSubmissionIncrement(): Unit = () + def schedulerSubmissionIncrement(): Unit = () + } + + test("fiber flushes the underlying LocalScheduler") { + val flushLatch = new CountDownLatch(1) + val localScheduler = new LocalScheduler { + override def flush(): Unit = { + super.flush() + flushLatch.countDown() + } + } + val localSchedulerFiber = new SchedulerWorkQueueFiber(localScheduler, new NoOpMetrics) + + var promiseSatisfied = false + + val runnable = new Runnable { + def run(): Unit = { + val p = Promise[Boolean]() + val f2 = new SchedulerWorkQueueFiber(localScheduler, new NoOpMetrics) + // Satisfy the promise on another fiber, whose work will get scheduled on the same thread + f2.submitTask(() => p.setValue(true)) + + // This will trigger a flush of the current fiber and the local scheduler + Await.result(p) + promiseSatisfied = true + } + } + + val t = new Thread({ () => + Fiber.let(localSchedulerFiber) { + localSchedulerFiber.submitTask(() => runnable.run()) + } + }) + t.start() + + // Ensure flush completed + flushLatch.await() + + // The promise should be satisfied even though the second fiber was scheduled on the + // same thread as the Await.result + eventually(Timeout(Span(1, Seconds))) { assert(promiseSatisfied) } + } +} diff --git a/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala b/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala index a7c1b6829e..6142448ae3 100644 --- a/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala @@ -326,7 +326,7 @@ class WorkQueueFiberTest extends AnyFunSuite { assert(executed) } - test("a flush outside the executing code is a no-op") { + test("a flush outside the executing code is a no-op for fiber") { val metrics = new TestMetrics val f = new TestFiber(metrics) @@ -347,4 +347,28 @@ class WorkQueueFiberTest extends AnyFunSuite { f.executePendingRunnable() assert(executed) } + + test("a flush triggers a scheduler flush whether or not its in the executing code") { + val metrics = new TestMetrics + var schedulerFlushes = 0 + val f = new WorkQueueFiber(metrics) { + override protected def schedulerSubmit(r: Runnable): Unit = { + r.run() + } + override protected def schedulerFlush(): Unit = { + schedulerFlushes += 1 + } + } + // flush from within executing code + f.submitTask(() => { + f.flush() + }) + assert(schedulerFlushes == 1) + assert(metrics.fiberFlushed.get == 1) + // flush from outside executing code + f.flush() + assert(schedulerFlushes == 2) + // the fiber didn't flush this time + assert(metrics.fiberFlushed.get == 1) + } } From 82491062cac4bb8ac4418d96e9e180747068de90 Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Tue, 25 Oct 2022 19:53:29 +0000 Subject: [PATCH 747/796] [finagle-core] Use work queue based fiber behind a toggle Problem: We would like to start using the work queue based fiber in the finagle stack. Solution: Gated by a toggle, pick the ExecutorWorkQueueFiber or the SchedulerWorkQueueFiber in the FiberForkFilter. Toggled on servers with offloading enabled will use the ExecutorWorkQueueFiber, which backs the fiber with the offload pool. All other toggled on servers will use the scheduler backed work queue fiber. JIRA Issues: CSL-12357 Differential Revision: https://phabricator.twitter.biz/D992075 --- util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala index d56468d617..dc9bebebc2 100644 --- a/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala +++ b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala @@ -218,7 +218,7 @@ private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.Fibe } } -private object WorkQueueFiber { +private[twitter] object WorkQueueFiber { /** Telemetry used to monitor [[WorkQueueFiber]]s */ abstract class FiberMetrics { From 8bcfd6a042fb6d46b4c7e876b00bba80ca1a16d0 Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Wed, 26 Oct 2022 00:08:36 +0000 Subject: [PATCH 748/796] [util-core] Add parameter documentation for WorkQueueFiber implementations Problem / Solution: Add more documentation for the parameters of the ExecutorWorkQueueFiber and SchedulerWorkQueueFiber classes. JIRA Issues: CSL-12357 Differential Revision: https://phabricator.twitter.biz/D1004983 --- .../scala/com/twitter/util/ExecutorWorkQueueFiber.scala | 5 +++++ .../scala/com/twitter/util/SchedulerWorkQueueFiber.scala | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/ExecutorWorkQueueFiber.scala b/util-core/src/main/scala/com/twitter/util/ExecutorWorkQueueFiber.scala index abc04766a4..a492fc9c35 100644 --- a/util-core/src/main/scala/com/twitter/util/ExecutorWorkQueueFiber.scala +++ b/util-core/src/main/scala/com/twitter/util/ExecutorWorkQueueFiber.scala @@ -10,6 +10,11 @@ import java.util.concurrent.Executor * across threads. The ExecutorWorkQueueFiber pairs up a thread pool and a WorkQueueFiber * in order to act as a scheduler and handle all task scheduling responsibilities * for a server. + * + * @param pool a task Executor that must either execute the work immediately or pass the + * work to another thread in order to avoid potential deadlocks. + * @param fiberMetrics telemetry used for tracking the execution of this fiber, and likely + * other fibers as well. */ private[twitter] final class ExecutorWorkQueueFiber( pool: Executor, diff --git a/util-core/src/main/scala/com/twitter/util/SchedulerWorkQueueFiber.scala b/util-core/src/main/scala/com/twitter/util/SchedulerWorkQueueFiber.scala index f22a24ebfb..a0f3e8e6bf 100644 --- a/util-core/src/main/scala/com/twitter/util/SchedulerWorkQueueFiber.scala +++ b/util-core/src/main/scala/com/twitter/util/SchedulerWorkQueueFiber.scala @@ -3,13 +3,16 @@ package com.twitter.util import com.twitter.concurrent.Scheduler /** - * A [[WorkQueueFiber]] implementation that submits its work queue to the global - * scheduler. + * A [[WorkQueueFiber]] implementation that submits its work queue to a scheduler. * * This allows us to gain the benefits of the WorkQueueFiber, namely the bundling of work * into Runnables of multiple tasks, while still utilizing the global scheduler. This * implementation of the WorkQueueFiber is meant for servers that are not well suited * for offloading tasks into a thread pool for execution. + * + * @param scheduler the task scheduler that will execute tasks. + * @param fiberMetrics telemetry used for tracking the execution of this fiber, and likely + * other fibers as well. */ private[twitter] final class SchedulerWorkQueueFiber( scheduler: Scheduler, From c17fd76419371159a8c652fa400cdd8d0bc70e34 Mon Sep 17 00:00:00 2001 From: Alicia Vargas-Morawetz Date: Tue, 1 Nov 2022 21:00:42 +0000 Subject: [PATCH 749/796] [util-core] Add resource tracking to WorkQueueFiber Problem: We want to do CPU resource tracking in fibers. Solution: Add a resource tracker to the WorkQueueFiber and track the cpu time to run the entire work loop. If the fiber is flushed it may do extra work, so subtract the cpu time it takes to flush. JIRA Issues: CSL-12268 Differential Revision: https://phabricator.twitter.biz/D1007425 --- .../com/twitter/util/ResourceTracker.scala | 1 + .../com/twitter/util/WorkQueueFiber.scala | 44 ++++++++--- .../util/ExecutorWorkQueueFiberTest.scala | 3 + .../com/twitter/util/WorkQueueFiberTest.scala | 76 +++++++++++++++++++ 4 files changed, 115 insertions(+), 9 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/ResourceTracker.scala b/util-core/src/main/scala/com/twitter/util/ResourceTracker.scala index 0c38e290d4..c1199898f5 100644 --- a/util-core/src/main/scala/com/twitter/util/ResourceTracker.scala +++ b/util-core/src/main/scala/com/twitter/util/ResourceTracker.scala @@ -101,6 +101,7 @@ private[twitter] class ResourceTracker { private[util] def addCpuTime(time: Long): Unit = cpuTime.add(time) private[util] def incContinuations(): Unit = continuations.getAndIncrement() + private[util] def addContinuations(count: Int): Unit = continuations.getAndAdd(count) /** * Retrieve the total accumulated CPU time diff --git a/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala index dc9bebebc2..9f5df55569 100644 --- a/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala +++ b/util-core/src/main/scala/com/twitter/util/WorkQueueFiber.scala @@ -19,6 +19,10 @@ import scala.annotation.tailrec * * @note that priority is given to the work submitted by this thread. * + * @note care should be taken with the FiberTasks submitted to this Fiber. If a FiberTask + * should throw an exception, the fiber could get stuck in an `executing` state and + * resource tracking would be incomplete. + * * @param fiberMetrics telemetry used for tracking the execution of this fiber, and likely * other fibers as well. */ @@ -51,6 +55,8 @@ private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.Fibe private[this] val workRunnable: Runnable = { () => startWork() } + private[twitter] val tracker: ResourceTracker = new ResourceTracker + /** * Submit the work to an actual thread for execution * @@ -86,7 +92,10 @@ private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.Fibe * with other work submitted to the fiber. */ final def flush(): Unit = { - if (Thread.currentThread() eq executingThread) { + val inFiber = Thread.currentThread() eq executingThread + // Record flush start time so we can account for the extra work done by this `flush` call. + val flushStartTime = if (inFiber) ResourceTracker.currentThreadCpuTime else 0l + if (inFiber) { fiberMetrics.flushIncrement() // We need to move all our work into a new `schedulerSubmit` call. That means moving all // our thread local work into the thread-safe queue and resubmitting this fiber if there @@ -109,9 +118,14 @@ private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.Fibe } } // Depending on the underlying execution engine, we may need to flush the scheduler as well. - // Special consideration will need to be taken for CPU usage tracking if that is the case. - // We can work this all out in the future. schedulerFlush() + // Subtract the `flush` time from this fiber's total cpu time because the tasks executed may + // not belong to this fiber, and if they do, they'd be double counted since we should be in the + // `startWork` method right now executing one of the FiberTasks. + if (inFiber) { + val flushTime = ResourceTracker.currentThreadCpuTime - flushStartTime + if (flushTime > 0) tracker.addCpuTime(-flushTime) + } } private[this] def threadLocalSubmit(r: FiberTask): Unit = { @@ -161,19 +175,21 @@ private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.Fibe assert(!multiThreadedQueue.isEmpty) // Expects that the first thing to do is execute a thread local task. - @tailrec def go(n: FiberTask): Unit = { + @tailrec def go(count: Int, n: FiberTask): Int = { n.doRun() // If we're not the executing thread anymore it is because a task has "flushed" us. // If we've been flushed then we just need to exit now. The `flush()` method will - // take care of the state management. - if (executingThread eq currentThread) { + // take care of the state management and account for resource tracking. + if (executingThread ne currentThread) { + count + } else { // Get our next task, if it exists. var nn = localNext() if (nn == null) { nn = multiThreadedQueue.poll() } - if (nn != null) go(nn) + if (nn != null) go(count + 1, nn) else { // We found our local queue and multi-threaded queues both empty. We now go about // exiting this fiber submission. @@ -189,13 +205,23 @@ private[twitter] abstract class WorkQueueFiber(fiberMetrics: WorkQueueFiber.Fibe // it to the scheduler because this loop was running. if (!multiThreadedQueue.isEmpty && !executing.getAndSet(true)) { executingThread = currentThread - go(multiThreadedQueue.poll()) + go(count + 1, multiThreadedQueue.poll()) + } else { + count } } } } - go(multiThreadedQueue.poll()) + // Start tracking cpu usage + val trackingStartTime = ResourceTracker.currentThreadCpuTime + val continuationCount = go(1, multiThreadedQueue.poll()) + tracker.addContinuations(continuationCount) + // See how much CPU time has elapsed from this fiber submission. + val elapsedTime = ResourceTracker.currentThreadCpuTime - trackingStartTime + if (elapsedTime > 0) { + tracker.addCpuTime(elapsedTime) + } } @inline private[this] def localHasNext: Boolean = r0 != null diff --git a/util-core/src/test/scala/com/twitter/util/ExecutorWorkQueueFiberTest.scala b/util-core/src/test/scala/com/twitter/util/ExecutorWorkQueueFiberTest.scala index a561c98b23..3a1b15ee5d 100644 --- a/util-core/src/test/scala/com/twitter/util/ExecutorWorkQueueFiberTest.scala +++ b/util-core/src/test/scala/com/twitter/util/ExecutorWorkQueueFiberTest.scala @@ -59,6 +59,9 @@ class ExecutorWorkQueueFiberTest extends AnyFunSuite { // wait for the task to finish executing so that the next task gets its own // submission to the pool latch.await() + // give the executing thread some time to finish up resource tracking and fully + // exit out of its work loop + Thread.sleep(1) } assert(pool.poolSubmitCount.get() == N) diff --git a/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala b/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala index 6142448ae3..537ee96f0f 100644 --- a/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala +++ b/util-core/src/test/scala/com/twitter/util/WorkQueueFiberTest.scala @@ -371,4 +371,80 @@ class WorkQueueFiberTest extends AnyFunSuite { // the fiber didn't flush this time assert(metrics.fiberFlushed.get == 1) } + + test("fiber tracks resources") { + val metrics = new TestMetrics + val f = new TestFiber(metrics) + + val executed = new AtomicInteger() + val N = 5 + for (_ <- 0 until N) { + f.submitTask { () => + executed.incrementAndGet() + } + } + + // The fiber hasn't done any work yet + assert(executed.get == 0) + assert(f.tracker.totalCpuTime() == 0) + assert(f.tracker.numContinuations() == 0) + + f.executePendingRunnable() + assert(executed.get == N) + // The resource tracker records the work done by the fiber + assert(f.tracker.totalCpuTime() > 0) + assert(f.tracker.numContinuations() == N) + } + + test("resource tracking with flush") { + val metrics = new TestMetrics + val f = new TestFiber(metrics) + + val latch1, latch2, flushed = new CountDownLatch(1) + + var innerTaskDone = false + f.submitTask { () => + latch1.countDown() + latch2.await() + f.submitTask { () => + innerTaskDone = true + } + + // Now flush + f.flush() + flushed.countDown() + } + + // Nothing recorded yet by the resource tracker + assert(f.tracker.totalCpuTime() == 0) + assert(f.tracker.numContinuations() == 0) + + val t = new Thread({ () => + f.executePendingRunnable() + }) + t.start() + + latch1.await() + // At this point nothing has been recorded with the resource tracker yet + assert(f.tracker.totalCpuTime() == 0) + assert(f.tracker.numContinuations() == 0) + + // Release thread t and then stall t right after the flush + latch2.countDown() + flushed.await() + t.join() + // Now the cpu time and the number of continuations should have been updated + val cpuTimeAfterFLush = f.tracker.totalCpuTime() + assert(cpuTimeAfterFLush > 0) + assert(f.tracker.numContinuations() == 1) + + assert(!innerTaskDone) + // We should now have the inner task awaiting whoever executes the next submission. Now run it. + f.executePendingRunnable() + assert(innerTaskDone) + // More cpu time should have been added as well as one more continuation + val cpuTimeAfterInnerTask = f.tracker.totalCpuTime() + assert(cpuTimeAfterInnerTask > cpuTimeAfterFLush) + assert(f.tracker.numContinuations() == 2) + } } From f75a6f7ed47b95f6eb6d54d65886237104f8d9e2 Mon Sep 17 00:00:00 2001 From: Amy Lee Date: Wed, 2 Nov 2022 02:06:58 +0000 Subject: [PATCH 750/796] util-stats: Update HistogramComponent.DefaultPercentiles to use correct values & respect metrics implementation ### Problem The motivation for the phab is to fix the bug where the `HistogramComponent.DefaultPercentiles` is set to incorrect values (i.e. 99 as opposed to 0.99), resulting in additional trailing zeroes in the metric name. In the initial review, there was a concern around how different metrics implementation (i.e. finagle vs ostrich) may need to handle the input differently. ### Solution 1. Fix the input passed to HistogramComponent.DefaultPercentiles 1. Update `ExpressionSchema` to respect different types of metrics implementation. `StatsFormatter` already handles the differences in metrics implementation, so this change includes a refactor so that `StatsFormatter` and `HistogramFormatter` share the implementation for generating the correct labels. JIRA Issues: CSL-12353, SSTR-7425 Differential Revision: https://phabricator.twitter.biz/D999055 --- .../finagle/stats/HistogramFormatter.scala | 109 +++++++++++++++--- .../finagle/stats/exp/ExpressionSchema.scala | 13 ++- .../stats/exp/HistogramComponent.scala | 4 +- .../stats/HistogramFormatterTest.scala | 43 +++++++ .../stats/exp/ExpressionSchemaTest.scala | 2 +- 5 files changed, 146 insertions(+), 25 deletions(-) create mode 100644 util-stats/src/test/scala/com/twitter/finagle/stats/HistogramFormatterTest.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramFormatter.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramFormatter.scala index 93a7c8a04a..c5cb743e2e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramFormatter.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramFormatter.scala @@ -1,27 +1,104 @@ package com.twitter.finagle.stats -/** - * The default behavior for formatting, align with Commons Metrics. - */ +import com.twitter.app.GlobalFlag + +object format + extends GlobalFlag[String]( + "commonsmetrics", + "Format style for metric names (ostrich|commonsmetrics|commonsstats)" + ) { + private[stats] val Ostrich = "ostrich" + private[stats] val CommonsMetrics = "commonsmetrics" + private[stats] val CommonsStats = "commonsstats" +} + private[twitter] object HistogramFormatter { - def labelPercentile(p: Double): String = { - // this has a strange quirk that p999 gets formatted as p9990 - // Round for precision issues; e.g. 0.9998999... converts to "p9998" with a direct int cast. - val gname: String = "p" + (p * 10000).round - if (3 < gname.length && ("00" == gname.substring(3))) { - gname.substring(0, 3) - } else { - gname + def default: HistogramFormatter = + format() match { + case format.Ostrich => Ostrich + case format.CommonsMetrics => CommonsMetrics + case format.CommonsStats => CommonsStats + } + + /** + * The default behavior for formatting as done by Commons Metrics. + * + * See Commons Metrics' `Metrics.sample()`. + */ + object CommonsMetrics extends HistogramFormatter { + def labelPercentile(p: Double): String = { + // this has a strange quirk that p999 gets formatted as p9990 + // Round for precision issues; e.g. 0.9998999... converts to "p9998" with a direct int cast. + val gname: String = "p" + (p * 10000).round + if (3 < gname.length && ("00" == gname.substring(3))) { + gname.substring(0, 3) + } else { + gname + } + } + + override def labelMin: String = "min" + + override def labelMax: String = "max" + + override def labelAverage: String = "avg" + } + + /** + * Replicates the behavior for formatting Ostrich stats. + * + * See Ostrich's `Distribution.toMap`. + */ + object Ostrich extends HistogramFormatter { + def labelPercentile(p: Double): String = { + p match { + case 0.5d => "p50" + case 0.9d => "p90" + case 0.95d => "p95" + case 0.99d => "p99" + case 0.999d => "p999" + case 0.9999d => "p9999" + case _ => + val padded = (p * 10000).toInt + s"p$padded" + } } + + override def labelMin: String = "minimum" + + override def labelMax: String = "maximum" + + override def labelAverage: String = "average" } - val labelMin: String = "min" + /** + * Replicates the behavior for formatting Commons Stats stats. + * + * See Commons Stats' `Stats.getVariables()`. + */ + object CommonsStats extends HistogramFormatter { + def labelPercentile(p: Double): String = + s"${p * 100}_percentile".replace(".", "_") + + override def labelMin: String = "min" + + override def labelMax: String = "max" + + override def labelAverage: String = "avg" + } +} + +sealed trait HistogramFormatter { + def labelPercentile(p: Double): String + + def labelMin: String = "min" + + def labelMax: String = "max" - val labelMax: String = "max" + def labelAverage: String = "avg" - val labelAverage: String = "avg" + def labelCount: String = "count" - val labelCount: String = "count" + def labelSum: String = "sum" - val labelSum: String = "sum" } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala index 4465488181..83b1e36e7c 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/ExpressionSchema.scala @@ -101,13 +101,14 @@ private[twitter] object ExpressionSchema { } private def histogramLabel(histoExpr: HistogramExpression): Map[String, String] = { + val defaultHistogramFormatter = HistogramFormatter.default val value = histoExpr.component match { - case Percentile(percentile) => HistogramFormatter.labelPercentile(percentile) - case Min => HistogramFormatter.labelMin - case Max => HistogramFormatter.labelMax - case Avg => HistogramFormatter.labelAverage - case Sum => HistogramFormatter.labelSum - case Count => HistogramFormatter.labelCount + case Percentile(percentile) => defaultHistogramFormatter.labelPercentile(percentile) + case Min => defaultHistogramFormatter.labelMin + case Max => defaultHistogramFormatter.labelMax + case Avg => defaultHistogramFormatter.labelAverage + case Sum => defaultHistogramFormatter.labelSum + case Count => defaultHistogramFormatter.labelCount } Map("bucket" -> value) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala index 954addb69e..89485bc1e6 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/exp/HistogramComponent.scala @@ -6,10 +6,10 @@ private[twitter] object HistogramComponent { case object Avg extends HistogramComponent case object Sum extends HistogramComponent case object Count extends HistogramComponent - case class Percentile(percentile: Double) extends HistogramComponent + case class Percentile(decimal: Double) extends HistogramComponent val DefaultPercentiles: Seq[HistogramComponent] = - Seq(Percentile(50), Percentile(99), Percentile(999), Percentile(9999)) + Seq(Percentile(0.50), Percentile(0.99), Percentile(0.999), Percentile(0.9999)) } private[twitter] sealed trait HistogramComponent diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/HistogramFormatterTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/HistogramFormatterTest.scala new file mode 100644 index 0000000000..b1f24504a7 --- /dev/null +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/HistogramFormatterTest.scala @@ -0,0 +1,43 @@ +package com.twitter.finagle.stats + +import org.scalatest.funsuite.AnyFunSuite + +class HistogramFormatterTest extends AnyFunSuite { + + test("Ostrich - formats as expected") { + assert(HistogramFormatter.Ostrich.labelAverage == "average") + assert(HistogramFormatter.Ostrich.labelMax == "maximum") + assert(HistogramFormatter.Ostrich.labelMin == "minimum") + assert(HistogramFormatter.Ostrich.labelCount == "count") + assert(HistogramFormatter.Ostrich.labelSum == "sum") + assert(HistogramFormatter.Ostrich.labelPercentile(0.50) == "p50") + assert(HistogramFormatter.Ostrich.labelPercentile(0.99) == "p99") + assert(HistogramFormatter.Ostrich.labelPercentile(0.999) == "p999") + assert(HistogramFormatter.Ostrich.labelPercentile(0.9999) == "p9999") + } + + test("CommonMetrics - formats as expected") { + assert(HistogramFormatter.CommonsMetrics.labelAverage == "avg") + assert(HistogramFormatter.CommonsMetrics.labelMax == "max") + assert(HistogramFormatter.CommonsMetrics.labelMin == "min") + assert(HistogramFormatter.CommonsMetrics.labelCount == "count") + assert(HistogramFormatter.CommonsMetrics.labelSum == "sum") + assert(HistogramFormatter.CommonsMetrics.labelPercentile(0.50) == "p50") + assert(HistogramFormatter.CommonsMetrics.labelPercentile(0.99) == "p99") + assert(HistogramFormatter.CommonsMetrics.labelPercentile(0.999) == "p9990") + assert(HistogramFormatter.CommonsMetrics.labelPercentile(0.9999) == "p9999") + } + + test("CommonStats - formats as expected") { + assert(HistogramFormatter.CommonsStats.labelAverage == "avg") + assert(HistogramFormatter.CommonsStats.labelMax == "max") + assert(HistogramFormatter.CommonsStats.labelMin == "min") + assert(HistogramFormatter.CommonsStats.labelCount == "count") + assert(HistogramFormatter.CommonsStats.labelSum == "sum") + assert(HistogramFormatter.CommonsStats.labelPercentile(0.50) == "50_0_percentile") + assert(HistogramFormatter.CommonsStats.labelPercentile(0.99) == "99_0_percentile") + assert(HistogramFormatter.CommonsStats.labelPercentile(0.999) == "99_9_percentile") + assert(HistogramFormatter.CommonsStats.labelPercentile(0.9999) == "99_99_percentile") + } + +} diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala index a8dacc1592..15e512aeff 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/exp/ExpressionSchemaTest.scala @@ -236,7 +236,7 @@ class ExpressionSchemaTest extends AnyFunSuite { ExpressionSchemaKey( "latencies", Map( - "bucket" -> HistogramFormatter.labelPercentile(p), + "bucket" -> HistogramFormatter.default.labelPercentile(p), "role" -> "Client", "service_name" -> "aclient"), Seq()))) From bcf9814380879a72e7dcfc7e875ab955dd6c3921 Mon Sep 17 00:00:00 2001 From: Mikhail Bezoyan Date: Wed, 23 Nov 2022 15:29:02 +0000 Subject: [PATCH 751/796] Add mbezoyan to finagle owners To satisify the linter Differential Revision: https://phabricator.twitter.biz/D1015143 --- util-app/PROJECT | 2 ++ util-benchmark/PROJECT | 2 ++ util-codec/PROJECT | 2 ++ util-core/PROJECT | 2 ++ util-jackson-annotations/PROJECT | 2 ++ util-jackson/PROJECT | 2 ++ util-logging/PROJECT | 2 ++ util-mock/PROJECT | 2 ++ util-security/PROJECT | 2 ++ util-slf4j-api/PROJECT | 2 ++ util-slf4j-jul-bridge/PROJECT | 2 ++ util-stats/PROJECT | 2 ++ util-validator-constraints/PROJECT | 4 +++- util-validator/PROJECT | 4 +++- util-zk-test/PROJECT | 4 +++- util-zk/PROJECT | 2 ++ 16 files changed, 35 insertions(+), 3 deletions(-) diff --git a/util-app/PROJECT b/util-app/PROJECT index a482ca9a20..51e3ed72fc 100644 --- a/util-app/PROJECT +++ b/util-app/PROJECT @@ -7,5 +7,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - csl-team@twitter.com diff --git a/util-benchmark/PROJECT b/util-benchmark/PROJECT index 85f4cb0288..405c5cefe7 100644 --- a/util-benchmark/PROJECT +++ b/util-benchmark/PROJECT @@ -6,3 +6,5 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson diff --git a/util-codec/PROJECT b/util-codec/PROJECT index 85f4cb0288..405c5cefe7 100644 --- a/util-codec/PROJECT +++ b/util-codec/PROJECT @@ -6,3 +6,5 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson diff --git a/util-core/PROJECT b/util-core/PROJECT index 79d52f717c..5c59c6a762 100644 --- a/util-core/PROJECT +++ b/util-core/PROJECT @@ -7,3 +7,5 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson diff --git a/util-jackson-annotations/PROJECT b/util-jackson-annotations/PROJECT index bd3e45cf4e..96e19de2ed 100644 --- a/util-jackson-annotations/PROJECT +++ b/util-jackson-annotations/PROJECT @@ -8,5 +8,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - csl-team@twitter.com diff --git a/util-jackson/PROJECT b/util-jackson/PROJECT index bd3e45cf4e..96e19de2ed 100644 --- a/util-jackson/PROJECT +++ b/util-jackson/PROJECT @@ -8,5 +8,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - csl-team@twitter.com diff --git a/util-logging/PROJECT b/util-logging/PROJECT index a482ca9a20..51e3ed72fc 100644 --- a/util-logging/PROJECT +++ b/util-logging/PROJECT @@ -7,5 +7,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - csl-team@twitter.com diff --git a/util-mock/PROJECT b/util-mock/PROJECT index fbed8cad0c..1c5441fe11 100644 --- a/util-mock/PROJECT +++ b/util-mock/PROJECT @@ -5,5 +5,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - csl-team:ldap diff --git a/util-security/PROJECT b/util-security/PROJECT index 828e37f74e..94a10cd86a 100644 --- a/util-security/PROJECT +++ b/util-security/PROJECT @@ -7,5 +7,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson - csl-core-security:phab - platform-security:ldap diff --git a/util-slf4j-api/PROJECT b/util-slf4j-api/PROJECT index a482ca9a20..51e3ed72fc 100644 --- a/util-slf4j-api/PROJECT +++ b/util-slf4j-api/PROJECT @@ -7,5 +7,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - csl-team@twitter.com diff --git a/util-slf4j-jul-bridge/PROJECT b/util-slf4j-jul-bridge/PROJECT index a482ca9a20..51e3ed72fc 100644 --- a/util-slf4j-jul-bridge/PROJECT +++ b/util-slf4j-jul-bridge/PROJECT @@ -7,5 +7,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - csl-team@twitter.com diff --git a/util-stats/PROJECT b/util-stats/PROJECT index fba5e978d3..20e3644d8c 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -9,3 +9,5 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson diff --git a/util-validator-constraints/PROJECT b/util-validator-constraints/PROJECT index 2541e2a188..96e19de2ed 100644 --- a/util-validator-constraints/PROJECT +++ b/util-validator-constraints/PROJECT @@ -8,5 +8,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - - csl-team@twitter.com \ No newline at end of file + - csl-team@twitter.com diff --git a/util-validator/PROJECT b/util-validator/PROJECT index 2541e2a188..96e19de2ed 100644 --- a/util-validator/PROJECT +++ b/util-validator/PROJECT @@ -8,5 +8,7 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - - csl-team@twitter.com \ No newline at end of file + - csl-team@twitter.com diff --git a/util-zk-test/PROJECT b/util-zk-test/PROJECT index a4d97bddf7..e1d56d6499 100644 --- a/util-zk-test/PROJECT +++ b/util-zk-test/PROJECT @@ -4,4 +4,6 @@ owners: - mnakamura - roanta - vkostyukov - - yufang \ No newline at end of file + - yufang + - mbezoyan + - mdickinson diff --git a/util-zk/PROJECT b/util-zk/PROJECT index 6199433a06..def0eb91d1 100644 --- a/util-zk/PROJECT +++ b/util-zk/PROJECT @@ -6,6 +6,8 @@ owners: - roanta - vkostyukov - yufang + - mbezoyan + - mdickinson watchers: - coord-team@twitter.com - csl-team:ldap From 9542ef8494d69d9e15a050b3673fceea52413df1 Mon Sep 17 00:00:00 2001 From: Jillian Crossley Date: Thu, 24 Nov 2022 19:15:24 +0000 Subject: [PATCH 752/796] Clean up CSL-owned PROJECT files Problem The owners in these files are no longer maintainers. Solution Clean up files and add new owners. Differential Revision: https://phabricator.twitter.biz/D1016550 --- PROJECT.yml | 13 +++---------- util-app/PROJECT | 8 ++------ util-benchmark/PROJECT | 7 ++----- util-codec/PROJECT | 7 ++----- util-core/PROJECT | 8 ++------ util-jackson-annotations/PROJECT | 9 ++------- util-jackson/PROJECT | 9 ++------- util-logging/PROJECT | 8 ++------ util-mock/PROJECT | 8 ++------ util-security/PROJECT | 6 ++---- util-slf4j-api/PROJECT | 8 ++------ util-slf4j-jul-bridge/PROJECT | 8 ++------ util-stats/PROJECT | 10 ++-------- util-validator-constraints/PROJECT | 9 ++------- util-validator/PROJECT | 9 ++------- util-zk-test/PROJECT | 7 ++----- util-zk/PROJECT | 7 ++----- 17 files changed, 35 insertions(+), 106 deletions(-) diff --git a/PROJECT.yml b/PROJECT.yml index eff27342aa..c9d152767c 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -1,14 +1,7 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - mnakamura - - roanta - - vkostyukov - - yufang -watchers: - csl-team:ldap - -fileOwners: - CHANGELOG.rst: - - csl-team:ldap + - jillianc + - mbezoyan + - nvong diff --git a/util-app/PROJECT b/util-app/PROJECT index 51e3ed72fc..2cc6737e78 100644 --- a/util-app/PROJECT +++ b/util-app/PROJECT @@ -1,13 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - ibennett - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - csl-team@twitter.com diff --git a/util-benchmark/PROJECT b/util-benchmark/PROJECT index 405c5cefe7..6dd4818bd6 100644 --- a/util-benchmark/PROJECT +++ b/util-benchmark/PROJECT @@ -1,10 +1,7 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong diff --git a/util-codec/PROJECT b/util-codec/PROJECT index 405c5cefe7..6dd4818bd6 100644 --- a/util-codec/PROJECT +++ b/util-codec/PROJECT @@ -1,10 +1,7 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong diff --git a/util-core/PROJECT b/util-core/PROJECT index 5c59c6a762..6dd4818bd6 100644 --- a/util-core/PROJECT +++ b/util-core/PROJECT @@ -1,11 +1,7 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - hallam - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong diff --git a/util-jackson-annotations/PROJECT b/util-jackson-annotations/PROJECT index 96e19de2ed..2cc6737e78 100644 --- a/util-jackson-annotations/PROJECT +++ b/util-jackson-annotations/PROJECT @@ -1,14 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - ibennett - - jyan - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - csl-team@twitter.com diff --git a/util-jackson/PROJECT b/util-jackson/PROJECT index 96e19de2ed..2cc6737e78 100644 --- a/util-jackson/PROJECT +++ b/util-jackson/PROJECT @@ -1,14 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - ibennett - - jyan - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - csl-team@twitter.com diff --git a/util-logging/PROJECT b/util-logging/PROJECT index 51e3ed72fc..2cc6737e78 100644 --- a/util-logging/PROJECT +++ b/util-logging/PROJECT @@ -1,13 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - ibennett - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - csl-team@twitter.com diff --git a/util-mock/PROJECT b/util-mock/PROJECT index 1c5441fe11..c9537aa408 100644 --- a/util-mock/PROJECT +++ b/util-mock/PROJECT @@ -1,11 +1,7 @@ owners: - - banderson - - ibennett - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - csl-team:ldap diff --git a/util-security/PROJECT b/util-security/PROJECT index 94a10cd86a..883a086213 100644 --- a/util-security/PROJECT +++ b/util-security/PROJECT @@ -3,11 +3,9 @@ owners: # Individual users are not responsible for providing a blocking review, but they # are capable of doing so in the event that members of the groups cannot. - - banderson - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong - csl-core-security:phab - platform-security:ldap diff --git a/util-slf4j-api/PROJECT b/util-slf4j-api/PROJECT index 51e3ed72fc..2cc6737e78 100644 --- a/util-slf4j-api/PROJECT +++ b/util-slf4j-api/PROJECT @@ -1,13 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - ibennett - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - csl-team@twitter.com diff --git a/util-slf4j-jul-bridge/PROJECT b/util-slf4j-jul-bridge/PROJECT index 51e3ed72fc..2cc6737e78 100644 --- a/util-slf4j-jul-bridge/PROJECT +++ b/util-slf4j-jul-bridge/PROJECT @@ -1,13 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - ibennett - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - csl-team@twitter.com diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 20e3644d8c..6dd4818bd6 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -1,13 +1,7 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - beisenman - - jyan - - lhe - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong diff --git a/util-validator-constraints/PROJECT b/util-validator-constraints/PROJECT index 96e19de2ed..2cc6737e78 100644 --- a/util-validator-constraints/PROJECT +++ b/util-validator-constraints/PROJECT @@ -1,14 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - ibennett - - jyan - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - csl-team@twitter.com diff --git a/util-validator/PROJECT b/util-validator/PROJECT index 96e19de2ed..2cc6737e78 100644 --- a/util-validator/PROJECT +++ b/util-validator/PROJECT @@ -1,14 +1,9 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - ibennett - - jyan - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - csl-team@twitter.com diff --git a/util-zk-test/PROJECT b/util-zk-test/PROJECT index e1d56d6499..d207c2c965 100644 --- a/util-zk-test/PROJECT +++ b/util-zk-test/PROJECT @@ -1,9 +1,6 @@ owners: - coord-review:ldap - - banderson - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong diff --git a/util-zk/PROJECT b/util-zk/PROJECT index def0eb91d1..19ba96f919 100644 --- a/util-zk/PROJECT +++ b/util-zk/PROJECT @@ -1,13 +1,10 @@ # See go/project-docs. Generated using ./pants run src/python/twitter/phabricator:owners_to_project -- --dirs=a/owned/path,b/owned/path owners: - - banderson - - mnakamura - - roanta - - vkostyukov - - yufang + - jillianc - mbezoyan - mdickinson + - nvong watchers: - coord-team@twitter.com - csl-team:ldap From 1a6ff6767ae4faff0d0f8ca2606c1a3943cffe7c Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Mon, 19 Dec 2022 16:59:51 +0000 Subject: [PATCH 753/796] [source] bump Jackson version to 2.14.1 Problem / Solution New Jackson version is out. Let's upgrade 2.13.4 to 2.14.1. JIRA Issues: DSTR-11384 Differential Revision: https://phabricator.twitter.biz/D1025778 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 717945d7e5..2c064c62b3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -30,6 +30,8 @@ Deprecations Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ +* util: Bump version of Jackson to 2.14.1. ``PHAB_ID=D1025778`` + * util-core: `Time.at`, `Time.toString`, and `Time.fromRss` now use `TimeFormatter` internally and thus have better performance and do not cause contention through synchronization. `toString` always uses `TimeFormatter` while `at` and `fromRss` only use it when run with Java 9 or later due to a diff --git a/build.sbt b/build.sbt index e483451629..b9abac63ea 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "22.10.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.13.4" +val jacksonVersion = "2.14.1" val json4sVersion = "4.0.3" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" From 2eecea1b1d2ae11de2b23c24e16fa39cdfc07a02 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Fri, 23 Dec 2022 17:25:13 +0000 Subject: [PATCH 754/796] Twitter-oss: Prepare OSS libraries for release 22.12.0 Problem We want to release the next versions of our Twitter OSS libraries 22.12.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D1026958 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2c064c62b3..6d1f7c1c4c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +22.12.0 +------- + New Features ~~~~~~~~~~~~ diff --git a/README.md b/README.md index 3f46da943f..a4ea5177b0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12) @@ -39,7 +39,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "22.7.0" +val utilCore = "com.twitter" %% "util-core" % "22.12.0" ``` # Units diff --git a/build.sbt b/build.sbt index b9abac63ea..853d4a76bd 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.10.0-SNAPSHOT" +val releaseVersion = "22.12.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.1" From 2f2f8c8748a04da02f1de554a38c6e041703b618 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Mon, 26 Dec 2022 04:22:12 +0000 Subject: [PATCH 755/796] Twitter-oss: update OSS libraries post-release to 23.3.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 23.3.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D1029024 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a4ea5177b0..8c891d2e80 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12) diff --git a/build.sbt b/build.sbt index 853d4a76bd..d14a48c63c 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "22.12.0" +val releaseVersion = "23.3.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.1" From ed33ca95c5b16aab5dc3bb6ecaab33838d90eb38 Mon Sep 17 00:00:00 2001 From: Mikhail Bezoyan Date: Wed, 4 Jan 2023 21:10:07 +0000 Subject: [PATCH 756/796] [finagle-stats] Introducing a thread-local bucketed histogram for Stat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old implementation in case of heavy contention becomes a bottleneck. This happens quite often when a metric is on a hot path and updated simultaneously by multiple threads. In some extreme cases stats collection could be responsible for nearly 10% percent cpu usage. The proposed solution trades increased memory usage for almost non-existent contention among threads by keeping a separate histogram for each thread that is accessed only by either the owning thread or by the thread that collect statistics from the histogram. The result below are collected on an Intel Xeon Gold 6122 and Java11 ``` Benchmark Mode Cnt Score Error Units BucketedHistogramBenchmark.addLockFree_Tx1 avgt 5 41.600 ± 1.061 ns/op BucketedHistogramBenchmark.addLockFree_Tx16 avgt 5 41.397 ± 0.109 ns/op BucketedHistogramBenchmark.addLockFree_Tx8 avgt 5 40.713 ± 0.343 ns/op BucketedHistogramBenchmark.add_Tx1 avgt 5 57.067 ± 3.355 ns/op BucketedHistogramBenchmark.add_Tx16 avgt 5 2193.103 ± 129.428 ns/op BucketedHistogramBenchmark.add_Tx8 avgt 5 1006.465 ± 308.512 ns/op ``` The performance is better even in the case without any contention. With more threads trying to update the stat concurrently the difference becomes enormous. To enable the new behaviour for your stat, please use the following template ``` statsReceiver.stat( MetricBuilder.forStat .withName(name) .withMetricUsageHints(Set(MetricUsageHint.HighContention))) ``` Differential Revision: https://phabricator.twitter.biz/D1028784 --- .../twitter/finagle/stats/MetricBuilder.scala | 40 +++++++++++-------- .../finagle/stats/MetricUsageHint.scala | 7 ++++ 2 files changed, 31 insertions(+), 16 deletions(-) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/MetricUsageHint.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index ee53b88056..12ba7bb94e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -110,21 +110,22 @@ object MetricBuilder { metricType: MetricType, ): MetricBuilder = { new MetricBuilder( - keyIndicator, - description, - units, - role, - verbosity, - sourceClass, + keyIndicator = keyIndicator, + description = description, + units = units, + role = role, + verbosity = verbosity, + sourceClass = sourceClass, // For now we're not allowing construction w"ith labels. // They need to be added later via `.withLabels`. - Identity(name, name), - relativeName, - processPath, - histogramFormat, - percentiles, - isStandard, - metricType + identity = Identity(name, name), + relativeName = relativeName, + processPath = processPath, + histogramFormat = histogramFormat, + percentiles = percentiles, + isStandard = isStandard, + metricType = metricType, + metricUsageHints = Set.empty ) } @@ -302,7 +303,8 @@ class MetricBuilder private ( val histogramFormat: HistogramFormat, val percentiles: IndexedSeq[Double], val isStandard: Boolean, - val metricType: MetricType) + val metricType: MetricType, + val metricUsageHints: Set[MetricUsageHint]) extends Metadata { /** @@ -323,7 +325,8 @@ class MetricBuilder private ( isStandard: Boolean = this.isStandard, histogramFormat: HistogramFormat = this.histogramFormat, percentiles: IndexedSeq[Double] = this.percentiles, - metricType: MetricType = this.metricType + metricType: MetricType = this.metricType, + metricUsageHints: Set[MetricUsageHint] = this.metricUsageHints ): MetricBuilder = { new MetricBuilder( keyIndicator = keyIndicator, @@ -338,7 +341,8 @@ class MetricBuilder private ( histogramFormat = histogramFormat, percentiles = percentiles, isStandard = isStandard, - metricType = metricType + metricType = metricType, + metricUsageHints = metricUsageHints ) } @@ -413,6 +417,10 @@ class MetricBuilder private ( this.copy(metricType = CounterishGaugeType) } + def withMetricUsageHints(hints: Set[MetricUsageHint]): MetricBuilder = { + this.copy(metricUsageHints = hints) + } + /** * Generates a CounterType MetricBuilder which can be used to create a counter in a StatsReceiver. * Used to test that builder class correctly propagates configured metadata. diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUsageHint.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUsageHint.scala new file mode 100644 index 0000000000..58b39c7759 --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricUsageHint.scala @@ -0,0 +1,7 @@ +package com.twitter.finagle.stats + +sealed trait MetricUsageHint {} + +object MetricUsageHint { + object HighContention extends MetricUsageHint +} From be906ed9405e9e61b862e64a6dc8182eb9f52d31 Mon Sep 17 00:00:00 2001 From: James Meador Date: Fri, 27 Jan 2023 17:49:25 +0000 Subject: [PATCH 757/796] [owners culling] Delete ex-employees from PROJECT files; delete sandboxes Differential Revision: https://phabricator.twitter.biz/D1039859 --- util-app/PROJECT | 1 - util-benchmark/PROJECT | 1 - util-codec/PROJECT | 1 - util-core/PROJECT | 1 - util-jackson-annotations/PROJECT | 1 - util-jackson/PROJECT | 1 - util-logging/PROJECT | 1 - util-mock/PROJECT | 1 - util-security/PROJECT | 1 - util-slf4j-api/PROJECT | 1 - util-slf4j-jul-bridge/PROJECT | 1 - util-stats/PROJECT | 1 - util-validator-constraints/PROJECT | 1 - util-validator/PROJECT | 1 - util-zk-test/PROJECT | 1 - util-zk/PROJECT | 1 - 16 files changed, 16 deletions(-) diff --git a/util-app/PROJECT b/util-app/PROJECT index 2cc6737e78..563cf7daa4 100644 --- a/util-app/PROJECT +++ b/util-app/PROJECT @@ -4,6 +4,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - csl-team@twitter.com diff --git a/util-benchmark/PROJECT b/util-benchmark/PROJECT index 6dd4818bd6..4edda67ec7 100644 --- a/util-benchmark/PROJECT +++ b/util-benchmark/PROJECT @@ -4,4 +4,3 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong diff --git a/util-codec/PROJECT b/util-codec/PROJECT index 6dd4818bd6..4edda67ec7 100644 --- a/util-codec/PROJECT +++ b/util-codec/PROJECT @@ -4,4 +4,3 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong diff --git a/util-core/PROJECT b/util-core/PROJECT index 6dd4818bd6..4edda67ec7 100644 --- a/util-core/PROJECT +++ b/util-core/PROJECT @@ -4,4 +4,3 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong diff --git a/util-jackson-annotations/PROJECT b/util-jackson-annotations/PROJECT index 2cc6737e78..563cf7daa4 100644 --- a/util-jackson-annotations/PROJECT +++ b/util-jackson-annotations/PROJECT @@ -4,6 +4,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - csl-team@twitter.com diff --git a/util-jackson/PROJECT b/util-jackson/PROJECT index 2cc6737e78..563cf7daa4 100644 --- a/util-jackson/PROJECT +++ b/util-jackson/PROJECT @@ -4,6 +4,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - csl-team@twitter.com diff --git a/util-logging/PROJECT b/util-logging/PROJECT index 2cc6737e78..563cf7daa4 100644 --- a/util-logging/PROJECT +++ b/util-logging/PROJECT @@ -4,6 +4,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - csl-team@twitter.com diff --git a/util-mock/PROJECT b/util-mock/PROJECT index c9537aa408..c9bd2bfcf8 100644 --- a/util-mock/PROJECT +++ b/util-mock/PROJECT @@ -2,6 +2,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - csl-team:ldap diff --git a/util-security/PROJECT b/util-security/PROJECT index 883a086213..3c5d5f9adb 100644 --- a/util-security/PROJECT +++ b/util-security/PROJECT @@ -6,6 +6,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong - csl-core-security:phab - platform-security:ldap diff --git a/util-slf4j-api/PROJECT b/util-slf4j-api/PROJECT index 2cc6737e78..563cf7daa4 100644 --- a/util-slf4j-api/PROJECT +++ b/util-slf4j-api/PROJECT @@ -4,6 +4,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - csl-team@twitter.com diff --git a/util-slf4j-jul-bridge/PROJECT b/util-slf4j-jul-bridge/PROJECT index 2cc6737e78..563cf7daa4 100644 --- a/util-slf4j-jul-bridge/PROJECT +++ b/util-slf4j-jul-bridge/PROJECT @@ -4,6 +4,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - csl-team@twitter.com diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 6dd4818bd6..4edda67ec7 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -4,4 +4,3 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong diff --git a/util-validator-constraints/PROJECT b/util-validator-constraints/PROJECT index 2cc6737e78..563cf7daa4 100644 --- a/util-validator-constraints/PROJECT +++ b/util-validator-constraints/PROJECT @@ -4,6 +4,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - csl-team@twitter.com diff --git a/util-validator/PROJECT b/util-validator/PROJECT index 2cc6737e78..563cf7daa4 100644 --- a/util-validator/PROJECT +++ b/util-validator/PROJECT @@ -4,6 +4,5 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - csl-team@twitter.com diff --git a/util-zk-test/PROJECT b/util-zk-test/PROJECT index d207c2c965..fe357575b2 100644 --- a/util-zk-test/PROJECT +++ b/util-zk-test/PROJECT @@ -3,4 +3,3 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong diff --git a/util-zk/PROJECT b/util-zk/PROJECT index 19ba96f919..07b1de2163 100644 --- a/util-zk/PROJECT +++ b/util-zk/PROJECT @@ -4,7 +4,6 @@ owners: - jillianc - mbezoyan - mdickinson - - nvong watchers: - coord-team@twitter.com - csl-team:ldap From 478c1d1d3967b62fc97647411caf774dbb789800 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Thu, 23 Feb 2023 02:13:07 +0000 Subject: [PATCH 758/796] [source] bump Jackson version to 2.14.2 Problem: Jackson dep is outdated Solution: Update to 2.14.2 Differential Revision: https://phabricator.twitter.biz/D1049772 --- CHANGELOG.rst | 5 +++++ build.sbt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6d1f7c1c4c..fd637e9783 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util: Bump version of Jackson to 2.14.2. ``PHAB_ID=D1049772`` + 22.12.0 ------- diff --git a/build.sbt b/build.sbt index d14a48c63c..053e6dbf4c 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "23.3.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.14.1" +val jacksonVersion = "2.14.2" val json4sVersion = "4.0.3" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" From 143adbdf5bc6dc55eddb9248c597eeb11a799094 Mon Sep 17 00:00:00 2001 From: Anton Ivanov Date: Tue, 11 Apr 2023 22:47:51 +0000 Subject: [PATCH 759/796] [util/util-core] optimise Local context substitution in Promise # Problem Async profiler shows these two methods having most CPU usage: ``` ns percent samples top ---------- ------- ------- --- 617846976340 13.98% 57290 com.twitter.util.Local$.com$twitter$util$Local$$saveRef 253182619190 5.73% 23158 com.twitter.util.Local$.restore ``` This is the most CPU consuming stack: ``` --- util-core/src/benchmark/scala/BUILD.bazel | 29 +++++++++++++++ .../com/twitter/util/LocalBenchmarks.scala | 29 +++++++++++++++ .../main/scala/com/twitter/util/Promise.scala | 35 ++++++++----------- 3 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 util-core/src/benchmark/scala/BUILD.bazel create mode 100644 util-core/src/benchmark/scala/com/twitter/util/LocalBenchmarks.scala diff --git a/util-core/src/benchmark/scala/BUILD.bazel b/util-core/src/benchmark/scala/BUILD.bazel new file mode 100644 index 0000000000..7f9fa00b5a --- /dev/null +++ b/util-core/src/benchmark/scala/BUILD.bazel @@ -0,0 +1,29 @@ +# Run locally: +# ./bazel run util/util-core/src/benchmark/scala $BENCHMARK_NAME_REGEXP +# +# Run on a remote host: +# Bundle: +# ./bazel bundle util/util-core/src/benchmark/scala +# Copy jars to a target host: +# rsync -Lrv --delete dist/util-core-jmh-bundle* $TARGET_HOST: +# Run on the remote host: +# ssh $TARGET_HOST /usr/lib/jvm/java-11-twitter/bin/java -jar util-core-jmh-bundle/util-core-jmh.jar $BENCHMARK_NAME_REGEXP +jvm_binary( + basename = "util-core-jmh", + main = "org.openjdk.jmh.Main", + runtime_platform = "java11", + dependencies = [ + "util/util-core/src/benchmark/scala:jmh_compiled_benchmark_lib", + ], +) + +scala_benchmark_jmh( + name = "jmh", + sources = [ + "**/*.scala", + ], + compiler_option_sets = ["fatal_warnings"], + dependencies = [ + "util/util-core", + ], +) diff --git a/util-core/src/benchmark/scala/com/twitter/util/LocalBenchmarks.scala b/util-core/src/benchmark/scala/com/twitter/util/LocalBenchmarks.scala new file mode 100644 index 0000000000..0aff5b3e7c --- /dev/null +++ b/util-core/src/benchmark/scala/com/twitter/util/LocalBenchmarks.scala @@ -0,0 +1,29 @@ +package com.twitter.util + +import com.twitter.util.Local.Context +import java.util.concurrent.ThreadLocalRandom +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.Warmup +import org.openjdk.jmh.infra.Blackhole + +@State(Scope.Benchmark) +@Warmup(iterations = 3) +@Fork(2) +class LocalBenchmarks { + + private[this] val key = new Local.Key + + @Benchmark + def let(blackhole: Blackhole): Unit = { + val randLong = ThreadLocalRandom.current().nextLong() + val newContext = Context.empty.set(key, Some(randLong)) + + Local.let(newContext) { + blackhole.consume(randLong) + } + } + +} diff --git a/util-core/src/main/scala/com/twitter/util/Promise.scala b/util-core/src/main/scala/com/twitter/util/Promise.scala index f7ef58e720..bb51047232 100644 --- a/util-core/src/main/scala/com/twitter/util/Promise.scala +++ b/util-core/src/main/scala/com/twitter/util/Promise.scala @@ -214,12 +214,10 @@ object Promise { private final class Monitored[A](saved: Local.Context, k: Try[A] => Unit) extends K[A](saved) { def apply(result: Try[A]): Unit = { - val current = Local.save() - if (current ne saved) - Local.restore(saved) - try k(result) - catch Monitor.catcher - finally Local.restore(current) + Local.let(saved) { + try k(result) + catch Monitor.catcher + } } } @@ -228,15 +226,14 @@ object Promise { protected[this] def k(r: Try[A]): Unit final def apply(result: Try[A]): Unit = { - val current = Local.save() - if (current ne saved) - Local.restore(saved) - try k(result) - catch { - case t: Throwable => - Monitor.handle(t) - throw t - } finally Local.restore(current) + Local.let(saved) { + try k(result) + catch { + case t: Throwable => + Monitor.handle(t) + throw t + } + } } } @@ -615,11 +612,9 @@ class Promise[A] extends Future[A] with Updatable[Try[A]] { case s: Interruptible[A] => if (!cas(s, new Interrupted(WaitQueue.merge(wq, s.waitq), intr))) raise0(intr, wq) else { - val current = Local.save() - if (current ne s.saved) - Local.restore(s.saved) - try s.handler.applyOrElse(intr, Promise.AlwaysUnit) - finally Local.restore(current) + Local.let(s.saved) { + s.handler.applyOrElse(intr, Promise.AlwaysUnit) + } } case p: Promise[A] /* Linked */ => p.raise0(intr, wq) From 1904b60ab81c07e1257f30dca9d41494899148a9 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Tue, 9 May 2023 19:13:38 +0000 Subject: [PATCH 760/796] [source] bump jackson to 2.14.3 from 2.14.2 Problem/Solution: Update Jackson deps from 2.14.2 to 2.14.3 Differential Revision: https://phabricator.twitter.biz/D1069160 --- CHANGELOG.rst | 2 ++ build.sbt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fd637e9783..2c735556bf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ Runtime Behavior Changes * util: Bump version of Jackson to 2.14.2. ``PHAB_ID=D1049772`` +* util: Bump version of Jackson to 2.14.3. ``PHAB_ID=D1069160`` + 22.12.0 ------- diff --git a/build.sbt b/build.sbt index 053e6dbf4c..ef572dfff7 100644 --- a/build.sbt +++ b/build.sbt @@ -5,7 +5,7 @@ Global / excludeLintKeys += scalacOptions // might be actually unused in util-do val releaseVersion = "23.3.0-SNAPSHOT" val slf4jVersion = "1.7.30" -val jacksonVersion = "2.14.2" +val jacksonVersion = "2.14.3" val json4sVersion = "4.0.3" val mockitoVersion = "3.3.3" val mockitoScalaVersion = "1.14.8" From 17e76b341830c87e509c9a10fd7f66375eb69252 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Fri, 22 Sep 2023 14:13:17 +0000 Subject: [PATCH 761/796] [Twitter-oss] bump release version [Twitter-oss] bump release version Differential Revision: https://phabricator.twitter.biz/D1100185 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index ef572dfff7..9a68d999be 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "23.3.0-SNAPSHOT" +val releaseVersion = "23.9.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.3" From 0571f975d9d7f01641b320d5b74ec794bde7bbbf Mon Sep 17 00:00:00 2001 From: andreoss Date: Tue, 31 Oct 2023 19:34:39 +0000 Subject: [PATCH 762/796] [util]: Portable shebang Problem: /bin/bash is not available on non-FHS distros, such as NixOS Solution: Replace /bin/bash with /usr/bin/env bash Result: Closes twitter/util#314 Differential Revision: https://phabricator.twitter.biz/D1107606 --- pushsite.bash | 2 +- sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pushsite.bash b/pushsite.bash index dab72393d9..a3d6a8ceb0 100755 --- a/pushsite.bash +++ b/pushsite.bash @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/sbt b/sbt index d08998b12d..dbe5a02103 100755 --- a/sbt +++ b/sbt @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eo pipefail From 053248d1baa2708b821591e3cfc8bff92e3061d2 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Tue, 31 Oct 2023 20:22:37 +0000 Subject: [PATCH 763/796] [util-security] add deserializeAndFilterOutInvalidCertificates Problem/Solution: Add `deserializeAndFilterOutInvalidCertificates` which wraps the `deserializeX509` call in a `Try` (as `certificate.checkValidity()` can return `CertificateExpiredException`, `CertificateNotYetValidException`) and separates out any expired or not yet valid certificates detected. JIRA Issues: PSEC-16977 Differential Revision: https://phabricator.twitter.biz/D1107551 --- CHANGELOG.rst | 3 +++ .../X509CertificateDeserializer.scala | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2c735556bf..dbd628b262 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,9 @@ Runtime Behavior Changes * util: Bump version of Jackson to 2.14.3. ``PHAB_ID=D1069160`` +* util-securty: Add `deserializeAndFilterOutInvalidCertificates` Which wraps + the `deserializeX509` call in a Try. ``PHAB_ID=D1107551`` + 22.12.0 ------- diff --git a/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala b/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala index 9b2b36f34d..48ab081d3e 100644 --- a/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala +++ b/util-security/src/main/scala/com/twitter/util/security/X509CertificateDeserializer.scala @@ -52,4 +52,28 @@ object X509CertificateDeserializer { messages.map(_.map(deserializeX509)) } + + /** + * Deserializes an [[InputStream]] that contains PEM-encoded X.509 + * Certificates. Wraps the `deserializeX509` call in a Try + * (as `certificate.checkValidity()` can return CertificateExpiredException, CertificateNotYetValidException) + * and separates out any expired or not yet valid certificates detected. + * + * Closes the InputStream once it has finished reading. + */ + def deserializeAndFilterOutInvalidCertificates( + rawPem: String, + name: String + ): (Seq[Try[X509Certificate]], Seq[Try[X509Certificate]]) = { + val pemBytes = new PemBytes(rawPem, name) + val messages: Try[Seq[Array[Byte]]] = pemBytes + .readMessages(MessageType) + messages + .map(certs => { + certs + .map(cert => { + Try(deserializeX509(cert)) + }).partition(_.isReturn) + }).get() + } } From 5873cb6eae2947f4596d18e57effc7b4ab290599 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Wed, 1 Nov 2023 13:44:53 +0000 Subject: [PATCH 764/796] [Twitter-oss] bump release version [Twitter-oss] bump release version Differential Revision: https://phabricator.twitter.biz/D1107767 --- build.sbt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 9a68d999be..676176a2f4 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "23.9.0-SNAPSHOT" +val releaseVersion = "23.11.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.3" @@ -712,12 +712,12 @@ lazy val utilValidatorConstraint = Project( id = "util-validator-constraints", base = file("util-validator-constraints") ).settings( - settingsCrossCompiledWithTwoTen -).settings( - libraryDependencies ++= Seq( - "jakarta.validation" % "jakarta.validation-api" % "3.0.0" + settingsCrossCompiledWithTwoTen + ).settings( + libraryDependencies ++= Seq( + "jakarta.validation" % "jakarta.validation-api" % "3.0.0" + ) ) -) lazy val utilZk = Project( id = "util-zk", From 1507298760aea0d5fa0aeff4a749999ae2258dc6 Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Wed, 1 Nov 2023 23:58:08 +0000 Subject: [PATCH 765/796] Cut version 23.11 Problem/Solution: Cut version 23.11 of: - util - scrooge - finagle - twitter-server - finatra Differential Revision: https://phabricator.twitter.biz/D1107956 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dbd628b262..43fa096fe4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +23.11.0 +------- + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 8c891d2e80..8e58bd0336 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12) @@ -39,7 +39,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "22.12.0" +val utilCore = "com.twitter" %% "util-core" % "23.11.0" ``` # Units diff --git a/build.sbt b/build.sbt index 676176a2f4..03688bb4ab 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "23.11.0-SNAPSHOT" +val releaseVersion = "23.11.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.3" From 69b940a69d0a7a0a3ea5c678f71b36462b77024e Mon Sep 17 00:00:00 2001 From: Matthew Dickinson Date: Thu, 2 Nov 2023 09:22:49 +0000 Subject: [PATCH 766/796] update to snapshot 24.2.0-snapshot Problem/Solution: Cut snapshot 24.2.0 of: - util - scrooge - finagle - twitter-server - finatra Differential Revision: https://phabricator.twitter.biz/D1108018 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e58bd0336..30637b974d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12) diff --git a/build.sbt b/build.sbt index 03688bb4ab..88544e2df5 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "23.11.0" +val releaseVersion = "24.2.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.3" From 644094e88f352ddb13b78278e01dfe4dc00c6b4e Mon Sep 17 00:00:00 2001 From: Jillian Crossley Date: Thu, 21 Dec 2023 23:06:56 +0000 Subject: [PATCH 767/796] util/util-app: Include the exception in the error message printed when the application encounters an error on startup Problem When the application exits prematurely due to an exception encountered on startup, this exception is not printed to stderr, only the message "Exception thrown in main on startup". The exception stack trace is printed to stdout, but this is unintuitive for the user. Solution Include the exception and stack trace in the exception printed to stderr. Differential Revision: https://phabricator.twitter.biz/D1116753 --- CHANGELOG.rst | 7 +++++++ PROJECT.yml | 1 - util-app/src/main/scala/com/twitter/app/App.scala | 10 +++++++++- util-app/src/test/scala/com/twitter/app/AppTest.scala | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 43fa096fe4..51b4ff60d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,13 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +Runtime Behavior Changes +~~~~~~~~~~~~~~~~~~~~~~~~ + +* util-app: When the application exits due to an error on startup, the error and + and stack trace are printed to stderr in addition to the existing stdout. ``PHAB_ID=D1116753`` + + 23.11.0 ------- diff --git a/PROJECT.yml b/PROJECT.yml index c9d152767c..675c29685a 100644 --- a/PROJECT.yml +++ b/PROJECT.yml @@ -4,4 +4,3 @@ owners: - csl-team:ldap - jillianc - mbezoyan - - nvong diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index cb4c916550..167fa1b6ec 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -3,6 +3,8 @@ package com.twitter.app import com.twitter.app.lifecycle.Event._ import com.twitter.conversions.DurationOps._ import com.twitter.util._ +import java.io.PrintWriter +import java.io.StringWriter import java.lang.reflect.InvocationTargetException import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicReference @@ -82,7 +84,13 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { System.err.println(throwable.getMessage) System.exit(1) case _ => - exitOnError("Exception thrown in main on startup") + util.Using(new StringWriter) { stringWriter => + util.Using(new PrintWriter(stringWriter)) { printWriter => + throwable.printStackTrace(printWriter) + printWriter.flush() + exitOnError(s"Exception thrown in main on startup: ${stringWriter.toString}") + } + } } } diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index 5c70b2f126..a6c9eedcdb 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -67,7 +67,7 @@ class AppTest extends AnyFunSuite { test1.main(Array()) - assert(test1.reason.contains("Exception thrown in main on startup")) + assert(test1.reason.exists(_.contains("Exception thrown in main on startup"))) } test("App: propagate underlying exception from fields in app") { From 82e0bdbad58db8180e344b7f2ee5c96b626509b7 Mon Sep 17 00:00:00 2001 From: Mikhail Bezoyan Date: Thu, 25 Jan 2024 17:08:32 +0000 Subject: [PATCH 768/796] [finagle-stats] Do not allocate a new IndexedSeq on every metric lookup Differential Revision: https://phabricator.twitter.biz/D1121923 --- .../main/scala/com/twitter/finagle/stats/MetricBuilder.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index 12ba7bb94e..de95e7b5d1 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -78,6 +78,8 @@ object MetricBuilder { val NoDescription: String = "No description provided" + private[this] val EmptyPercentiles = IndexedSeq.empty + /** * Construct a MethodBuilder. * @@ -105,7 +107,8 @@ object MetricBuilder { relativeName: Seq[String] = Seq.empty, processPath: Option[String] = None, histogramFormat: HistogramFormat = HistogramFormat.Default, - percentiles: IndexedSeq[Double] = IndexedSeq.empty, + percentiles: IndexedSeq[Double] = + EmptyPercentiles, // IndexedSeq.empty creates a new collection on every call isStandard: Boolean = false, metricType: MetricType, ): MetricBuilder = { From cbb1330fbb4aa4641421b670b0e17d2935f2b57b Mon Sep 17 00:00:00 2001 From: Tapan Shah Date: Fri, 9 Feb 2024 16:16:38 +0000 Subject: [PATCH 769/796] [source] Update PROJECT owner references to reflect deactivated ldap [source] Update PROJECT owner references to reflect deactivated ldap Differential Revision: https://phabricator.twitter.biz/D1125334 --- util-app/PROJECT | 1 - util-benchmark/PROJECT | 1 - util-core/PROJECT | 1 - util-jackson-annotations/PROJECT | 1 - util-jackson/PROJECT | 2 +- util-logging/PROJECT | 1 - util-mock/PROJECT | 1 - util-security/PROJECT | 2 +- util-slf4j-jul-bridge/PROJECT | 1 - util-stats/PROJECT | 1 - util-validator-constraints/PROJECT | 1 - util-validator/PROJECT | 1 - util-zk-test/PROJECT | 1 - util-zk/PROJECT | 1 - 14 files changed, 2 insertions(+), 14 deletions(-) diff --git a/util-app/PROJECT b/util-app/PROJECT index 563cf7daa4..cda5765dfa 100644 --- a/util-app/PROJECT +++ b/util-app/PROJECT @@ -3,6 +3,5 @@ owners: - jillianc - mbezoyan - - mdickinson watchers: - csl-team@twitter.com diff --git a/util-benchmark/PROJECT b/util-benchmark/PROJECT index 4edda67ec7..009bd1c5fb 100644 --- a/util-benchmark/PROJECT +++ b/util-benchmark/PROJECT @@ -3,4 +3,3 @@ owners: - jillianc - mbezoyan - - mdickinson diff --git a/util-core/PROJECT b/util-core/PROJECT index 4edda67ec7..009bd1c5fb 100644 --- a/util-core/PROJECT +++ b/util-core/PROJECT @@ -3,4 +3,3 @@ owners: - jillianc - mbezoyan - - mdickinson diff --git a/util-jackson-annotations/PROJECT b/util-jackson-annotations/PROJECT index 563cf7daa4..cda5765dfa 100644 --- a/util-jackson-annotations/PROJECT +++ b/util-jackson-annotations/PROJECT @@ -3,6 +3,5 @@ owners: - jillianc - mbezoyan - - mdickinson watchers: - csl-team@twitter.com diff --git a/util-jackson/PROJECT b/util-jackson/PROJECT index 563cf7daa4..c37ef6c2b7 100644 --- a/util-jackson/PROJECT +++ b/util-jackson/PROJECT @@ -3,6 +3,6 @@ owners: - jillianc - mbezoyan - - mdickinson + - tapans watchers: - csl-team@twitter.com diff --git a/util-logging/PROJECT b/util-logging/PROJECT index 563cf7daa4..cda5765dfa 100644 --- a/util-logging/PROJECT +++ b/util-logging/PROJECT @@ -3,6 +3,5 @@ owners: - jillianc - mbezoyan - - mdickinson watchers: - csl-team@twitter.com diff --git a/util-mock/PROJECT b/util-mock/PROJECT index c9bd2bfcf8..0f66d02b27 100644 --- a/util-mock/PROJECT +++ b/util-mock/PROJECT @@ -1,6 +1,5 @@ owners: - jillianc - mbezoyan - - mdickinson watchers: - csl-team:ldap diff --git a/util-security/PROJECT b/util-security/PROJECT index 3c5d5f9adb..291d1a5373 100644 --- a/util-security/PROJECT +++ b/util-security/PROJECT @@ -5,6 +5,6 @@ owners: # are capable of doing so in the event that members of the groups cannot. - jillianc - mbezoyan - - mdickinson + - tapans - csl-core-security:phab - platform-security:ldap diff --git a/util-slf4j-jul-bridge/PROJECT b/util-slf4j-jul-bridge/PROJECT index 563cf7daa4..cda5765dfa 100644 --- a/util-slf4j-jul-bridge/PROJECT +++ b/util-slf4j-jul-bridge/PROJECT @@ -3,6 +3,5 @@ owners: - jillianc - mbezoyan - - mdickinson watchers: - csl-team@twitter.com diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 4edda67ec7..009bd1c5fb 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -3,4 +3,3 @@ owners: - jillianc - mbezoyan - - mdickinson diff --git a/util-validator-constraints/PROJECT b/util-validator-constraints/PROJECT index 563cf7daa4..cda5765dfa 100644 --- a/util-validator-constraints/PROJECT +++ b/util-validator-constraints/PROJECT @@ -3,6 +3,5 @@ owners: - jillianc - mbezoyan - - mdickinson watchers: - csl-team@twitter.com diff --git a/util-validator/PROJECT b/util-validator/PROJECT index 563cf7daa4..cda5765dfa 100644 --- a/util-validator/PROJECT +++ b/util-validator/PROJECT @@ -3,6 +3,5 @@ owners: - jillianc - mbezoyan - - mdickinson watchers: - csl-team@twitter.com diff --git a/util-zk-test/PROJECT b/util-zk-test/PROJECT index fe357575b2..1aaeccf787 100644 --- a/util-zk-test/PROJECT +++ b/util-zk-test/PROJECT @@ -2,4 +2,3 @@ owners: - coord-review:ldap - jillianc - mbezoyan - - mdickinson diff --git a/util-zk/PROJECT b/util-zk/PROJECT index 07b1de2163..08d46e03da 100644 --- a/util-zk/PROJECT +++ b/util-zk/PROJECT @@ -3,7 +3,6 @@ owners: - jillianc - mbezoyan - - mdickinson watchers: - coord-team@twitter.com - csl-team:ldap From ee95c70b924506098d89f4974d53dbc93725725b Mon Sep 17 00:00:00 2001 From: Chetan Tutika Date: Thu, 2 May 2024 15:30:24 +0000 Subject: [PATCH 770/796] Twitter-oss: Prepare OSS libraries for release 24.2.0 Problem We want to release the next versions of our Twitter OSS libraries 24.2.0 - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D1140781 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 51b4ff60d0..ebda71d435 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +24.2.0 +------ + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 30637b974d..5b957b3f6a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12) @@ -39,7 +39,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "23.11.0" +val utilCore = "com.twitter" %% "util-core" % "24.2.0" ``` # Units diff --git a/build.sbt b/build.sbt index 88544e2df5..0d92ce55f3 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "24.2.0-SNAPSHOT" +val releaseVersion = "24.2.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.3" From 3af4bdd12ee1d1cfb24c3a82efd5c24177edf2b4 Mon Sep 17 00:00:00 2001 From: Chetan Tutika Date: Mon, 6 May 2024 18:03:23 +0000 Subject: [PATCH 771/796] Revert "Twitter-oss: Prepare OSS libraries for release 24.2.0" This reverts commit 996858fd593b43d101d3c546d5a9968dc5408855. There were issues with the release. Will be retrying it Differential Revision: https://phabricator.twitter.biz/D1141484 --- CHANGELOG.rst | 3 --- README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ebda71d435..51b4ff60d0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,9 +7,6 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- -24.2.0 ------- - Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 5b957b3f6a..30637b974d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12) @@ -39,7 +39,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "24.2.0" +val utilCore = "com.twitter" %% "util-core" % "23.11.0" ``` # Units diff --git a/build.sbt b/build.sbt index 0d92ce55f3..88544e2df5 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "24.2.0" +val releaseVersion = "24.2.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.3" From f704bc9aff837934888da390941c8fd316efd51d Mon Sep 17 00:00:00 2001 From: Chetan Tutika Date: Mon, 6 May 2024 20:21:10 +0000 Subject: [PATCH 772/796] Twitter-oss: Prepare OSS libraries for release 24.5.0 Problem We want to release the next versions of our Twitter OSS libraries 24.5.0 util scrooge finagle twitter-server finatra Solution Prepare libraries for their next releases. Differential Revision: https://phabricator.twitter.biz/D1141504 --- CHANGELOG.rst | 3 +++ README.md | 4 ++-- build.sbt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 51b4ff60d0..4526dfba24 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +24.5.0 +------ + Runtime Behavior Changes ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/README.md b/README.md index 30637b974d..a7717cfdde 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12) @@ -39,7 +39,7 @@ for more details about how to contribute. An example SBT dependency string for the `util-core` library would look like this: ```scala -val utilCore = "com.twitter" %% "util-core" % "23.11.0" +val utilCore = "com.twitter" %% "util-core" % "24.5.0" ``` # Units diff --git a/build.sbt b/build.sbt index 88544e2df5..8f1b58ee87 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "24.2.0-SNAPSHOT" +val releaseVersion = "24.5.0" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.3" From dfc1c17761945ffcc66f37a35d3bd4c557ee4b47 Mon Sep 17 00:00:00 2001 From: Chetan Tutika Date: Tue, 7 May 2024 19:44:32 +0000 Subject: [PATCH 773/796] Twitter-oss: update OSS libraries post-release to 24.8.0-SNAPSHOT Problem We want to update to the next SNAPSHOT version of our Twitter OSS libraries 24.8.0-SNAPSHOT - util - scrooge - finagle - twitter-server - finatra Solution Prepare libraries for their next SNAPSHOT version. Differential Revision: https://phabricator.twitter.biz/D1141538 --- README.md | 2 +- build.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a7717cfdde..5960d3cbe2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Twitter Util -[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=release)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Arelease) +[![Build Status](https://github.com/twitter/util/workflows/continuous%20integration/badge.svg?branch=develop)](https://github.com/twitter/util/actions?query=workflow%3A%22continuous+integration%22+branch%3Adevelop) [![Project status](https://img.shields.io/badge/status-active-brightgreen.svg)](#status) [![Gitter](https://badges.gitter.im/twitter/finagle.svg)](https://gitter.im/twitter/finagle?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.twitter/util-core_2.12) diff --git a/build.sbt b/build.sbt index 8f1b58ee87..ca8c9f4f31 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ Global / onChangedBuildSource := ReloadOnSourceChanges Global / excludeLintKeys += scalacOptions // might be actually unused in util-doc module but not sure // All Twitter library releases are date versioned as YY.MM.patch -val releaseVersion = "24.5.0" +val releaseVersion = "24.8.0-SNAPSHOT" val slf4jVersion = "1.7.30" val jacksonVersion = "2.14.3" From 8700f19a672a7e09694a1ef6a1089e1341e73897 Mon Sep 17 00:00:00 2001 From: Kerem Celik Date: Thu, 29 Aug 2024 16:36:06 +0000 Subject: [PATCH 774/796] util-jackson: share PrettyPrinter properly The `PrettyPrinter` configured for a Jackson `ObjectWriter` is shared between all downstream `JsonGenerator`s. To make this thread-safe (in order to prevent multiple generators from corrupting the shared indentation level), a class derived from `PrettyPrinter` can also inherit from `Instantiable`, which basically turns the `ObjectWriter`-level `PrettyPrinter` into a factory that produces a new `PrettyPrinter` for each generator. Since `util-jackson`'s `ArrayElementsOnNewLinesPrettyPrinter` inherits from `DefaultPrettyPrinter`, which has state that must be protected from corruption, we need to abide by this factory pattern. Differential Revision: https://phabricator.twitter.biz/D1167308 --- .../main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala b/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala index 3925995598..496122b164 100644 --- a/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala +++ b/util-jackson/src/main/scala/com/twitter/util/jackson/ScalaObjectMapper.scala @@ -412,7 +412,7 @@ object ScalaObjectMapper { private[jackson] object ArrayElementsOnNewLinesPrettyPrinter extends DefaultPrettyPrinter { _arrayIndenter = DefaultIndenter.SYSTEM_LINEFEED_INSTANCE - override def createInstance(): DefaultPrettyPrinter = this + override def createInstance(): DefaultPrettyPrinter = new DefaultPrettyPrinter(this) } /** From b5bc478a38b045c4616f38231a69ce90e8aff044 Mon Sep 17 00:00:00 2001 From: Mikhail Bezoyan Date: Mon, 30 Sep 2024 08:29:16 +0000 Subject: [PATCH 775/796] [util-core] Avoid unnecessary allocations in ByteArray.extract Unapply in pattern matching creates a tuple and unnecessarily boxes integers. Rewriting it in a more imperative style for efficiency. Differential Revision: https://phabricator.twitter.biz/D1173579 --- util-core/src/main/scala/com/twitter/io/Buf.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index f942c6b09d..051de125da 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -778,13 +778,15 @@ object Buf { * * A copy may be performed if necessary. */ - def extract(buf: Buf): Array[Byte] = Buf.ByteArray.coerce(buf) match { - case Buf.ByteArray.Owned(bytes, 0, end) if end == bytes.length => - bytes - case Buf.ByteArray.Shared(bytes) => + def extract(buf: Buf): Array[Byte] = { + val byteArray = Buf.ByteArray.coerce(buf) + if (byteArray.begin == 0 && byteArray.end == byteArray.bytes.length) { + byteArray.bytes + } else { // If the unsafe version included offsets, we need to create a new array // containing only the relevant bytes. - bytes + byteArray.copiedByteArray + } } } From 22eec97266ac822a50c8eb26a680c195bb5dfd16 Mon Sep 17 00:00:00 2001 From: Evgenii Slutckii Date: Wed, 2 Oct 2024 15:51:51 +0000 Subject: [PATCH 776/796] compiling java 8 using java 11 In a preparation for arm builds that require java 11 toolchains (as it is arm unlike java 8), migrating java/scala and code generation to java 11. (arm follow up diff https://phabricator.twitter.biz/D1171895) Main idea of the diff is to propagate -release 8 flag into `javac` and `scalac` that would compile classes into java 8 bytecode level as well as will use only java 8 API and won't leak java 11 APIs. One limitation of it is one cannot open up private packages like `sun.annotation.processor`. This makes targets that are using those private jdk packages uncompilable so I am just using java 8 for compilation of those (should not be a big deal and long term we should just get rid of java 8). Code generation tools are using java 11 as the runtime platform. In this diff: - adding java 11 toolchain with release 8 and propagating it through target_field_transforms.bzl that takes care of jdk transformations - using java8-do-not-use for targets that have private jdk packages. Have to override java8-do-not-use to java8 for scala_library as it has some REPL modules where one cannot override runtime_platform - making scrooge code generation use java 11 at runtime cls is failing because of unrelated diff: https://phabricator.twitter.biz/D1173582 Differential Revision: https://phabricator.twitter.biz/D1173098 --- util-validator/src/main/scala/com/twitter/util/validation/BUILD | 2 +- .../src/main/scala/com/twitter/util/validation/internal/BUILD | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/BUILD index b715b02ab6..917474ec39 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/BUILD @@ -1,7 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - platform = "java8", + platform = "java8-do-not-use", provides = scala_artifact( org = "com.twitter", name = "util-validation-core", diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD index 577f8846a6..63a2a193e6 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD @@ -1,7 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - platform = "java8", + platform = "java8-do-not-use", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal", From 7a99a04dc255e39ddbc4ec6fb210a892585f6a4f Mon Sep 17 00:00:00 2001 From: Mikhail Bezoyan Date: Fri, 4 Oct 2024 15:56:49 +0000 Subject: [PATCH 777/796] [scrooge] Preallocate hashmaps during deserialization Problem When a map is deserialized in scrooge we create it with default capacity, which means that for larger maps we need to resize it multiple times and each time recalculate key hashes Solution Allocate a map with sufficient capacity from the start. Differential Revision: https://phabricator.twitter.biz/D1174445 --- .../main/scala-2.12-/com/twitter/util/MapUtil.scala | 13 +++++++++++++ .../main/scala-2.13+/com/twitter/util/MapUtil.scala | 10 ++++++++++ 2 files changed, 23 insertions(+) create mode 100644 util-core/src/main/scala-2.12-/com/twitter/util/MapUtil.scala create mode 100644 util-core/src/main/scala-2.13+/com/twitter/util/MapUtil.scala diff --git a/util-core/src/main/scala-2.12-/com/twitter/util/MapUtil.scala b/util-core/src/main/scala-2.12-/com/twitter/util/MapUtil.scala new file mode 100644 index 0000000000..6e180c84be --- /dev/null +++ b/util-core/src/main/scala-2.12-/com/twitter/util/MapUtil.scala @@ -0,0 +1,13 @@ +package com.twitter.util + +import scala.collection.mutable + +object MapUtil { + + def newHashMap[K, V](initialCapacity: Int, loadFactor: Double = 0.75): mutable.HashMap[K, V] = { + new mutable.HashMap[K, V]() { + this._loadFactor = (loadFactor * 1000).toInt + override protected val initialSize: Int = (size.toLong / loadFactor).toInt + } + } +} diff --git a/util-core/src/main/scala-2.13+/com/twitter/util/MapUtil.scala b/util-core/src/main/scala-2.13+/com/twitter/util/MapUtil.scala new file mode 100644 index 0000000000..2994ca876c --- /dev/null +++ b/util-core/src/main/scala-2.13+/com/twitter/util/MapUtil.scala @@ -0,0 +1,10 @@ +package com.twitter.util + +import scala.collection.mutable + +object MapUtil { + + def newHashMap[K, V](initialCapacity: Int, loadFactor: Double = 0.75): mutable.HashMap[K, V] = { + new mutable.HashMap[K, V](initialCapacity, loadFactor) + } +} From cd6b189503505a5310ce87eb442d2a45f32bedb7 Mon Sep 17 00:00:00 2001 From: Mikhail Bezoyan Date: Wed, 9 Oct 2024 09:59:25 +0000 Subject: [PATCH 778/796] [util-core] Allocate a hash table of sufficient size in MapUtil.newHashMap Differential Revision: https://phabricator.twitter.biz/D1175381 --- .../main/scala-2.12-/com/twitter/util/MapUtil.scala | 11 +++++++++-- .../main/scala-2.13+/com/twitter/util/MapUtil.scala | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/util-core/src/main/scala-2.12-/com/twitter/util/MapUtil.scala b/util-core/src/main/scala-2.12-/com/twitter/util/MapUtil.scala index 6e180c84be..2c9bc8fd9d 100644 --- a/util-core/src/main/scala-2.12-/com/twitter/util/MapUtil.scala +++ b/util-core/src/main/scala-2.12-/com/twitter/util/MapUtil.scala @@ -1,13 +1,20 @@ package com.twitter.util +import java.lang.Integer.numberOfLeadingZeros import scala.collection.mutable object MapUtil { - def newHashMap[K, V](initialCapacity: Int, loadFactor: Double = 0.75): mutable.HashMap[K, V] = { + def newHashMap[K, V](expectedSize: Int, loadFactor: Double = 0.75): mutable.HashMap[K, V] = { new mutable.HashMap[K, V]() { this._loadFactor = (loadFactor * 1000).toInt - override protected val initialSize: Int = (size.toLong / loadFactor).toInt + override protected def initialSize: Int = { + roundUpToNextPowerOfTwo((expectedSize / loadFactor).toInt max 4) + } } } + + private[this] def roundUpToNextPowerOfTwo(target: Int): Int = { + 1 << -numberOfLeadingZeros(target - 1) + } } diff --git a/util-core/src/main/scala-2.13+/com/twitter/util/MapUtil.scala b/util-core/src/main/scala-2.13+/com/twitter/util/MapUtil.scala index 2994ca876c..af84a676ab 100644 --- a/util-core/src/main/scala-2.13+/com/twitter/util/MapUtil.scala +++ b/util-core/src/main/scala-2.13+/com/twitter/util/MapUtil.scala @@ -4,7 +4,7 @@ import scala.collection.mutable object MapUtil { - def newHashMap[K, V](initialCapacity: Int, loadFactor: Double = 0.75): mutable.HashMap[K, V] = { - new mutable.HashMap[K, V](initialCapacity, loadFactor) + def newHashMap[K, V](expectedSize: Int, loadFactor: Double = 0.75): mutable.HashMap[K, V] = { + new mutable.HashMap[K, V]((expectedSize / loadFactor).toInt, loadFactor) } } From c1570b0add1801102cb2e874972dc20aca405812 Mon Sep 17 00:00:00 2001 From: Mikhail Bezoyan Date: Thu, 10 Oct 2024 16:31:20 +0000 Subject: [PATCH 779/796] util-hashing: Faster ConsistentHashingDistributor Problem ConsistentHashingDistributor uses TreeMap to find a node for a key. TreeMap is relatively slow. Solution Replace the TreeMap with two arrays, one sorted for keys and one matching the first for values. Use binarySearch to find the matching key. Because keys stored in array are not scattered in memory and are primitive values binary search is much faster. Based on ad-hoc benchmarking the gain is at least 50% Differential Revision: https://phabricator.twitter.biz/D1175980 --- .../ConsistentHashingDistributor.scala | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala b/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala index a7ab923bdf..5fdb7503b2 100644 --- a/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala +++ b/util-hashing/src/main/scala/com/twitter/hashing/ConsistentHashingDistributor.scala @@ -1,6 +1,7 @@ package com.twitter.hashing import java.security.MessageDigest +import java.util case class HashNode[A](identifier: String, weight: Int, handle: A) @@ -53,7 +54,7 @@ class ConsistentHashingDistributor[A]( md.update(('0' + j).toByte) } - private[this] val continuum: java.util.TreeMap[Long, HashNode[A]] = { + private[this] val (continuumKeys: Array[Long], continuumValues: Array[HashNode[A]]) = { val md5 = MessageDigest.getInstance("MD5") val underlying = new java.util.TreeMap[Long, HashNode[A]]() val dash: Byte = '-'.toByte @@ -98,7 +99,19 @@ class ConsistentHashingDistributor[A]( assert(underlying.size >= numReps * (nodeCount - 1)) } - underlying + val keys = new Array[Long](underlying.size()) + val values = new Array[HashNode[A]](underlying.size()) + + var i = 0; + val underlyingIterator = underlying.entrySet().iterator() + while (underlyingIterator.hasNext) { + val entry = underlyingIterator.next() + keys(i) = entry.getKey + values(i) = entry.getValue + i += 1 + } + + (keys, values) } def nodes: Seq[A] = hashNodes.map(_.handle) @@ -108,20 +121,29 @@ class ConsistentHashingDistributor[A]( // we need to maintain compatibility with libmemcached private[this] def truncateHash(hash: Long): Long = hash & 0xffffffffL - private def mapEntryForHash(hash: Long): java.util.Map.Entry[Long, HashNode[A]] = { + private def mapEntryForHash(hash: Long): Int = { val truncatedHash = truncateHash(hash) - val entry = continuum.ceilingEntry(truncatedHash) - if (entry == null) continuum.firstEntry else entry + val index = util.Arrays.binarySearch(continuumKeys, truncatedHash) + if (index >= 0) { + index + } else { + val insertionPoint = -(index + 1) + if (insertionPoint < continuumValues.length) { + insertionPoint + } else { + 0 + } + } } def partitionIdForHash(hash: Long): Long = - mapEntryForHash(hash).getKey + continuumKeys(mapEntryForHash(hash)) def entryForHash(hash: Long): (Long, A) = { - val entry = mapEntryForHash(hash) - (entry.getKey, entry.getValue.handle) + val index = mapEntryForHash(hash) + (continuumKeys(index), continuumValues(index).handle) } def nodeForHash(hash: Long): A = - mapEntryForHash(hash).getValue.handle + continuumValues(mapEntryForHash(hash)).handle } From 17c8187d948846756fabcdbb2ff6838b3665968b Mon Sep 17 00:00:00 2001 From: Jillian Crossley Date: Fri, 11 Oct 2024 11:43:31 +0000 Subject: [PATCH 780/796] util/util-jvm: Add gc pause stats for all collectors Problem Currently we only have gc pause stats for the ParNew collector, but we'd like to be able to see this information for the G1 pools. Solution Add collection duration stats for all collector pools. Keep the jvm/gc/eden/pause_msec stat for compatibility, but this will be the same as jvm/gc/ParNew/pause_msec. Differential Revision: https://phabricator.twitter.biz/D1176049 --- CHANGELOG.rst | 6 +++++ .../main/scala/com/twitter/jvm/JvmStats.scala | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4526dfba24..b597c6fcc5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,12 @@ Note that ``PHAB_ID=#`` and ``RB_ID=#`` correspond to associated messages in com Unreleased ---------- +New Features +~~~~~~~~~~~~ + +* util-jvm: Add gc pause stats for all collector pools, including G1. ``PHAB_ID=D1176049`` + + 24.5.0 ------ diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index a62bf1982a..7428f9f08a 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -1,5 +1,6 @@ package com.twitter.jvm +import com.sun.management.GarbageCollectionNotificationInfo import com.twitter.conversions.StringOps._ import com.twitter.finagle.stats.MetricBuilder.GaugeType import com.twitter.finagle.stats.Bytes @@ -9,6 +10,10 @@ import com.twitter.finagle.stats.exp.Expression import com.twitter.finagle.stats.exp.ExpressionSchema import java.lang.management.BufferPoolMXBean import java.lang.management.ManagementFactory +import javax.management.Notification +import javax.management.NotificationEmitter +import javax.management.NotificationListener +import javax.management.openmbean.CompositeData import scala.collection.mutable import scala.jdk.CollectionConverters._ @@ -186,6 +191,26 @@ object JvmStats { gc.getCollectionTime.toFloat } + val gcPauseStat = gcStats.stat(name, "pause_msec") + gc.asInstanceOf[NotificationEmitter].addNotificationListener( + new NotificationListener { + override def handleNotification( + notification: Notification, + handback: Any + ): Unit = { + notification.getType + if (notification.getType == GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION) { + gcPauseStat.add( + GarbageCollectionNotificationInfo + .from(notification.getUserData + .asInstanceOf[CompositeData]).getGcInfo.getDuration) + } + } + }, + null, + null + ) + gcStats.registerExpression( ExpressionSchema(s"gc_cycles", Expression(poolCycles.metadata)) .withLabel(ExpressionSchema.Role, "jvm") From 1d6ff9b18424449036f3a9311af28f32f52b7e6f Mon Sep 17 00:00:00 2001 From: Jillian Crossley Date: Mon, 11 Nov 2024 17:40:11 +0000 Subject: [PATCH 781/796] finagle-core: Ability to configure StatsFilter with a HistogramCounterFactory to track request burstiness Problem While we have a counter for the number of requests, we're limited by the frequency of metrics collection to see how these requests are spread out; that is, how "bursty" they are. Solution Introduce a HistogramCounter, created via a HistogramCounterFactory, that can be used to track request burstiness. The factory is configured on a client/server, and, if configured, is used in StatsFilter to track request burstiness. Differential Revision: https://phabricator.twitter.biz/D1180751 --- .../finagle/stats/HistogramCounter.scala | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 util-stats/src/main/scala/com/twitter/finagle/stats/HistogramCounter.scala diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramCounter.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramCounter.scala new file mode 100644 index 0000000000..719f3484ee --- /dev/null +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/HistogramCounter.scala @@ -0,0 +1,106 @@ +package com.twitter.finagle.stats + +import com.twitter.conversions.DurationOps._ +import com.twitter.util.Closable +import com.twitter.util.Duration +import com.twitter.util.Future +import com.twitter.util.Time +import com.twitter.util.Timer +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.LongAdder +import scala.collection.JavaConverters._ + +private[twitter] sealed abstract class StatsFrequency(val frequency: Duration) { + def suffix: String +} + +private[twitter] object StatsFrequency { + case object HundredMilliSecondly extends StatsFrequency(100.millis) { + override def suffix = "hundredMilliSecondly" + } +} + +/** + * Class for creating [[HistogramCounter]]s. It is expected that there be one [[HistogramCounterFactory]] + * per process -- otherwise we will schedule multiple timer tasks for aggregating the counter into + * a stat, and there can be multiple aggregations for a single stat which may produce unexpected + * results. + */ +private[twitter] class HistogramCounterFactory(timer: Timer, nowMs: () => Long) extends Closable { + + @volatile private[this] var closed = false + + private[this] val frequencyToStats: Map[ + StatsFrequency, + ConcurrentHashMap[Stat, HistogramCounter] + ] = Map( + StatsFrequency.HundredMilliSecondly -> new ConcurrentHashMap[Stat, HistogramCounter] + ) + + frequencyToStats.map { + case (statsFrequency, statToCounter) => + timer.doLater(statsFrequency.frequency)(recordStatsForCounters(statsFrequency, statToCounter)) + } + + def apply( + name: Seq[String], + frequency: StatsFrequency, + statsReceiver: StatsReceiver + ): HistogramCounter = { + val stat = statsReceiver.stat(normalizeName(name) :+ frequency.suffix: _*) + val histogramCounter = new HistogramCounter(stat, nowMs, frequency.frequency.inMillis) + val existing = frequencyToStats(frequency).putIfAbsent(stat, histogramCounter) + if (existing == null) { + histogramCounter + } else { + existing + } + } + + override def close(deadline: Time): Future[Unit] = { + closed = true + Future.Done + } + + private[this] def recordStatsForCounters( + statsFrequency: StatsFrequency, + statToCounter: ConcurrentHashMap[Stat, HistogramCounter] + ): Unit = { + statToCounter.values().asScala.foreach { counter => + counter.recordAndReset() + } + if (!closed) { + timer.doLater(statsFrequency.frequency)(recordStatsForCounters(statsFrequency, statToCounter)) + } + } + + private[this] def normalizeName(name: Seq[String]): Seq[String] = { + if (name.forall(!_.contains("/"))) { + name + } else { + name.map(_.split("/")).flatten + } + } +} + +private[stats] class HistogramCounter(stat: Stat, nowMs: () => Long, windowSizeMs: Long) { + private[this] val counter: LongAdder = new LongAdder + @volatile private[this] var lastRecordAndResetMs = nowMs() + + private[stats] def recordAndReset(): Unit = { + val count = counter.sumThenReset() + val now = nowMs() + val elapsed = Math.max(0, now - lastRecordAndResetMs) + val elapsedWindows = elapsed.toFloat / windowSizeMs + stat.add(count / elapsedWindows) + lastRecordAndResetMs = now + } + + def incr(delta: Long): Unit = { + counter.add(delta) + } + + def incr(): Unit = { + counter.increment() + } +} From 7d74b9ce7dbaee1962eef81a9a6d562140e0a6b6 Mon Sep 17 00:00:00 2001 From: Jillian Crossley Date: Tue, 12 Nov 2024 16:05:53 +0000 Subject: [PATCH 782/796] util/util-jvm: Add support for g1 allocation stats Problem Currently we only support ParNew collector for eden allocation stats. Solution Support G1 as well. Differential Revision: https://phabricator.twitter.biz/D1182243 --- .../scala/com/twitter/jvm/Allocations.scala | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala b/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala index fc6597ff73..556eb1390b 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala @@ -1,17 +1,18 @@ package com.twitter.jvm import com.twitter.finagle.stats.StatsReceiver -import java.lang.management.{MemoryPoolMXBean, MemoryUsage, ManagementFactory} +import java.lang.management.MemoryPoolMXBean +import java.lang.management.MemoryUsage +import java.lang.management.ManagementFactory import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.atomic.AtomicLong import java.util.{List => juList} -import javax.management.openmbean.{CompositeData, TabularData} -import javax.management.{ - ListenerNotFoundException, - Notification, - NotificationListener, - NotificationEmitter -} +import javax.management.openmbean.CompositeData +import javax.management.openmbean.TabularData +import javax.management.ListenerNotFoundException +import javax.management.Notification +import javax.management.NotificationListener +import javax.management.NotificationEmitter import scala.collection.mutable import scala.jdk.CollectionConverters._ @@ -30,8 +31,7 @@ private[jvm] class Allocations(statsReceiver: StatsReceiver) { private[this] val edenPool: Option[MemoryPoolMXBean] = ManagementFactory.getMemoryPoolMXBeans.asScala.find { bean => - // todo: see if we can support the g1 collector - bean.getName == "Par Eden Space" || bean.getName == "PS Eden Space" + bean.getName == "Par Eden Space" || bean.getName == "PS Eden Space" || bean.getName == "G1 Eden Space" } private[this] val edenSizeAfterLastGc = new AtomicLong() From 81bab63ae10cec593915754d59666f90202dfa0a Mon Sep 17 00:00:00 2001 From: Evgenii Slutckii Date: Fri, 13 Dec 2024 19:59:10 +0000 Subject: [PATCH 783/796] strato concurrent tests 1. making strato related tests concurrent as they are timing out on CI at this point 2. adding syncronized to Hotspot snapshot method as it is not thread safe. in particular when we lookup GC name we call PerfStringCounter that calls super PerfByteArrayCounter that resets buffer position to 0. if called concurrently one thread will read the counter resetting position to buffer length and we get BufferUnderflow i.e. race condition https://github.com/AdoptOpenJDK/openjdk-jdk11u/blob/master/src/java.management/share/classes/sun/management/counter/perf/PerfByteArrayCounter.java#L53 https://github.com/AdoptOpenJDK/openjdk-jdk11u/blob/master/src/java.management/share/classes/sun/management/counter/perf/PerfStringCounter.java#L64 Stack trace: https://ton.pdxa.twitter.com/scoot/prod_d70f787a-276c-4db8-7e76-36016e1ef442_08fbec44-75f9-482d-7f87-b240c9af3c1e/stdlog ``` generated columnDescs match generated ops(com.twitter.strato.config.compile_tests.ColumnDescTest) java.nio.BufferUnderflowException at java.base/java.nio.DirectByteBuffer.get(DirectByteBuffer.java:294) at java.base/java.nio.ByteBuffer.get(ByteBuffer.java:770) at java.management/sun.management.counter.perf.PerfByteArrayCounter.byteArrayValue(PerfByteArrayCounter.java:57) at java.management/sun.management.counter.perf.PerfStringCounter.stringValue(PerfStringCounter.java:64) at java.management/sun.management.counter.perf.PerfStringCounter.getValue(PerfStringCounter.java:58) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at com.twitter.jvm.Hotspot.$anonfun$getGc$5(Hotspot.scala:90) at scala.Option.map(Option.scala:230) at com.twitter.jvm.Hotspot.$anonfun$getGc$4(Hotspot.scala:90) at com.twitter.jvm.Hotspot.$anonfun$getGc$4$adapted(Hotspot.scala:89) at scala.Option.flatMap(Option.scala:271) at com.twitter.jvm.Hotspot.$anonfun$getGc$2(Hotspot.scala:89) at com.twitter.jvm.Hotspot.$anonfun$getGc$2$adapted(Hotspot.scala:88) at scala.Option.flatMap(Option.scala:271) at com.twitter.jvm.Hotspot.getGc(Hotspot.scala:88) at com.twitter.jvm.Hotspot.snap(Hotspot.scala:149) at com.twitter.strato.config.loading.ConfigResourceMonitoring$.collect(ConfigResourceMonitoring.scala:55) at com.twitter.strato.config.loading.CompiledStratoCatalog.(CompiledStratoCatalog.scala:56) at com.twitter.strato.config.loading.CompiledStratoCatalog$.apply(CompiledStratoCatalog.scala:493) at com.twitter.strato.test.config.ConfigTestCatalog$$anon$1.mkConfigCatalog(ConfigTestCatalog.scala:333) at com.twitter.strato.test.config.ConfigTestCatalog$.apply(ConfigTestCatalog.scala:159) at com.twitter.strato.test.config.StratoConfigSuite.mkCatalog(StratoConfigSuite.scala:164) at com.twitter.strato.config.compile_tests.ColumnDescTest.$anonfun$new$1(ColumnDescTest.scala:18) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) at com.twitter.servo.testing.prop.PropertySuite.$anonfun$test$1(PropertySuite.scala:71) ``` Differential Revision: https://phabricator.twitter.biz/D1188541 --- .../main/scala/com/twitter/jvm/Hotspot.scala | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala index a46cbfbfce..9b9c50e4df 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Hotspot.scala @@ -4,15 +4,15 @@ import com.twitter.conversions.StorageUnitOps._ import com.twitter.conversions.DurationOps._ import com.twitter.util.Time import java.lang.management.ManagementFactory -import java.util.logging.{Level, Logger} -import javax.management.openmbean.{CompositeData, CompositeDataSupport} -import javax.management.{ - Notification, - NotificationEmitter, - NotificationListener, - ObjectName, - RuntimeMBeanException -} +import java.util.logging.Level +import java.util.logging.Logger +import javax.management.openmbean.CompositeData +import javax.management.openmbean.CompositeDataSupport +import javax.management.Notification +import javax.management.NotificationEmitter +import javax.management.NotificationListener +import javax.management.ObjectName +import javax.management.RuntimeMBeanException import javax.naming.OperationNotSupportedException import scala.jdk.CollectionConverters._ import scala.language.reflectiveCalls @@ -115,7 +115,7 @@ class Hotspot extends Jvm { } } - def snap: Snapshot = { + def snap: Snapshot = synchronized { val cs = counters("") val heap = for { invocations <- cs.get("sun.gc.collector.0.invocations").map(long) From ef724965483d4e62957a3181065317552a1a8c41 Mon Sep 17 00:00:00 2001 From: Kerem Celik Date: Fri, 31 Jan 2025 18:59:27 +0000 Subject: [PATCH 784/796] util/util-core: Increase visibility of Buf.Composite When Bufs are built by concatenation they are usually represented as a `Buf.Composite`. Since this class is private, it is difficult to consume the underlying discontiguous Bufs without copying. Making the class public allows constructions like: ``` def consume(buf: Buf)(eat: Array[Byte] => Unit): Unit = buf match { case array: Buf.ByteArray => eat(array.unsafeRawByteArray) // [unsafeArrayOffset + length] case Buf.ByteBuffer(bb) if bb.hasArray => eat(bb.array()) // [+ bb.arrayOffset()] case Buf.Composite(bufs, _) => bufs.foreach(consume(_)(eat)) case _ => eat(Buf.ByteArray.Owned.extract(buf)) } ``` Differential Revision: https://phabricator.twitter.biz/D1198118 --- util-core/src/main/scala/com/twitter/io/Buf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-core/src/main/scala/com/twitter/io/Buf.scala b/util-core/src/main/scala/com/twitter/io/Buf.scala index 051de125da..6d59d3cb25 100644 --- a/util-core/src/main/scala/com/twitter/io/Buf.scala +++ b/util-core/src/main/scala/com/twitter/io/Buf.scala @@ -404,7 +404,7 @@ object Buf { def length: Int = 3 } - private final case class Composite(bufs: IndexedSeq[Buf], length: Int) extends Buf { + final case class Composite(bufs: IndexedSeq[Buf], length: Int) extends Buf { // the factory method requires non-empty Bufs override def isEmpty: Boolean = false From 6d4f4cb5c90bab718a51a7660f4b9f05471ae591 Mon Sep 17 00:00:00 2001 From: Kerem Celik Date: Wed, 19 Mar 2025 18:08:13 +0000 Subject: [PATCH 785/796] util-core: avoid unnecessary unboxing in activation map The first call to `LocalScheduler::get` registers the activation in a `Map[Activation, Boolean]` by writing a value of `java.lang.Boolean.TRUE`; this requires a boxing call which has negligible performance impact but can pollute profiles with many async dispatches. Differential Revision: https://phabricator.twitter.biz/D1209771 --- .../src/main/scala/com/twitter/concurrent/Scheduler.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala index 7f3d897f7e..0927ff875f 100644 --- a/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala +++ b/util-core/concurrent-extra/src/main/scala/com/twitter/concurrent/Scheduler.scala @@ -4,7 +4,8 @@ import com.twitter.util.Awaitable.CanAwait import java.util.ArrayDeque import java.util.concurrent._ import java.util.concurrent.atomic.AtomicInteger -import java.util.logging.{Level, Logger} +import java.util.logging.Level +import java.util.logging.Logger import scala.collection.mutable /** @@ -202,7 +203,7 @@ class LocalScheduler(lifo: Boolean) extends Scheduler { // use weak refs to prevent Activations from causing a memory leak // thread-safety provided by synchronizing on `activations` - private[this] val activations = new mutable.WeakHashMap[Activation, Boolean]() + private[this] val activations = new mutable.WeakHashMap[Activation, java.lang.Boolean]() private[this] val local = new ThreadLocal[Activation]() From 9b5b007df3b03cd6a24617b88d6735fb12bef856 Mon Sep 17 00:00:00 2001 From: Kerem Celik Date: Fri, 18 Apr 2025 19:43:32 +0000 Subject: [PATCH 786/796] finagle: more comprehensive support for dimensional metrics ==== Problem Finagle doesn't have comprehensive support for dimensional metrics === Solution 1. Promote the visibility of `hierarchicalScope`, `dimensionalScope`, and `label` in `StatsReceiver` to public. This allows consumers to define their metric schemas for dual-consumption by the flat JSON & Prometheus exporters 2. Add the flag `com.twitter.finagle.stats.fullTranslation`. This allows service owners to express that raw `.scope` calls on `StatsReceiver` instances to propagate both hierarchical & dimensional metrics 3. Add the flag `com.twitter.finagle.stats.biasToFull`. This allows service owners to express that metrics with an indeterminate identity (have not been explicitly set as dimensional) to be exported through Prometheus anyways. 4. Remove the redundant dimensional scopes set on default Finagle client/service metrics 5. Remove the redundant `app` dimensional scope set on metrics exporter through `DefaultStatsReceiver` 6. Adjust, backwards-compatibly, the JVM metrics to utilize the new APIs and export the memory manager, pool, etc as a label instead of a hierarchical scope 7. Adjust, backwards-compatibly, some other internal use-cases to use labels instead of hierarchical scope Differential Revision: https://phabricator.twitter.biz/D1218090 --- CHANGELOG.rst | 4 +++ .../scala/com/twitter/jvm/Allocations.scala | 3 +- .../main/scala/com/twitter/jvm/JvmStats.scala | 28 +++++++++++-------- .../finagle/stats/LoadedStatsReceiver.scala | 5 +--- .../twitter/finagle/stats/MetricBuilder.scala | 9 +++++- .../twitter/finagle/stats/StatsReceiver.scala | 6 ++-- .../stats/DefaultStatsReceiverTest.scala | 4 +-- 7 files changed, 36 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b597c6fcc5..38368363c2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,8 @@ New Features ~~~~~~~~~~~~ * util-jvm: Add gc pause stats for all collector pools, including G1. ``PHAB_ID=D1176049`` +* util-stats: Expose dimensional metrics APIs and allow metrics with an indeterminate + identity to be exported through the Prometheus exporter. ``PHAB_ID=D1218090`` 24.5.0 @@ -21,6 +23,8 @@ Runtime Behavior Changes * util-app: When the application exits due to an error on startup, the error and and stack trace are printed to stderr in addition to the existing stdout. ``PHAB_ID=D1116753`` +* util-jvm: Memory manager & pool metrics are labelled dimensionally rather than hierarchically + when exported as dimensional metrics. ``PHAB_ID=D1218090`` 23.11.0 diff --git a/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala b/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala index 556eb1390b..996a65baae 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala @@ -41,7 +41,8 @@ private[jvm] class Allocations(statsReceiver: StatsReceiver) { private[this] val beanAndListeners = new LinkedBlockingQueue[(NotificationEmitter, NotificationListener)]() - private[this] val edenGcPauses = statsReceiver.scope("eden").stat("pause_msec") + private[this] val edenGcPauses = statsReceiver + .hierarchicalScope("eden").label("jmm", "eden").stat("pause_msec") private[jvm] def start(): Unit = { edenPool diff --git a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala index 7428f9f08a..b167903e82 100644 --- a/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala +++ b/util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala @@ -113,15 +113,17 @@ object JvmStats { val postGCStats = memStats.scope("postGC") memPool.foreach { pool => val name = pool.getName.regexSub("""[^\w]""".r) { m => "_" } + val currentPoolScope = currentMem.hierarchicalScope(name).label("pool", name) + val postGcPoolScope = postGCStats.hierarchicalScope(name).label("pool", name) if (pool.getCollectionUsage != null) { def usage = pool.getCollectionUsage // this is a snapshot, we can't reuse the value - gauges.add(postGCStats.addGauge(name, "used") { usage.getUsed.toFloat }) + gauges.add(postGcPoolScope.addGauge("used") { usage.getUsed.toFloat }) } if (pool.getUsage != null) { def usage = pool.getUsage // this is a snapshot, we can't reuse the value - val usageGauge = currentMem.addGauge(name, "used") { usage.getUsed.toFloat } + val usageGauge = currentPoolScope.addGauge("used") { usage.getUsed.toFloat } gauges.add(usageGauge) - gauges.add(currentMem.addGauge(name, "max") { usage.getMax.toFloat }) + gauges.add(currentPoolScope.addGauge("max") { usage.getMax.toFloat }) // register memory usage expression currentMem.registerExpression( @@ -171,9 +173,10 @@ object JvmStats { val bufferPoolStats = memStats.scope("buffer") jBufferPool.asScala.foreach { bp => val name = bp.getName - gauges.add(bufferPoolStats.addGauge(name, "count") { bp.getCount.toFloat }) - gauges.add(bufferPoolStats.addGauge(name, "used") { bp.getMemoryUsed.toFloat }) - gauges.add(bufferPoolStats.addGauge(name, "max") { bp.getTotalCapacity.toFloat }) + val bufferPoolScope = bufferPoolStats.hierarchicalScope(name).label("pool", name) + gauges.add(bufferPoolScope.addGauge("count") { bp.getCount.toFloat }) + gauges.add(bufferPoolScope.addGauge("used") { bp.getMemoryUsed.toFloat }) + gauges.add(bufferPoolScope.addGauge("max") { bp.getTotalCapacity.toFloat }) } } @@ -181,17 +184,18 @@ object JvmStats { val gcStats = stats.scope("gc") gcPool.foreach { gc => val name = gc.getName.regexSub("""[^\w]""".r) { m => "_" } + val poolScope = gcStats.hierarchicalScope(name).label("jmm", name) val poolCycles = - gcStats.addGauge( - gcStats.metricBuilder(GaugeType).withCounterishGauge.withName(name, "cycles")) { + poolScope.addGauge( + poolScope.metricBuilder(GaugeType).withCounterishGauge.withName("cycles")) { gc.getCollectionCount.toFloat } - val poolMsec = gcStats.addGauge( - gcStats.metricBuilder(GaugeType).withCounterishGauge.withName(name, "msec")) { + val poolMsec = poolScope.addGauge( + poolScope.metricBuilder(GaugeType).withCounterishGauge.withName("msec")) { gc.getCollectionTime.toFloat } - val gcPauseStat = gcStats.stat(name, "pause_msec") + val gcPauseStat = poolScope.stat("pause_msec") gc.asInstanceOf[NotificationEmitter].addNotificationListener( new NotificationListener { override def handleNotification( @@ -244,7 +248,7 @@ object JvmStats { allocations.start() if (allocations.trackingEden) { val allocationStats = memStats.scope("allocations") - val eden = allocationStats.scope("eden") + val eden = allocationStats.hierarchicalScope("eden").label("space", "eden") gauges.add(eden.addGauge("bytes") { allocations.eden.toFloat }) } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala index 34b8a71d8a..a3098a5cd7 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala @@ -23,10 +23,7 @@ object LoadedStatsReceiver extends StatsReceiverProxy { * [[com.twitter.finagle.util.LoadService]] mechanism. */ object DefaultStatsReceiver extends StatsReceiverProxy { - val self: StatsReceiver = - LoadedStatsReceiver - .dimensionalScope("app") - .label("implementation", "app") + val self: StatsReceiver = LoadedStatsReceiver.label("implementation", "app") override def repr: DefaultStatsReceiver.type = this diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala index de95e7b5d1..47a7ddf8b3 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala @@ -1,5 +1,6 @@ package com.twitter.finagle.stats +import com.twitter.app.GlobalFlag import com.twitter.finagle.stats.MetricBuilder.CounterType import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType import com.twitter.finagle.stats.MetricBuilder.GaugeType @@ -10,6 +11,12 @@ import com.twitter.finagle.stats.MetricBuilder.MetricType import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter import scala.annotation.varargs +object biasToFull + extends GlobalFlag[Boolean]( + false, + "Bias metrics with an indeterminate identity to be exported through all available exporters" + ) + /** * Represents the "role" this service plays with respect to this metric. * @@ -225,7 +232,7 @@ object MetricBuilder { * The current behavior is to default to [[IdentityType.HierarchicalOnly]] */ private[twitter] def toResolvedIdentityType(identityType: IdentityType): ResolvedIdentityType = - identityType.bias(IdentityType.HierarchicalOnly) + identityType.bias(if (biasToFull()) IdentityType.Full else IdentityType.HierarchicalOnly) /** An [[IdentityType]] that cannot be [[NonDeterminate]] */ sealed abstract class ResolvedIdentityType extends IdentityType diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala index 01131f0bed..e984e7bec5 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala @@ -401,7 +401,7 @@ trait StatsReceiver { * Create a new `StatsReceiver` that will add a scope that is only used when the metric is * emitted in hierarchical form. */ - private[finagle] final def hierarchicalScope(namespace: String): StatsReceiver = { + def hierarchicalScope(namespace: String): StatsReceiver = { if (namespace == "") this else new ScopeTranslatingStatsReceiver( @@ -414,7 +414,7 @@ trait StatsReceiver { * Create a new `StatsReceiver` that will add a scope that is only used when the metric is * emitted in dimensional form. */ - private[finagle] final def dimensionalScope(namespace: String): StatsReceiver = { + def dimensionalScope(namespace: String): StatsReceiver = { if (namespace == "") this else new ScopeTranslatingStatsReceiver( @@ -426,7 +426,7 @@ trait StatsReceiver { /** * Create a new `StatsReceiver` that will add the specified label to all created metrics. */ - private[finagle] final def label(labelName: String, labelValue: String): StatsReceiver = { + def label(labelName: String, labelValue: String): StatsReceiver = { require(labelName.nonEmpty) if (labelValue == "") this else new LabelTranslatingStatsReceiver(this, labelName, labelValue) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/DefaultStatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/DefaultStatsReceiverTest.scala index 0fdeecf1fd..ca25a83fe2 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/DefaultStatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/DefaultStatsReceiverTest.scala @@ -3,12 +3,12 @@ package com.twitter.finagle.stats import org.scalatest.funsuite.AnyFunSuite class DefaultStatsReceiverTest extends AnyFunSuite { - test("DefaultStatsReceiver has app in dimensional scope and label") { + test("DefaultStatsReceiver initializes dimensional scope and label") { LoadedStatsReceiver.self = new InMemoryStatsReceiver val fooCounter = DefaultStatsReceiver.counter("foo") val identity = fooCounter.metadata.toMetricBuilder.get.identity - assert(identity.dimensionalName == Seq("app", "foo")) + assert(identity.dimensionalName == Seq("foo")) assert(identity.hierarchicalName == Seq("foo")) assert(identity.labels == Map("implementation" -> "app")) } From 9602a290cdea9717a7ab837e417233cb2ab3dbcb Mon Sep 17 00:00:00 2001 From: Kerem Celik Date: Mon, 21 Apr 2025 02:38:21 +0000 Subject: [PATCH 787/796] finagle: more integration with dimensional metrics === Problem 1. Finagle & Finatra exceptions are reported only as hierarchical metrics. 2. Finatra route metrics are reported only as hierarchical metrics. 3. Certain Thrift client metrics are reported only as hierarchical metrics. === Solution 1. Continues to add first-class dimensional metrics support to Finagle by translating some hierarchically-scoped exception stats to labelled metrics. 2. Add dimensional equivalents to Finatra per-route metrics. 3. Report `ServicePerEndpoint` Thrift metrics dimensionally. Exceptions here are still hierarchical, for now. Differential Revision: https://phabricator.twitter.biz/D1218341 --- .../scala/com/twitter/util/Throwables.scala | 11 ++++++ .../finagle/stats/ExceptionStatsHandler.scala | 30 +++++++++++++-- .../finagle/stats/RollupStatsReceiver.scala | 38 +++++++++++-------- ...ategorizingExceptionStatsHandlerTest.scala | 16 ++++++++ .../finagle/stats/StatsReceiverTest.scala | 32 +++++++++++++++- 5 files changed, 108 insertions(+), 19 deletions(-) diff --git a/util-core/src/main/scala/com/twitter/util/Throwables.scala b/util-core/src/main/scala/com/twitter/util/Throwables.scala index 7e33824745..9057a172a5 100644 --- a/util-core/src/main/scala/com/twitter/util/Throwables.scala +++ b/util-core/src/main/scala/com/twitter/util/Throwables.scala @@ -1,6 +1,7 @@ package com.twitter.util import scala.annotation.tailrec +import scala.collection.mutable object Throwables { @@ -38,5 +39,15 @@ object Throwables { object RootCause { def unapply(e: Throwable): Option[Throwable] = Option(e.getCause) + + def nested(ex: Throwable): Throwable = { + var rootException = ex + val exceptions = mutable.HashSet[Throwable]() + while (rootException.getCause != null && !exceptions.contains(rootException)) { + exceptions.add(rootException) + rootException = rootException.getCause + } + rootException + } } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala index 46da54d10a..bd2d2b7033 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/ExceptionStatsHandler.scala @@ -111,12 +111,18 @@ class CategorizingExceptionStatsHandler( * failures/restartable/com.twitter.finagle.Failure/com.twitter.finagle.CancelledConnectionException: 1 * failures/restartable/com.twitter.finagle.Failure/com.twitter.finagle.CancelledConnectionException/java.net.ConnectException: 1 * + * `f1` is also recorded dimensionally as: + * failures{interrupted=true, restartable=true, exception=java.net.ConnectException} + * * `f2` is recorded as: * failures * failures/com.twitter.finagle.IndividualRequestTimeoutException * sourcedfailures/myservice * sourcedfailures/myservice/com.twitter.finagle.IndividualRequestTimeoutException * + * `f2` is also recorded dimensionally as: + * failures{interrupted=true, source=myservice, restartable=true, exception=com.twitter.finagle.IndividualRequestTimeoutException} + * * @param mkLabel label prefix, default to 'failures' * @param mkFlags extracting flags if the Throwable is a Failure * @param mkSource extracting source when configured @@ -139,14 +145,32 @@ private[finagle] class MultiCategorizingExceptionStatsHandler( if (flags.isEmpty) Seq(Seq(parentLabel)) else flags.toSeq.map(Seq(parentLabel, _)) - val labels: Seq[Seq[String]] = mkSource(t) match { + val maybeSource = mkSource(t) + val labels: Seq[Seq[String]] = maybeSource match { case Some(service) => flagLabels :+ Seq(SourcedFailures, service) case None => flagLabels } val paths: Seq[Seq[String]] = statPaths(t, labels, rollup) - if (flags.nonEmpty) statsReceiver.counter(parentLabel).incr() - paths.foreach { path => statsReceiver.counter(path: _*).incr() } + val dimensionalLabels = Map("exception" -> Throwables.RootCause.nested(t).getClass.getName) ++ + flags.map { flag => flag -> "true" } ++ + maybeSource.map("source" -> _).toSeq + + val dimensionalMetric = MetricBuilder.forCounter + .withName(parentLabel) + .withLabels(dimensionalLabels) + .withDimensionalSupport + + statsReceiver.counter(dimensionalMetric).incr() + paths.foreach { + case Seq(comp) if comp == parentLabel => + // The "flat" counter is used to build up the rollup hierarchies but since we also use it + // to hold the dimensional metric, we skip it when recording hierarchical metrics. + case path => + statsReceiver + .counter(MetricBuilder.forCounter.withHierarchicalOnly.withName(path: _*)) + .incr() + } } } diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala index 88c49b11fb..0b16ce0c6e 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/RollupStatsReceiver.scala @@ -9,37 +9,45 @@ package com.twitter.finagle.stats * - "/errors" * - "/errors/clientErrors" * - "/errors/clientErrors/java_net_ConnectException" + * + * @param self the [[StatsReceiver]] to proxy metrics creation to + * @param hierarchicalOnly whether non-root scopes should be created with the HierarchicalOnly metrics identity */ -class RollupStatsReceiver(protected val self: StatsReceiver) extends StatsReceiverProxy { +class RollupStatsReceiver(protected val self: StatsReceiver, hierarchicalOnly: Boolean = false) + extends StatsReceiverProxy { - private[this] def tails[A](s: Seq[A]): Seq[Seq[A]] = { + /** + * @return Seq(metrics namespace -> whether the namespace is the "parent" scope) + */ + private[this] def tails[A](s: Seq[A]): Seq[(Seq[A], Boolean)] = { s match { case s @ Seq(_) => - Seq(s) + Seq(s -> true) case Seq(hd, tl @ _*) => - Seq(Seq(hd)) ++ (tails(tl) map { t => Seq(hd) ++ t }) + Seq(Seq(hd) -> true) ++ (tails(tl) map { case (t, _) => (Seq(hd) ++ t) -> false }) } } - override def counter(metricBuilder: MetricBuilder) = new Counter { - private[this] val allCounters = BroadcastCounter( - tails(metricBuilder.name).map(n => self.counter(metricBuilder.withName(n: _*))) - ) + private[this] def metrics(parent: MetricBuilder): Seq[MetricBuilder] = tails(parent.name).map { + case (name, isRoot) => + val builder = parent.withName(name: _*) + if (isRoot || !hierarchicalOnly) builder else builder.withHierarchicalOnly + } + + override def counter(metricBuilder: MetricBuilder): Counter = new Counter { + private[this] val allCounters = BroadcastCounter(metrics(metricBuilder).map(self.counter)) def incr(delta: Long): Unit = allCounters.incr(delta) def metadata: Metadata = allCounters.metadata } - override def stat(metricBuilder: MetricBuilder) = new Stat { - private[this] val allStats = BroadcastStat( - tails(metricBuilder.name).map(n => self.stat(metricBuilder.withName(n: _*))) - ) + override def stat(metricBuilder: MetricBuilder): Stat = new Stat { + private[this] val allStats = BroadcastStat(metrics(metricBuilder).map(self.stat)) def add(value: Float): Unit = allStats.add(value) def metadata: Metadata = allStats.metadata } - override def addGauge(metricBuilder: MetricBuilder)(f: => Float) = new Gauge { - private[this] val underlying = - tails(metricBuilder.name).map(n => self.addGauge(metricBuilder.withName(n: _*))(f)) + override def addGauge(metricBuilder: MetricBuilder)(f: => Float): Gauge = new Gauge { + private[this] val underlying = metrics(metricBuilder).map { self.addGauge(_)(f) } def remove(): Unit = underlying.foreach(_.remove()) def metadata: Metadata = MultiMetadata(underlying.map(_.metadata)) } diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala index 578ca6f73e..55ec7d313c 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/MultiCategorizingExceptionStatsHandlerTest.scala @@ -20,6 +20,16 @@ class MultiCategorizingExceptionStatsHandlerTest extends AnyFunSuite { val keys = receiver.counters.keys.map(_.mkString("/")).toSeq.sorted + val identity = receiver.schemas(Seq("clienterrors")).toMetricBuilder.get.identity + assert( + identity.labels == Map( + "interrupted" -> "true", + "restartable" -> "true", + "source" -> "service", + "exception" -> "java.lang.Exception" + ) + ) + assert(receiver.counters(Seq("clienterrors")) == 1) assert(receiver.counters(Seq("clienterrors", "interrupted")) == 1) assert(receiver.counters(Seq("clienterrors", "restartable")) == 1) @@ -99,6 +109,9 @@ class MultiCategorizingExceptionStatsHandlerTest extends AnyFunSuite { val keys = receiver.counters.keys.map(_.mkString("/")).toSeq.sorted + val identity = receiver.schemas(Seq("clienterrors")).toMetricBuilder.get.identity + assert(identity.labels == Map("source" -> "service", "exception" -> "java.lang.Exception")) + assert(receiver.counters(Seq("clienterrors")) == 1) assert(receiver.counters(Seq("clienterrors", classOf[RuntimeException].getName)) == 1) assert( @@ -142,6 +155,9 @@ class MultiCategorizingExceptionStatsHandlerTest extends AnyFunSuite { val keys = receiver.counters.keys.map(_.mkString("/")).toSeq.sorted + val identity = receiver.schemas(Seq("failures")).toMetricBuilder.get.identity + assert(identity.labels == Map("exception" -> "java.lang.Exception")) + assert(receiver.counters.view.filterKeys(_.contains("failures")).size == 3) assert(receiver.counters.view.filterKeys(_.contains("sourcedfailures")).size == 0) diff --git a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala index 399178de82..d279075565 100644 --- a/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala +++ b/util-stats/src/test/scala/com/twitter/finagle/stats/StatsReceiverTest.scala @@ -16,7 +16,7 @@ import scala.collection.mutable.ArrayBuffer class StatsReceiverTest extends AnyFunSuite { - test("RollupStatsReceiver counter/stats") { + test("RollupStatsReceiver counter/stats - full translation") { val mem = new InMemoryStatsReceiver val receiver = new RollupStatsReceiver(mem) @@ -30,6 +30,36 @@ class StatsReceiverTest extends AnyFunSuite { assert(mem.counters(Seq("toto", "titi")) == 2) assert(mem.counters(Seq("toto", "titi", "tata")) == 1) assert(mem.counters(Seq("toto", "titi", "tutu")) == 1) + + def identity(path: String*): IdentityType = mem.schemas(path).identity.identityType + + assert(identity("toto") == IdentityType.NonDeterminate) + assert(identity("toto", "titi") == IdentityType.NonDeterminate) + assert(identity("toto", "titi", "tata") == IdentityType.NonDeterminate) + assert(identity("toto", "titi", "tutu") == IdentityType.NonDeterminate) + } + + test("RollupStatsReceiver counter/stats - hierarchical only") { + val mem = new InMemoryStatsReceiver + val receiver = new RollupStatsReceiver(mem, hierarchicalOnly = true) + + receiver.counter("toto", "titi", "tata").incr() + assert(mem.counters(Seq("toto")) == 1) + assert(mem.counters(Seq("toto", "titi")) == 1) + assert(mem.counters(Seq("toto", "titi", "tata")) == 1) + + receiver.counter("toto", "titi", "tutu").incr() + assert(mem.counters(Seq("toto")) == 2) + assert(mem.counters(Seq("toto", "titi")) == 2) + assert(mem.counters(Seq("toto", "titi", "tata")) == 1) + assert(mem.counters(Seq("toto", "titi", "tutu")) == 1) + + def identity(path: String*): IdentityType = mem.schemas(path).identity.identityType + + assert(identity("toto") == IdentityType.NonDeterminate) + assert(identity("toto", "titi") == IdentityType.HierarchicalOnly) + assert(identity("toto", "titi", "tata") == IdentityType.HierarchicalOnly) + assert(identity("toto", "titi", "tutu") == IdentityType.HierarchicalOnly) } test("Broadcast Counter/Stat") { From f92c2829171bbf4f92bf112fe3e12ca910308cc3 Mon Sep 17 00:00:00 2001 From: Kerem Celik Date: Tue, 29 Apr 2025 20:35:42 +0000 Subject: [PATCH 788/796] util-stats: full translation for BroadcastStatsReceiver === Problem `BroadcastStatsReceiver` hides the underlying receivers' translation mode. === Solution Just translate all metrics with `FullTranslation`. Differential Revision: https://phabricator.twitter.biz/D1220816 --- .../com/twitter/finagle/stats/BroadcastStatsReceiver.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala index 2ea477f780..82c4b484e2 100644 --- a/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala +++ b/util-stats/src/main/scala/com/twitter/finagle/stats/BroadcastStatsReceiver.scala @@ -22,6 +22,9 @@ object BroadcastStatsReceiver { with DelegatingStatsReceiver { val repr: AnyRef = this + override def scopeTranslation: NameTranslatingStatsReceiver.Mode = + NameTranslatingStatsReceiver.FullTranslation + def counter(metricBuilder: MetricBuilder) = new BroadcastCounter.Two( first.counter(metricBuilder), second.counter(metricBuilder) @@ -63,6 +66,9 @@ object BroadcastStatsReceiver { private class N(srs: Seq[StatsReceiver]) extends StatsReceiver with DelegatingStatsReceiver { val repr: AnyRef = this + override def scopeTranslation: NameTranslatingStatsReceiver.Mode = + NameTranslatingStatsReceiver.FullTranslation + def counter(metricBuilder: MetricBuilder) = BroadcastCounter(srs.map { _.counter(metricBuilder) }) From 0c9cc53ff8275af19f36ad9a0ab65a472152a789 Mon Sep 17 00:00:00 2001 From: Zhenzhao Wang Date: Wed, 14 May 2025 20:35:06 +0000 Subject: [PATCH 789/796] Add observability team to the owner of util/util-stats. Differential Revision: https://phabricator.twitter.biz/D1224521 --- util-stats/PROJECT | 1 + 1 file changed, 1 insertion(+) diff --git a/util-stats/PROJECT b/util-stats/PROJECT index 009bd1c5fb..fac8decd61 100644 --- a/util-stats/PROJECT +++ b/util-stats/PROJECT @@ -3,3 +3,4 @@ owners: - jillianc - mbezoyan + - observability-team:ldap From 7d158ce6807df04b21ceffa44a81de81bffd1526 Mon Sep 17 00:00:00 2001 From: Benjamin Erichsen Date: Tue, 3 Jun 2025 16:34:46 +0000 Subject: [PATCH 790/796] Add ability to register a jvm shutdown hook to close app Add ability to register a jvm shutdown hook to close app Differential Revision: https://phabricator.twitter.biz/D1229286 --- util-app/src/main/scala/com/twitter/app/App.scala | 7 +++++++ .../main/scala/com/twitter/finagle/util/flags.scala | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/util-app/src/main/scala/com/twitter/app/App.scala b/util-app/src/main/scala/com/twitter/app/App.scala index 167fa1b6ec..27f6fd50e0 100644 --- a/util-app/src/main/scala/com/twitter/app/App.scala +++ b/util-app/src/main/scala/com/twitter/app/App.scala @@ -10,7 +10,9 @@ import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicReference import scala.collection.mutable import scala.jdk.CollectionConverters._ +import scala.sys.ShutdownHookThread import scala.util.control.NonFatal +import com.twitter.finagle.util.enableJvmShutdownHook /** * A composable application trait that includes flag parsing as well @@ -444,6 +446,11 @@ trait App extends ClosableOnce with CloseOnceAwaitably with Lifecycle { catch { case _: NoSuchMethodException => None } + if (enableJvmShutdownHook()) { + ShutdownHookThread { + Await.result(close(defaultCloseGracePeriod)) + } + } // Invoke main() if it exists. mainMethod.foreach { method => diff --git a/util-app/src/main/scala/com/twitter/finagle/util/flags.scala b/util-app/src/main/scala/com/twitter/finagle/util/flags.scala index 62a4d60387..eedcb9af1c 100644 --- a/util-app/src/main/scala/com/twitter/finagle/util/flags.scala +++ b/util-app/src/main/scala/com/twitter/finagle/util/flags.scala @@ -1,6 +1,7 @@ package com.twitter.finagle.util import com.twitter.app +import com.twitter.app.GlobalFlag /** * A deny list of implementations to ignore. Keys are the fully qualified class names. @@ -44,3 +45,12 @@ object loadServiceIgnoredPaths Seq.empty[String], "Additional packages to be excluded from recursive directory scan" ) + +/** + * This flag enables another way for graceful shutdown. + */ +object enableJvmShutdownHook + extends GlobalFlag[Boolean]( + false, + "Registers a JVM shutdown hook which invokes close()" + ) From 8842c086e90c957c984d8b25cc6c5274a40cc773 Mon Sep 17 00:00:00 2001 From: Benjamin Erichsen Date: Tue, 1 Jul 2025 16:22:16 +0000 Subject: [PATCH 791/796] util: Add a jvm shutdown hook to call app.quit() Differential Revision: https://phabricator.twitter.biz/D1236932 --- CHANGELOG.rst | 2 +- .../src/main/scala/com/twitter/finagle/util/flags.scala | 2 +- util-app/src/test/scala/com/twitter/app/AppTest.scala | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 38368363c2..266e2e3281 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,7 +25,7 @@ Runtime Behavior Changes and stack trace are printed to stderr in addition to the existing stdout. ``PHAB_ID=D1116753`` * util-jvm: Memory manager & pool metrics are labelled dimensionally rather than hierarchically when exported as dimensional metrics. ``PHAB_ID=D1218090`` - +* util-app: Enables a jvm shutdown hook to call app.quit(). ``PHAB_ID=D1236932`` 23.11.0 ------- diff --git a/util-app/src/main/scala/com/twitter/finagle/util/flags.scala b/util-app/src/main/scala/com/twitter/finagle/util/flags.scala index eedcb9af1c..8bab4a7f38 100644 --- a/util-app/src/main/scala/com/twitter/finagle/util/flags.scala +++ b/util-app/src/main/scala/com/twitter/finagle/util/flags.scala @@ -51,6 +51,6 @@ object loadServiceIgnoredPaths */ object enableJvmShutdownHook extends GlobalFlag[Boolean]( - false, + true, "Registers a JVM shutdown hook which invokes close()" ) diff --git a/util-app/src/test/scala/com/twitter/app/AppTest.scala b/util-app/src/test/scala/com/twitter/app/AppTest.scala index a6c9eedcdb..f86edb7770 100644 --- a/util-app/src/test/scala/com/twitter/app/AppTest.scala +++ b/util-app/src/test/scala/com/twitter/app/AppTest.scala @@ -5,11 +5,13 @@ import com.twitter.app.lifecycle.Event import com.twitter.app.lifecycle.Observer import com.twitter.app.LoadService.Binding import com.twitter.conversions.DurationOps._ +import com.twitter.finagle.util.enableJvmShutdownHook import com.twitter.util._ import java.util.concurrent.ConcurrentLinkedQueue import scala.jdk.CollectionConverters._ import scala.language.reflectiveCalls import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.BeforeAndAfterEach class TestApp(f: () => Unit) extends App { var reason: Option[String] = None @@ -61,7 +63,12 @@ trait ErrorOnExitApp extends App { } } -class AppTest extends AnyFunSuite { +class AppTest extends AnyFunSuite with BeforeAndAfterEach { + + override protected def beforeEach(): Unit = { + enableJvmShutdownHook.parse("false") + } + test("App: make sure system.exit called on exception from main") { val test1 = new TestApp(() => throw new RuntimeException("simulate main failing")) From d9f19aa086edbc04b7a6a4fcb57ed71463521c60 Mon Sep 17 00:00:00 2001 From: Eugene Slutsky Date: Thu, 7 Aug 2025 17:48:47 +0000 Subject: [PATCH 792/796] no need in java 8 Differential Revision: https://phabricator.twitter.biz/D1247374 --- util-validator/src/main/scala/com/twitter/util/validation/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/BUILD index 917474ec39..b715b02ab6 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/BUILD @@ -1,7 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - platform = "java8-do-not-use", + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-core", From 33effb447324284bc21f2bb8c97c866d50213532 Mon Sep 17 00:00:00 2001 From: Eugene Slutsky Date: Fri, 8 Aug 2025 17:27:31 +0000 Subject: [PATCH 793/796] removing jdk 8 Differential Revision: https://phabricator.twitter.biz/D1247464 --- .../validation/internal/AnnotationFactory.scala | 16 +++++++++++++--- .../com/twitter/util/validation/internal/BUILD | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/AnnotationFactory.scala b/util-validator/src/main/scala/com/twitter/util/validation/internal/AnnotationFactory.scala index c84eff1458..ede256adc0 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/AnnotationFactory.scala +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/AnnotationFactory.scala @@ -1,9 +1,10 @@ package com.twitter.util.validation.internal +import org.hibernate.validator.internal.util.annotation.AnnotationDescriptor + import java.lang.annotation.Annotation import java.lang.reflect.Method import java.util -import sun.reflect.annotation.AnnotationParser private[validation] object AnnotationFactory { @@ -11,7 +12,6 @@ private[validation] object AnnotationFactory { * Creates a new Annotation (AnnotationProxy) instance of * the given annotation type with the custom values applied. * - * @see [[AnnotationParser.annotationForMap]] */ def newInstance[A <: Annotation]( annotationType: Class[A], @@ -33,6 +33,16 @@ private[validation] object AnnotationFactory { val key = keysIterator.next() attributes.put(key, customValues(key).asInstanceOf[AnyRef]) } - AnnotationParser.annotationForMap(annotationType, attributes).asInstanceOf[A] + annotationForMap(annotationType, attributes).asInstanceOf[A] + } + + private def annotationForMap[A <: Annotation]( + annotationClass: Class[A], + memberValues: util.Map[String, AnyRef] + ): Annotation = { + val ad: AnnotationDescriptor[A] = + new AnnotationDescriptor.Builder[A](annotationClass, memberValues).build() + org.hibernate.validator.internal.util.annotation.AnnotationFactory + .create(ad) } } diff --git a/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD index 63a2a193e6..577f8846a6 100644 --- a/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD +++ b/util-validator/src/main/scala/com/twitter/util/validation/internal/BUILD @@ -1,7 +1,7 @@ scala_library( sources = ["*.scala"], compiler_option_sets = ["fatal_warnings"], - platform = "java8-do-not-use", + platform = "java8", provides = scala_artifact( org = "com.twitter", name = "util-validation-internal", From 1a5a53dad8ed1a83f845f6a92a8b0cb7d18c3a8c Mon Sep 17 00:00:00 2001 From: Jaspreet Sandhu Date: Tue, 26 Aug 2025 17:44:19 +0000 Subject: [PATCH 794/796] [Loglens] Add overrideFatalWithError flag to LoglensAppender **Problem**: There is an inconsistency in the way a `FileAppender` and a `LoglensHandler` produce levels for log records. - LoglensHandler and LoglensFormatter use the `java.util.logging` package to build log records and transform them to `com.twitter.logging`, where the level `ERROR`/`SEVERE` is enumerated to be equivalent to level `FATAL`. - Conversely, a FileAppender uses the `ch.qos.logback` package to produce records, where events with level `ERROR` are not transformed, and are written directly. **Solution**: Add a flag for users to include in their `logback.xml` file to optionally override `FATAL` logs with the `ERROR` level, so that records sent to file and those sent to ClickHouse are consistent. Differential Revision: https://phabricator.twitter.biz/D1250623 --- .../main/scala/com/twitter/logging/Formatter.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala index 3a1769df7b..725d394101 100644 --- a/util-logging/src/main/scala/com/twitter/logging/Formatter.scala +++ b/util-logging/src/main/scala/com/twitter/logging/Formatter.scala @@ -17,9 +17,13 @@ package com.twitter.logging import com.twitter.util.TwitterDateFormat -import java.text.{MessageFormat, DateFormat} +import java.text.MessageFormat +import java.text.DateFormat import java.util.regex.Pattern -import java.util.{Date, GregorianCalendar, TimeZone, logging => javalog} +import java.util.Date +import java.util.GregorianCalendar +import java.util.TimeZone +import java.util.{logging => javalog} import scala.collection.mutable import java.{util => ju} @@ -125,14 +129,15 @@ class Formatter( /** * Return the string representation of a given log level's name */ - def formatLevelName(level: javalog.Level): String = { + def formatLevelName(level: javalog.Level, overrideFatalWithError: Boolean = false): String = { level match { case x: Level => x.name case x: javalog.Level => Logger.levels.get(x.intValue) match { case None => "%03d".format(x.intValue) - case Some(level) => level.name + case Some(level) => + if (level == Level.FATAL && overrideFatalWithError) "ERROR" else level.name } } } From f8735d20dc18acb728fe89bf0373a744e46ae67d Mon Sep 17 00:00:00 2001 From: Shatil Rafiullah Date: Wed, 15 Oct 2025 23:07:36 +0000 Subject: [PATCH 795/796] util/util-app: Lint file(s) prior to making changes JIRA Issues: KUBE-3034 Differential Revision: https://phabricator.twitter.biz/D1268019 --- .../scala/com/twitter/app/Flaggable.scala | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/util-app/src/main/scala/com/twitter/app/Flaggable.scala b/util-app/src/main/scala/com/twitter/app/Flaggable.scala index d84696c59c..624b9dfe8a 100644 --- a/util-app/src/main/scala/com/twitter/app/Flaggable.scala +++ b/util-app/src/main/scala/com/twitter/app/Flaggable.scala @@ -1,18 +1,21 @@ package com.twitter.app -import com.twitter.util.{Duration, StorageUnit, Time, TimeFormat} +import com.twitter.util.Duration +import com.twitter.util.StorageUnit +import com.twitter.util.Time +import com.twitter.util.TimeFormat import java.io.File import java.lang.reflect.Type -import java.lang.{ - Boolean => JBoolean, - Double => JDouble, - Float => JFloat, - Integer => JInteger, - Long => JLong -} +import java.lang.{Boolean => JBoolean} +import java.lang.{Double => JDouble} +import java.lang.{Float => JFloat} +import java.lang.{Integer => JInteger} +import java.lang.{Long => JLong} import java.net.InetSocketAddress import java.time.LocalTime -import java.util.{List => JList, Map => JMap, Set => JSet} +import java.util.{List => JList} +import java.util.{Map => JMap} +import java.util.{Set => JSet} import scala.jdk.CollectionConverters._ import scala.reflect.ClassTag From 8ea6954b5f8afc602fc5abe66e3987272305ec32 Mon Sep 17 00:00:00 2001 From: Jillian Crossley Date: Mon, 8 Dec 2025 13:32:36 +0000 Subject: [PATCH 796/796] util/util-core: Introduce ImmediateValueFuture (continuations execute immediately) Problem The transforming continuations (ie not side-effects) of `Future.const` go through the scheduler. While this is safe in all use cases, it can have a negative impact on execution times under certain scenarios due to reordering. Solution Introduce `ImmediateValueFuture` that bypasses the scheduler in all cases and executes continuations immediately. This Future type is not safe to use in all scenarios and we outline this in the doc. Differential Revision: https://phabricator.twitter.biz/D1285047 --- .../twitter/util/ImmediateValueFuture.scala | 72 ++++++ .../util/ImmediateValueFutureTest.scala | 221 ++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 util-core/src/main/scala/com/twitter/util/ImmediateValueFuture.scala create mode 100644 util-core/src/test/scala/com/twitter/util/ImmediateValueFutureTest.scala diff --git a/util-core/src/main/scala/com/twitter/util/ImmediateValueFuture.scala b/util-core/src/main/scala/com/twitter/util/ImmediateValueFuture.scala new file mode 100644 index 0000000000..2bd5fe26fb --- /dev/null +++ b/util-core/src/main/scala/com/twitter/util/ImmediateValueFuture.scala @@ -0,0 +1,72 @@ +package com.twitter.util + +import scala.util.control.NonFatal + +/** + * Successful Future that contains a value. Transformations are executed immediately (don't go + * through the scheduler). Unlike `Future.const`, Future recursion *will* grow the stack (see + * `ImmediateValueFutureTest` for an example of this) -- it is therefore extremely important that + * you understand the full context of how this Future will be used in order to avoid this. + * + * DO NOT USE THIS without thoroughly understanding the risks! + */ +private[twitter] class ImmediateValueFuture[A](result: A) extends Future[A] { + + private[this] val ReturnResult = Return(result) + + def respond(f: Try[A] => Unit): Future[A] = { + val saved = Local.save() + try { + f(ReturnResult) + } catch Monitor.catcher + finally { + Local.restore(saved) + } + this + } + + override def proxyTo[B >: A](other: Promise[B]): Unit = { + other.update(ReturnResult) + } + + def raise(interrupt: Throwable): Unit = () + + override def rescue[B >: A](rescueException: PartialFunction[Throwable, Future[B]]): Future[B] = { + this + } + + protected def transformTry[B](f: Try[A] => Try[B]): Future[B] = { + val saved = Local.save() + try { + f(ReturnResult) match { + case Return(result) => new ImmediateValueFuture(result) + case t @ Throw(_) => Future.const(t) + } + } catch { + case NonFatal(e) => Future.const(Throw(e)) + } finally { + Local.restore(saved) + } + } + + def transform[B](f: Try[A] => Future[B]): Future[B] = { + val saved = Local.save() + try { + f(ReturnResult) + } catch { + case NonFatal(e) => Future.const(Throw(e)) + } finally { + Local.restore(saved) + } + } + + def poll: Option[Try[A]] = Some(ReturnResult) + + override def toString: String = s"ImmediateValueFuture($result)" + + def ready(timeout: Duration)(implicit permit: Awaitable.CanAwait): this.type = this + + def result(timeout: Duration)(implicit permit: Awaitable.CanAwait): A = result + + def isReady(implicit permit: Awaitable.CanAwait): Boolean = true +} diff --git a/util-core/src/test/scala/com/twitter/util/ImmediateValueFutureTest.scala b/util-core/src/test/scala/com/twitter/util/ImmediateValueFutureTest.scala new file mode 100644 index 0000000000..bc650e7765 --- /dev/null +++ b/util-core/src/test/scala/com/twitter/util/ImmediateValueFutureTest.scala @@ -0,0 +1,221 @@ +package com.twitter.util + +import com.twitter.concurrent.Scheduler +import java.util.concurrent.atomic.AtomicBoolean +import org.scalatest.funsuite.AnyFunSuite + +class ImmediateValueFutureTest extends AnyFunSuite { + + // The following is intended to demonstrate the differing recursive behaviour of `ConstFuture` + // and `ImmediateValueFuture`. `ConstFuture` essentially does tail-call elimination which means + // the stack will not grow during a recursive call. `ImmediateValueFuture` does not, so the stack + // will grow with each call! + def recurseAndGetStackSizes(f: Future[Unit]): Seq[Int] = { + val stop = new AtomicBoolean(false) + + @volatile var loopStackSizes: Seq[Int] = Seq.empty + + def loop(): Future[Unit] = { + if (stop.get) { + Future.Done + } else { + f.flatMap { _ => + loopStackSizes = loopStackSizes :+ new Throwable().getStackTrace.length + loop() + } + } + } + + FuturePool.unboundedPool { + loop() + } + + while (loopStackSizes.size < 10) {} + stop.set(true) + loopStackSizes + } + + test("ConstFuture recursion does not grow the stack") { + val loopStackSizes = recurseAndGetStackSizes(Future.const(Return[Unit](()))) + assert(loopStackSizes.forall(_ == loopStackSizes.head)) + } + + test("ImmediateValueFuture recursion does grow the stack") { + val loopStackSizes = recurseAndGetStackSizes(new ImmediateValueFuture(())) + assert(loopStackSizes.zip(loopStackSizes.tail).forall { case (a, b) => a < b }) + } + + test(s"ImmediateValueFuture.interruptible should do nothing") { + val f = new ImmediateValueFuture(()) + val i = f.interruptible() + i.raise(new Exception()) + assert(f.poll.contains(Return[Unit](()))) + assert(i.poll.contains(Return[Unit](()))) + } + + test(s"ImmediateValueFuture should propagate locals and restore original context in `respond`") { + val local = new Local[Int] + val f = new ImmediateValueFuture(111) + + var ran = 0 + local() = 1010 + + f.ensure { + assert(local().contains(1010)) + local() = 1212 + f.ensure { + assert(local().contains(1212)) + local() = 1313 + ran += 1 + } + assert(local().contains(1212)) + ran += 1 + } + + assert(local().contains(1010)) + assert(ran == 2) + } + + test( + s"ImmediateValueFuture should propagate locals and restore original context in `transform`") { + val local = new Local[Int] + val f = new ImmediateValueFuture(111) + + var ran = 0 + local() = 1010 + + f.transform { tryRes => + assert(local().contains(1010)) + local() = 1212 + f.transform { tryRes => + assert(local().contains(1212)) + local() = 1313 + ran += 1 + Future.const(tryRes) + } + assert(local().contains(1212)) + ran += 1 + Future.const(tryRes) + } + + assert(local().contains(1010)) + assert(ran == 2) + } + + test(s"ImmediateValueFuture should propagate locals and restore original context in `map`") { + val local = new Local[Int] + val f = new ImmediateValueFuture(111) + + var ran = 0 + local() = 1010 + + f.map { i => + assert(local().contains(1010)) + local() = 1212 + f.map { i => + assert(local().contains(1212)) + local() = 1313 + ran += 1 + i + } + assert(local().contains(1212)) + ran += 1 + i + } + + assert(local().contains(1010)) + assert(ran == 2) + } + + test(s"ImmediateValueFuture should not delay execution") { + val numDispatchesBefore = Scheduler().numDispatches + val f = new ImmediateValueFuture(111) + + var count = 0 + f.onSuccess { _ => + assert(count == 0) + f.ensure { + assert(count == 0) + count += 1 + } + + assert(count == 1) + count += 1 + } + + assert(count == 2) + assert(Scheduler().numDispatches == numDispatchesBefore) + } + + test(s"ImmediateValueFuture side effects should be monitored") { + val inner = new ImmediateValueFuture(111) + val exc = new Exception("a raw exception") + + var monitored = false + + val monitor = new Monitor { + override def handle(exc: Throwable): Boolean = { + monitored = true + true + } + } + + Monitor.using(monitor) { + val f = inner.respond { _ => + throw exc + } + + assert(f.poll.contains(Return(111))) + assert(monitored == true) + } + } + + test("ImmediateValueFuture.rescue returns self") { + val f = new ImmediateValueFuture(111) + val r = f.rescue { + case e: Exception => Future.value(1) + } + + assert(f eq r) + } + + test("ImmediateValueFuture.map returns ImmediateValueFuture if f does not throw") { + val f1 = new ImmediateValueFuture(111) + + val f2 = f1.map { _ => + "hello" + } + + assert(f2.isInstanceOf[ImmediateValueFuture[String]]) + } + + test("ImmediateValueFuture.map returns Future.exception if f throws") { + val f1 = new ImmediateValueFuture(111) + + val f2 = f1.map { _ => + throw new Exception("boom!") + } + + intercept[Exception](Await.result(f2)) + } + + test("ImmediateValueFuture.flatMap returns Future.exception if f returns exceptional Future") { + val f1 = new ImmediateValueFuture(111) + + val f2 = f1.flatMap { _ => + Future.exception(new Exception("boom!")) + } + + intercept[Exception](Await.result(f2)) + } + + test("ImmediateValueFuture.transform returns Future.exception if f throws") { + val f1 = new ImmediateValueFuture(111) + + val f2 = f1.transform { _ => + throw new Exception("boom!") + } + + intercept[Exception](Await.result(f2)) + } +}